poikilos
/
EnlivenMinetest
mirror of https://github.com/poikilos/EnlivenMinetest
1
0
Fork 0
Code Issues 2 Projects Releases Wiki Activity
Browse Source

Add a flat cache directory option. Move the caches_path option to the options dict. Make the 410 error message uniform for the purpose of the caller checking for one.

master
poikilos 3 years ago
parent
a40fda6296
commit
eabd2df697
1 changed files with 91 additions and 37 deletions
Whitespace
Show all changes Ignore whitespace when comparing lines Ignore changes in amount of whitespace Ignore changes in whitespace at EOL
Unified View
Diff Options
Show Stats Download Patch File Download Diff File
  1. 128
      utilities/enissue.py

128
utilities/enissue.py
View File

@ -157,6 +157,14 @@ def debug(msg):
if verbose: if verbose:
error("[debug] " + msg) error("[debug] " + msg)
def set_verbose(on):
global verbose
if on is True:
verbose = True
elif on is False:
verbose = True
else:
raise ValueError("on must be True or False.")
def decode_safe(b): def decode_safe(b):
try: try:
@ -282,8 +290,7 @@ class Repo:
def __init__( def __init__(
self, self,
options, options
caches_path=None,
): ):
''' '''
Keyword arguments: Keyword arguments:
@ -321,15 +328,23 @@ class Repo:
search_results_key -- If the URL described by search_results_key -- If the URL described by
search_issues_url_fmt returns a dict, specify the key in search_issues_url_fmt returns a dict, specify the key in
the dict that is a list of issues. the dict that is a list of issues.
caches_path -- Store cached json files here: Specifically, in an caches_path -- Store cached json files here: Specifically,
"issues" directory or other directory under the user and in an "issues" directory or other directory under the
repo directory. For example, if caches_path is None and uses user and repo directory. For example, if caches_path is
the default ~/.cache/enissue, the numbered json files None and uses the default ~/.cache/enissue, the
(and numbered folders containing timeline.json or other numbered json files (and numbered folders containing
connected data) for issues will appear in timeline.json or other connected data) for issues will
"~/.cache/enissue/poikilos/EnlivenMinetest/issues". To set appear in
it later, use the setCachesPath method. "~/.cache/enissue/poikilos/EnlivenMinetest/issues". To
set it later, use the setCachesPath method.
This is the default behavior, and the default is
os.path.join(userprofile, ".cache").
single_cache -- Store all issue data in one flat directory
structure (except for issues and other subdirectories
which mimic the web API routes). Only use this option
if you set a different single_cache for each repo!
''' '''
self.ERROR_410 = "Error 410"
repo_url = options.get('repo_url') repo_url = options.get('repo_url')
debug("* using URL {}".format(repo_url)) debug("* using URL {}".format(repo_url))
if repo_url is None: if repo_url is None:
@ -414,12 +429,27 @@ class Repo:
" required. The URL \"{}\" is not" " required. The URL \"{}\" is not"
" accessible, so you may have exceeded the" " accessible, so you may have exceeded the"
" rate limit and be blocked temporarily") " rate limit and be blocked temporarily")
if caches_path is None:
caches_path = os.path.join(Repo.profile, ".cache", _caches_path = options.get('caches_path')
"enissue") _single_cache = options.get('single_cache')
if _caches_path is not None:
if _single_cache is not None:
raise ValueError("You can't set both caches_path and"
" single_cache. The caches_path option"
" creates a"
" <caches_path>/<user>/<repo>/"
" structure.")
self.setCachesPath(_caches_path, flat=False)
elif _single_cache is not None:
self.setCachesPath(_single_cache, flat=True)
else:
_caches_path = os.path.join(Repo.profile, ".cache",
"enissue")
self.setCachesPath(_caches_path, flat=False)
del _caches_path
del _single_cache
self.remote_user = remote_user self.remote_user = remote_user
self.repo_name = repo_name self.repo_name = repo_name
self.setCachesPath(caches_path)
self.search_results_key = options.get('search_results_key') self.search_results_key = options.get('search_results_key')
self.page = options.get('page') self.page = options.get('page')
@ -459,18 +489,26 @@ class Repo:
self.issues = None self.issues = None
self.last_query_s = None self.last_query_s = None
def setCachesPath(self, caches_path): def setCachesPath(self, path, flat=True):
''' '''
This directory will contain <remote_user>/<repo_name>/ This repo cache directory will be <remote_user>/<repo_name>/
which will contain issues/ and potentially other directories unless flat is True, in which case it will be path.
that mimic the API web URL structure (See _get_issues code for
subdirectories and files). The repo cache will contain issues/ and potentially other
directories that mimic the API web URL structure (See
_get_issues code for subdirectories and files).
''' '''
self.caches_path = caches_path if path is None:
self.c_remote_user_path = os.path.join(self.caches_path, raise ValueError("path must not be None")
self.remote_user) self.caches_path = path
self.c_repo_path = os.path.join(self.c_remote_user_path, if flat:
self.repo_name) self.c_remote_user_path = self.caches_path
self.c_repo_path = self.caches_path
else:
self.c_remote_user_path = os.path.join(self.caches_path,
self.remote_user)
self.c_repo_path = os.path.join(self.c_remote_user_path,
self.repo_name)
def _get_issues(self, options, query=None, issue_no=None, def _get_issues(self, options, query=None, issue_no=None,
search_terms=[]): search_terms=[]):
@ -515,6 +553,7 @@ class Repo:
ValueError if query is not None and issue_no is not None. ValueError if query is not None and issue_no is not None.
''' '''
quiet = options.get('quiet') is True
debug("get_issues...") debug("get_issues...")
debug(" options={}".format(options)) debug(" options={}".format(options))
debug(" query={}".format(query)) debug(" query={}".format(query))
@ -574,6 +613,11 @@ class Repo:
# print("c_issues_name: {}".format(c_issues_name)) # print("c_issues_name: {}".format(c_issues_name))
# print("query_part: {}".format(query_part)) # print("query_part: {}".format(query_part))
if self.c_repo_path is None:
raise RuntimeError("self.c_repo_path must not be None."
" The __init__ method should call"
" setCachesPath so this should never"
" happen!")
c_issues_path = os.path.join(self.c_repo_path, c_issues_name) c_issues_path = os.path.join(self.c_repo_path, c_issues_name)
make_list = False make_list = False
c_path = c_issues_path c_path = c_issues_path
@ -639,10 +683,12 @@ class Repo:
max_cache_d_s = "never_expire" max_cache_d_s = "never_expire"
expires_s = "never_expire" expires_s = "never_expire"
if (refresh is not True) and (is_fresh or never_expire): if (refresh is not True) and (is_fresh or never_expire):
print(p+"Loading cache: \"{}\"".format(c_path)) if not quiet:
print(p+"Loading cache: \"{}\"".format(c_path))
debug(p+"Cache time limit: {}".format(max_cache_delta)) debug(p+"Cache time limit: {}".format(max_cache_delta))
debug(p+" for URL: {}".format(query_s)) debug(p+" for URL: {}".format(query_s))
print(p+"Cache expires: {}".format(expires_s)) if not quiet:
print(p+"Cache expires: {}".format(expires_s))
with open(c_path) as json_file: with open(c_path) as json_file:
result = json.load(json_file) result = json.load(json_file)
max_issue = None max_issue = None
@ -674,25 +720,29 @@ class Repo:
return results, None return results, None
else: else:
if refresh is True: if refresh is True:
print(p+"Refreshing...".format(max_cache_delta)) if not quiet:
print(p+"Refreshing...".format(max_cache_delta))
else: else:
print(p+"Cache time limit: {}".format(max_cache_delta)) if not quiet:
print(p+"The cache has expired: \"{}\"".format( print(p+"Cache time limit: {}".format(max_cache_delta))
c_path print(p+"The cache has expired: \"{}\"".format(
)) c_path
))
else: else:
print(p+"There is no cache for \"{}\"".format( if not quiet:
c_path print(p+"There is no cache for \"{}\"".format(
)) c_path
))
try: try:
debug(p+"Query URL (query_s): {}".format(query_s)) debug(p+"Query URL (query_s): {}".format(query_s))
response = request.urlopen(query_s) response = request.urlopen(query_s)
except HTTPError as e: except HTTPError as e:
if "Error 410" in str(e): if self.ERROR_410 in str(e):
return ( return (
None, None,
"The issue was apparently deleted (response 410)." ("The issue was apparently deleted ({})."
"".format(self.ERROR_410))
) )
msg = str(e) + ": " + self.rateLimitFmt.format(query_s) msg = str(e) + ": " + self.rateLimitFmt.format(query_s)
return None, msg return None, msg
@ -720,7 +770,7 @@ class Repo:
quiet=False): quiet=False):
''' '''
This gets the cached page using the cache location This gets the cached page using the cache location
cache directory specified in self.caches_path and further cache directory specified in options['caches_path'] and further
narrowed down to self.c_repo_path then narrowed down using the narrowed down to self.c_repo_path then narrowed down using the
URL. For example, https://api.github.com/repos/poikilos/EnlivenMinetest/issues?q=page:1 URL. For example, https://api.github.com/repos/poikilos/EnlivenMinetest/issues?q=page:1
@ -730,6 +780,10 @@ class Repo:
and and
~/.cache/enissue/poikilos/EnlivenMinetest/ ~/.cache/enissue/poikilos/EnlivenMinetest/
...unless options['single_path'] is set: then there will be no
automatically-created subdirectories (except for the usual
"issues" and other directories below that one described below).
https://api.github.com/repos/poikilos/EnlivenMinetest/issues/515/timeline https://api.github.com/repos/poikilos/EnlivenMinetest/issues/515/timeline
The reactions to a timeline event are from a URL such as: The reactions to a timeline event are from a URL such as:

Write Preview
Loading…
Cancel
Save