Browse Source

Write an outline for writing issues. Always track HTTPError and trickle the error up to the caller.

master
poikilos 3 years ago
parent
commit
8fe94c3993
  1. 154
      utilities/enissue.py
  2. 58
      utilities/pyissuesyncd

154
utilities/enissue.py

@ -12,7 +12,7 @@ Known issues:
- Get attachments from GitHub
- Get inline files from GitHub (such as pasted images)
Options:")
Options:
--cache-base <dir> Set the directory for cached files.
--verbose Enable verbose mode.
--debug Enable verbose mode (same as --debug).
@ -737,15 +737,26 @@ class Repo:
try:
debug(p+"Query URL (query_s): {}".format(query_s))
response = request.urlopen(query_s)
except HTTPError as e:
if self.ERROR_410 in str(e):
except HTTPError as ex:
msg = ex.reason
if ex.code == 410:
msg = ("The issue was apparently deleted ({})."
"".format(self.ERROR_410))
return (
None,
("The issue was apparently deleted ({})."
"".format(self.ERROR_410))
{
'code': ex.code,
'reason': msg,
}
)
msg = str(e) + ": " + self.rateLimitFmt.format(query_s)
return None, msg
# msg = str(ex) + ": " + self.rateLimitFmt.format(query_s)
return (
None,
{
'code': ex.code,
'reason': msg,
}
)
response_s = decode_safe(response.read())
if not os.path.isdir(self.c_repo_path):
os.makedirs(self.c_repo_path)
@ -769,7 +780,11 @@ class Repo:
def getCachedJsonDict(self, url, refresh=False, never_expire=False,
quiet=False):
'''
This gets the cached page using the cache location
Get a cached page, unless the 2nd returned param is not None in
which case that will contain a standard web response 'code'
(RFC 2616) number and 'reason' string.
The cached page is obtained using the cache location
cache directory specified in options['caches_path'] and further
narrowed down to self.c_repo_path then narrowed down using the
URL. For example, https://api.github.com/repos/poikilos/EnlivenMinetest/issues?q=page:1
@ -910,21 +925,34 @@ class Repo:
" and will be overwritten if loads"
"".format(c_path))
result = None
# Do NOT set err NOR set to a tuple (A result
# of None means it will load from the web
# below)!
if result is not None:
return result
return result, None
res = request.urlopen(url)
data_s = decode_safe(res.read())
parent = os.path.split(c_path)[0]
if not os.path.isdir(parent):
os.makedirs(parent)
data = json.loads(data_s)
# Only save if loads didn't raise an exception.
with open(c_path, 'w') as outs:
outs.write(data_s)
debug(p+"Wrote {}".format(c_path))
try:
res = request.urlopen(url)
data_s = decode_safe(res.read())
parent = os.path.split(c_path)[0]
if not os.path.isdir(parent):
os.makedirs(parent)
data = json.loads(data_s)
# Only save if loads didn't raise an exception.
with open(c_path, 'w') as outs:
outs.write(data_s)
debug(p+"Wrote {}".format(c_path))
except HTTPError as ex:
return (
None,
{
'code': ex.code,
'reason': ex.reason,
'headers': ex.headers,
}
)
return data
return data, None
def show_issue(self, issue, refresh=False, never_expire=False):
@ -932,6 +960,21 @@ class Repo:
Display an issue dictionary as formatted text after getting the
issue body and other data from the internet. Gather all of the
additional metadata as well.
Sequential arguments:
issue -- Provide a partial issue dict such as from a list result
page that can be used to identify and obtain the full issue.
Returns:
(full_issue_dict, None)
or
(full_issue_dict, error_dict) (where error_dict is something
non-fatal such as missing timeline)
or
(None, error_dict)
where (in both cases) error_dict contains a 'reason' key
and possibly a 'code' key (standard website error number, or
else 'code' is not present).
'''
p = self.log_prefix
print("")
@ -941,12 +984,15 @@ class Repo:
print("")
issue_data = issue
html_url = issue['html_url']
issue_data = self.getCachedJsonDict(
issue_data, err = self.getCachedJsonDict(
issue["url"],
refresh=refresh,
never_expire=never_expire,
)
if err is not None:
return None, err
markdown = issue_data.get("body")
# ^ It is (always?) present but allowed to be None by GitHub!
if markdown is not None:
@ -1016,13 +1062,17 @@ class Repo:
'''
this_tmln_json_url = issue_data.get('timeline_url')
data = []
msg = None
if this_tmln_json_url is not None:
tmln_data = self.getCachedJsonDict(
tmln_data, err = self.getCachedJsonDict(
this_tmln_json_url,
refresh=refresh,
quiet=True,
never_expire=never_expire,
)
if err is not None:
msg = ("Accessing the timeline URL failed: {}"
"".format(err.get('reason')))
# Example:
# <https://api.github.com/repos/poikilos/EnlivenMinetest/
# issues/202/timeline>
@ -1038,18 +1088,30 @@ class Repo:
rn=self.repo_name,
)
if comments_url is not None:
cmts_data = self.getCachedJsonDict(
cmts_data, err = self.getCachedJsonDict(
comments_url,
refresh=refresh,
quiet=True,
never_expire=never_expire,
)
if err is not None:
msg = ("Accessing the timeline URL failed: {}"
"".format(err.get('reason')))
return (
issue_data,
{
'code': err.code,
'reason': msg,
'headers': err.headers,
}
)
data = cmts_data
else:
error("WARNING: comments={} but there is no"
" comments_url in:"
"".format(comments))
error(json.dumps(issue_data, indent=4, sort_keys=True))
msg = ("WARNING: comments={} but there is no"
" comments_url in:"
"".format(comments))
# error(msg)
# error(json.dumps(issue_data, indent=4, sort_keys=True))
for evt in data:
user = evt.get('user')
@ -1134,12 +1196,15 @@ class Repo:
# Example: <https://api.github.com/repos/poikilos/
# EnlivenMinetest/
# issues/comments/968357490/reactions>
reac_data = self.getCachedJsonDict(
reac_data, err = self.getCachedJsonDict(
reactions_url,
refresh=refresh,
quiet=True,
never_expire=never_expire,
)
if err is not None:
error("Accessing the reactions URL failed: {}"
"".format(err.get('reason')))
if reac_data is not None:
for reac in reac_data:
reac_user = reac.get('user')
@ -1181,7 +1246,13 @@ class Repo:
print("")
print("")
return True
err = None
if msg is not None:
err = {
'reason': msg,
}
return issue_data, err
def load_issues(self, options, query=None, issue_no=None,
search_terms=None):
@ -1285,6 +1356,31 @@ class Repo:
return {'issue':matching_issue, 'count':match_count}
def create_issue(self, src_issue, src_repo):
'''
Remotely create a new issue using the web API.
Sequential arguments:
src_issue -- The issue dictionary for one issue in the format of
the src_repo.
src_repo -- Provide the Repo object that originated the data
for the purpose of translating the issue format.
'''
raise NotImplementedError("create_issue")
def update_issue(self, src_issue, src_repo):
'''
Remotely update an existing issue using the web API.
Sequential arguments:
src_issue -- The issue dictionary for one issue in the format of
the src_repo.
src_repo -- Provide the Repo object that originated the data
for the purpose of translating the issue format.
'''
raise NotImplementedError("update_issue")
def main():
global verbose

58
utilities/pyissuesyncd

@ -97,16 +97,23 @@ def get_issue(repo, options, issue_no):
)
if results is None:
if err is not None:
if repo.ERROR_410 in err:
# The issue was deleted
if err.get('code') == 410:
# error("The issue was deleted")
pass
elif err.get('code') == 404:
# error("The issue doesn't exist")
pass
error(err)
return None, err
else:
msg = ("Unknown error: Results should not be None unless"
" there is an error (issue_no={})."
"".format(issue_no))
return None, msg
return (
None,
{
'reason': msg,
}
)
elif not isinstance(results, list):
raise RuntimeError("Results must be a list even if there is"
" only one result.")
@ -133,20 +140,37 @@ def get_issue(repo, options, issue_no):
def start_issuesyncd(src_options, dst_options):
# src_never_expire = src_options.get('never_expire') is True
issue_no = 1
non_issue = 1
issue_no = non_issue - 1
# while True:
src_repo = Repo(src_options)
src_issue, err = get_issue(src_repo, src_options, issue_no)
print("src_issue:")
print(json.dumps(src_issue, indent=2))
enissue.set_verbose(True)
dst_repo = Repo(dst_options)
dst_issue, err = get_issue(dst_repo, dst_options, issue_no)
print("dst_issue:")
print(json.dumps(dst_issue, indent=2))
issue_no += 1
while issue_no < non_issue: # for debug only
issue_no += 1
src_repo = Repo(src_options)
src_issue, err = get_issue(src_repo, src_options, issue_no)
if err is not None:
error("Error accessing source issue {}: {}: {}"
"".format(issue_no, err.get('code'),
err.get('reason')))
continue
print("[pyissuesyncd] src_issue:")
print(json.dumps(src_issue, indent=2))
enissue.set_verbose(True)
dst_repo = Repo(dst_options)
dst_issue, err = get_issue(dst_repo, dst_options, issue_no)
print("[pyissuesyncd] dst_issue:")
print(json.dumps(dst_issue, indent=2))
if err is not None:
if err.get('code') == 404:
dst_repo.create_issue(src_issue, src_repo)
continue
error("Error accessing destination issue {}: {}: {}"
"".format(issue_no, err.get('code'),
err.get('reason')))
continue
dst_repo.update_issue(src_issue, src_repo)
def usage():
print(__doc__)

Loading…
Cancel
Save