almalinux/pungi
14
1
Fork 0
Code Issues Pull Requests Releases Activity

Retry watching koji tasks on server outage

Browse Source 
Fixes: https://pagure.io/pungi/issue/1285
JIRA: COMPOSE-3896
Signed-off-by: Haibo Lin <hlin@redhat.com>
This commit is contained in:
Haibo Lin 2019-11-01 15:23:24 +08:00
parent 5751de1096
commit 114df77c6b
2 changed files with 23 additions and 3 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

10
pungi/wrappers/kojiwrapper.py
View File

@ -308,6 +308,10 @@ class KojiWrapper(object):
"""Checks if output indicates connection error.""" """Checks if output indicates connection error."""
return re.search('error: failed to connect\n$', output) return re.search('error: failed to connect\n$', output)
def _has_offline_error(self, output):
"""Check if output indicates server offline."""
return re.search('koji: ServerOffline:', output)
def _wait_for_task(self, task_id, logfile=None, max_retries=None): def _wait_for_task(self, task_id, logfile=None, max_retries=None):
"""Tries to wait for a task to finish. On connection error it will """Tries to wait for a task to finish. On connection error it will
retry with `watch-task` command. retry with `watch-task` command.
@ -318,8 +322,8 @@ class KojiWrapper(object):
while True: while True:
retcode, output = run(cmd, can_fail=True, logfile=logfile, universal_newlines=True) retcode, output = run(cmd, can_fail=True, logfile=logfile, universal_newlines=True)
if retcode == 0 or not self._has_connection_error(output): if retcode == 0 or not (self._has_connection_error(output) or self._has_offline_error(output)):
# Task finished for reason other than connection error. # Task finished for reason other than connection error or server offline error.
return retcode, output return retcode, output
attempt += 1 attempt += 1
@ -345,7 +349,7 @@ class KojiWrapper(object):
% (" ".join(command), output)) % (" ".join(command), output))
task_id = int(match.groups()[0]) task_id = int(match.groups()[0])
if retcode != 0 and self._has_connection_error(output): if retcode != 0 and (self._has_connection_error(output) or self._has_offline_error(output)):
retcode, output = self._wait_for_task(task_id, logfile=log_file, max_retries=max_retries) retcode, output = self._wait_for_task(task_id, logfile=log_file, max_retries=max_retries)
return { return {

16
tests/test_koji_wrapper.py
View File

@ -701,6 +701,22 @@ class RunBlockingCmdTest(KojiWrapperBaseTestCase):
can_fail=True, logfile=None, universal_newlines=True)]) can_fail=True, logfile=None, universal_newlines=True)])
self.assertEqual(sleep.mock_calls, [mock.call(i * 10) for i in range(1, 2)]) self.assertEqual(sleep.mock_calls, [mock.call(i * 10) for i in range(1, 2)])
@mock.patch('pungi.wrappers.kojiwrapper.run')
def test_server_offline_and_retry(self, run):
output = 'Created task: 1234\nkoji: ServerOffline:'
retry = 'Created task: 1234\nOook\n'
run.side_effect = [(1, output), (0, retry)]
result = self.koji.run_blocking_cmd('cmd')
self.assertDictEqual(result, {'retcode': 0, 'output': retry, 'task_id': 1234})
self.assertEqual(run.mock_calls,
[mock.call('cmd', can_fail=True, logfile=None, env=None,
universal_newlines=True),
mock.call(['koji', '--profile=custom-koji', 'watch-task', '1234'],
can_fail=True, logfile=None, universal_newlines=True)])
RPM_QA_QF_OUTPUT = """ RPM_QA_QF_OUTPUT = """
cjkuni-uming-fonts-0.2.20080216.1-56.fc23.noarch cjkuni-uming-fonts-0.2.20080216.1-56.fc23.noarch