From 114df77c6bd94b39c553750dede10238487f427e Mon Sep 17 00:00:00 2001
From: Haibo Lin <hlin@redhat.com>
Date: Fri, 1 Nov 2019 15:23:24 +0800
Subject: [PATCH] Retry watching koji tasks on server outage

Fixes: https://pagure.io/pungi/issue/1285
JIRA: COMPOSE-3896
Signed-off-by: Haibo Lin <hlin@redhat.com>
---
 pungi/wrappers/kojiwrapper.py | 10 +++++++---
 tests/test_koji_wrapper.py    | 16 ++++++++++++++++
 2 files changed, 23 insertions(+), 3 deletions(-)

diff --git a/pungi/wrappers/kojiwrapper.py b/pungi/wrappers/kojiwrapper.py
index e8b54e1d..f18f3e38 100644
--- a/pungi/wrappers/kojiwrapper.py
+++ b/pungi/wrappers/kojiwrapper.py
@@ -308,6 +308,10 @@ class KojiWrapper(object):
         """Checks if output indicates connection error."""
         return re.search('error: failed to connect\n$', output)
 
+    def _has_offline_error(self, output):
+        """Check if output indicates server offline."""
+        return re.search('koji: ServerOffline:', output)
+
     def _wait_for_task(self, task_id, logfile=None, max_retries=None):
         """Tries to wait for a task to finish. On connection error it will
         retry with `watch-task` command.
@@ -318,8 +322,8 @@ class KojiWrapper(object):
         while True:
             retcode, output = run(cmd, can_fail=True, logfile=logfile, universal_newlines=True)
 
-            if retcode == 0 or not self._has_connection_error(output):
-                # Task finished for reason other than connection error.
+            if retcode == 0 or not (self._has_connection_error(output) or self._has_offline_error(output)):
+                # Task finished for reason other than connection error or server offline error.
                 return retcode, output
 
             attempt += 1
@@ -345,7 +349,7 @@ class KojiWrapper(object):
                                % (" ".join(command), output))
         task_id = int(match.groups()[0])
 
-        if retcode != 0 and self._has_connection_error(output):
+        if retcode != 0 and (self._has_connection_error(output) or self._has_offline_error(output)):
             retcode, output = self._wait_for_task(task_id, logfile=log_file, max_retries=max_retries)
 
         return {
diff --git a/tests/test_koji_wrapper.py b/tests/test_koji_wrapper.py
index 80282f31..d47c9f67 100644
--- a/tests/test_koji_wrapper.py
+++ b/tests/test_koji_wrapper.py
@@ -701,6 +701,22 @@ class RunBlockingCmdTest(KojiWrapperBaseTestCase):
                                     can_fail=True, logfile=None, universal_newlines=True)])
         self.assertEqual(sleep.mock_calls, [mock.call(i * 10) for i in range(1, 2)])
 
+    @mock.patch('pungi.wrappers.kojiwrapper.run')
+    def test_server_offline_and_retry(self, run):
+        output = 'Created task: 1234\nkoji: ServerOffline:'
+        retry = 'Created task: 1234\nOook\n'
+        run.side_effect = [(1, output), (0, retry)]
+
+        result = self.koji.run_blocking_cmd('cmd')
+
+        self.assertDictEqual(result, {'retcode': 0, 'output': retry, 'task_id': 1234})
+        self.assertEqual(run.mock_calls,
+                         [mock.call('cmd', can_fail=True, logfile=None, env=None,
+                                    universal_newlines=True),
+                          mock.call(['koji', '--profile=custom-koji', 'watch-task', '1234'],
+                                    can_fail=True, logfile=None, universal_newlines=True)])
+
+
 
 RPM_QA_QF_OUTPUT = """
 cjkuni-uming-fonts-0.2.20080216.1-56.fc23.noarch