Add @retry decorator and use it to retry connection on PDC on IOError and in SCM's retry_run.

Signed-off-by: Jan Kaluza <jkaluza@redhat.com>

pungi/phases/pkgset/sources/source_koji.py

@@ -21,7 +21,7 @@ import json
 import pungi.wrappers.kojiwrapper
 import pungi.phases.pkgset.pkgsets
 from pungi.arch import get_valid_arches
-from pungi.util import is_arch_multilib
+from pungi.util import is_arch_multilib, retry
 
 from pungi.phases.pkgset.common import create_arch_repos, create_global_repo, populate_arch_pkgsets
 
@@ -62,6 +62,7 @@ def variant_dict_from_str(module_str):
     return module_info
 
 
+@retry(wait_on=IOError)
 def get_module(session, module_info, strict=False):
     """
     :param session : PDCClient instance

pungi/util.py

@@ -28,6 +28,7 @@ import contextlib
 import traceback
 import tempfile
 import time
+import functools
 
 from kobo.shortcuts import run, force_list
 from productmd.common import get_major_version
@@ -756,3 +757,22 @@ def version_generator(compose, gen):
     if gen and gen[0] == '!':
         raise RuntimeError("Unknown version generator '%s'" % gen)
     return gen
+
+
+def retry(timeout=120, interval=30, wait_on=Exception):
+    """ A decorator that allows to retry a section of code until success or
+        timeout.
+    """
+    def wrapper(function):
+        @functools.wraps(function)
+        def inner(*args, **kwargs):
+            start = time.time()
+            while True:
+                if (time.time() - start) >= timeout:
+                    raise  # This re-raises the last exception.
+                try:
+                    return function(*args, **kwargs)
+                except wait_on:
+                    time.sleep(interval)
+        return inner
+    return wrapper

pungi/wrappers/scm.py

@@ -19,36 +19,25 @@ import os
 import shutil
 import pipes
 import glob
-import time
 
 import kobo.log
 from kobo.shortcuts import run, force_list
-from pungi.util import explode_rpm_package, makedirs, copy_all, temp_dir
+from pungi.util import (explode_rpm_package, makedirs, copy_all, temp_dir,
+                        retry)
 
 
 class ScmBase(kobo.log.LoggingBase):
     def __init__(self, logger=None):
         kobo.log.LoggingBase.__init__(self, logger=logger)
 
-    def retry_run(self, cmd, retries=5, timeout=60, **kwargs):
+    @retry(interval=60, timeout=300, wait_on=RuntimeError)
+    def retry_run(self, cmd, **kwargs):
         """
         @param cmd - cmd passed to kobo.shortcuts.run()
-        @param retries=5 - attempt to execute n times
-        @param timeout=60 - seconds before next try
         @param **kwargs - args passed to kobo.shortcuts.run()
         """
 
-        for n in range(1, retries + 1):
-            try:
-                self.log_debug("Retrying execution %s/%s of '%s'" % (n, retries, cmd))
-                return run(cmd, **kwargs)
-            except RuntimeError as ex:
-                if n == retries:
-                    raise ex
-                self.log_debug("Waiting %s seconds to retry execution of '%s'" % (timeout, cmd))
-                time.sleep(timeout)
-
-        raise RuntimeError("Something went wrong during execution of '%s'" % cmd)
+        return run(cmd, **kwargs)
 
 
 class FileWrapper(ScmBase):