diff --git a/pungi/phases/pkgset/sources/source_koji.py b/pungi/phases/pkgset/sources/source_koji.py index 5e6b2e56..08059022 100644 --- a/pungi/phases/pkgset/sources/source_koji.py +++ b/pungi/phases/pkgset/sources/source_koji.py @@ -21,7 +21,7 @@ import json import pungi.wrappers.kojiwrapper import pungi.phases.pkgset.pkgsets from pungi.arch import get_valid_arches -from pungi.util import is_arch_multilib +from pungi.util import is_arch_multilib, retry from pungi.phases.pkgset.common import create_arch_repos, create_global_repo, populate_arch_pkgsets @@ -62,6 +62,7 @@ def variant_dict_from_str(module_str): return module_info +@retry(wait_on=IOError) def get_module(session, module_info, strict=False): """ :param session : PDCClient instance diff --git a/pungi/util.py b/pungi/util.py index d3a0981a..25540bdb 100644 --- a/pungi/util.py +++ b/pungi/util.py @@ -28,6 +28,7 @@ import contextlib import traceback import tempfile import time +import functools from kobo.shortcuts import run, force_list from productmd.common import get_major_version @@ -756,3 +757,22 @@ def version_generator(compose, gen): if gen and gen[0] == '!': raise RuntimeError("Unknown version generator '%s'" % gen) return gen + + +def retry(timeout=120, interval=30, wait_on=Exception): + """ A decorator that allows to retry a section of code until success or + timeout. + """ + def wrapper(function): + @functools.wraps(function) + def inner(*args, **kwargs): + start = time.time() + while True: + if (time.time() - start) >= timeout: + raise # This re-raises the last exception. + try: + return function(*args, **kwargs) + except wait_on: + time.sleep(interval) + return inner + return wrapper diff --git a/pungi/wrappers/scm.py b/pungi/wrappers/scm.py index a8be3571..4fceab5d 100644 --- a/pungi/wrappers/scm.py +++ b/pungi/wrappers/scm.py @@ -19,36 +19,25 @@ import os import shutil import pipes import glob -import time import kobo.log from kobo.shortcuts import run, force_list -from pungi.util import explode_rpm_package, makedirs, copy_all, temp_dir +from pungi.util import (explode_rpm_package, makedirs, copy_all, temp_dir, + retry) class ScmBase(kobo.log.LoggingBase): def __init__(self, logger=None): kobo.log.LoggingBase.__init__(self, logger=logger) - def retry_run(self, cmd, retries=5, timeout=60, **kwargs): + @retry(interval=60, timeout=300, wait_on=RuntimeError) + def retry_run(self, cmd, **kwargs): """ @param cmd - cmd passed to kobo.shortcuts.run() - @param retries=5 - attempt to execute n times - @param timeout=60 - seconds before next try @param **kwargs - args passed to kobo.shortcuts.run() """ - for n in range(1, retries + 1): - try: - self.log_debug("Retrying execution %s/%s of '%s'" % (n, retries, cmd)) - return run(cmd, **kwargs) - except RuntimeError as ex: - if n == retries: - raise ex - self.log_debug("Waiting %s seconds to retry execution of '%s'" % (timeout, cmd)) - time.sleep(timeout) - - raise RuntimeError("Something went wrong during execution of '%s'" % cmd) + return run(cmd, **kwargs) class FileWrapper(ScmBase):