Add @retry decorator and use it to retry connection on PDC on IOError and in SCM's retry_run.

Signed-off-by: Jan Kaluza <jkaluza@redhat.com>
This commit is contained in:
Jan Kaluza 2017-05-29 10:44:43 +02:00
parent dc410b58b5
commit 68351fa5a8
3 changed files with 27 additions and 17 deletions

View File

@ -21,7 +21,7 @@ import json
import pungi.wrappers.kojiwrapper
import pungi.phases.pkgset.pkgsets
from pungi.arch import get_valid_arches
from pungi.util import is_arch_multilib
from pungi.util import is_arch_multilib, retry
from pungi.phases.pkgset.common import create_arch_repos, create_global_repo, populate_arch_pkgsets
@ -62,6 +62,7 @@ def variant_dict_from_str(module_str):
return module_info
@retry(wait_on=IOError)
def get_module(session, module_info, strict=False):
"""
:param session : PDCClient instance

View File

@ -28,6 +28,7 @@ import contextlib
import traceback
import tempfile
import time
import functools
from kobo.shortcuts import run, force_list
from productmd.common import get_major_version
@ -756,3 +757,22 @@ def version_generator(compose, gen):
if gen and gen[0] == '!':
raise RuntimeError("Unknown version generator '%s'" % gen)
return gen
def retry(timeout=120, interval=30, wait_on=Exception):
""" A decorator that allows to retry a section of code until success or
timeout.
"""
def wrapper(function):
@functools.wraps(function)
def inner(*args, **kwargs):
start = time.time()
while True:
if (time.time() - start) >= timeout:
raise # This re-raises the last exception.
try:
return function(*args, **kwargs)
except wait_on:
time.sleep(interval)
return inner
return wrapper

View File

@ -19,36 +19,25 @@ import os
import shutil
import pipes
import glob
import time
import kobo.log
from kobo.shortcuts import run, force_list
from pungi.util import explode_rpm_package, makedirs, copy_all, temp_dir
from pungi.util import (explode_rpm_package, makedirs, copy_all, temp_dir,
retry)
class ScmBase(kobo.log.LoggingBase):
def __init__(self, logger=None):
kobo.log.LoggingBase.__init__(self, logger=logger)
def retry_run(self, cmd, retries=5, timeout=60, **kwargs):
@retry(interval=60, timeout=300, wait_on=RuntimeError)
def retry_run(self, cmd, **kwargs):
"""
@param cmd - cmd passed to kobo.shortcuts.run()
@param retries=5 - attempt to execute n times
@param timeout=60 - seconds before next try
@param **kwargs - args passed to kobo.shortcuts.run()
"""
for n in range(1, retries + 1):
try:
self.log_debug("Retrying execution %s/%s of '%s'" % (n, retries, cmd))
return run(cmd, **kwargs)
except RuntimeError as ex:
if n == retries:
raise ex
self.log_debug("Waiting %s seconds to retry execution of '%s'" % (timeout, cmd))
time.sleep(timeout)
raise RuntimeError("Something went wrong during execution of '%s'" % cmd)
class FileWrapper(ScmBase):