From 224463030b8d990e133f1c6d80646ea1582e24d0 Mon Sep 17 00:00:00 2001 From: Ralph Bean Date: Tue, 1 Apr 2014 15:56:29 -0400 Subject: [PATCH] Use a lockfile around things that modify the cachedir. --- src/bin/pungi.py | 87 +++++++++++++++++++++-------------------- src/pypungi/__init__.py | 46 +++++++++++++++++++++- 2 files changed, 90 insertions(+), 43 deletions(-) diff --git a/src/bin/pungi.py b/src/bin/pungi.py index 261939a7..9d0e7068 100755 --- a/src/bin/pungi.py +++ b/src/bin/pungi.py @@ -103,49 +103,52 @@ def main(): # Actually do work. mypungi = pypungi.Pungi(config, ksparser) + with mypungi.yumlock: + if not opts.sourceisos: + if opts.do_all or opts.do_gather or opts.do_buildinstall: + mypungi._inityum() # initialize the yum object for things that need it + if opts.do_all or opts.do_gather: + mypungi.gather() + if opts.nodownload: + for line in mypungi.list_packages(): + flags_str = ",".join(line["flags"]) + if flags_str: + flags_str = "(%s)" % flags_str + sys.stdout.write("RPM%s: %s\n" % (flags_str, line["path"])) + sys.stdout.flush() + else: + mypungi.downloadPackages() + mypungi.makeCompsFile() + if not opts.nodebuginfo: + mypungi.getDebuginfoList() + if opts.nodownload: + for line in mypungi.list_debuginfo(): + flags_str = ",".join(line["flags"]) + if flags_str: + flags_str = "(%s)" % flags_str + sys.stdout.write("DEBUGINFO%s: %s\n" % (flags_str, line["path"])) + sys.stdout.flush() + else: + mypungi.downloadDebuginfo() + if not opts.nosource: + if opts.nodownload: + for line in mypungi.list_srpms(): + flags_str = ",".join(line["flags"]) + if flags_str: + flags_str = "(%s)" % flags_str + sys.stdout.write("SRPM%s: %s\n" % (flags_str, line["path"])) + sys.stdout.flush() + else: + mypungi.downloadSRPMs() + + print "RPM size: %s MiB" % (mypungi.size_packages() / 1024 ** 2) + if not opts.nodebuginfo: + print "DEBUGINFO size: %s MiB" % (mypungi.size_debuginfo() / 1024 ** 2) + if not opts.nosource: + print "SRPM size: %s MiB" % (mypungi.size_srpms() / 1024 ** 2) + + # Furthermore (but without the yumlock...) if not opts.sourceisos: - if opts.do_all or opts.do_gather or opts.do_buildinstall: - mypungi._inityum() # initialize the yum object for things that need it - if opts.do_all or opts.do_gather: - mypungi.gather() - if opts.nodownload: - for line in mypungi.list_packages(): - flags_str = ",".join(line["flags"]) - if flags_str: - flags_str = "(%s)" % flags_str - sys.stdout.write("RPM%s: %s\n" % (flags_str, line["path"])) - sys.stdout.flush() - else: - mypungi.downloadPackages() - mypungi.makeCompsFile() - if not opts.nodebuginfo: - mypungi.getDebuginfoList() - if opts.nodownload: - for line in mypungi.list_debuginfo(): - flags_str = ",".join(line["flags"]) - if flags_str: - flags_str = "(%s)" % flags_str - sys.stdout.write("DEBUGINFO%s: %s\n" % (flags_str, line["path"])) - sys.stdout.flush() - else: - mypungi.downloadDebuginfo() - if not opts.nosource: - if opts.nodownload: - for line in mypungi.list_srpms(): - flags_str = ",".join(line["flags"]) - if flags_str: - flags_str = "(%s)" % flags_str - sys.stdout.write("SRPM%s: %s\n" % (flags_str, line["path"])) - sys.stdout.flush() - else: - mypungi.downloadSRPMs() - - print "RPM size: %s MiB" % (mypungi.size_packages() / 1024 ** 2) - if not opts.nodebuginfo: - print "DEBUGINFO size: %s MiB" % (mypungi.size_debuginfo() / 1024 ** 2) - if not opts.nosource: - print "SRPM size: %s MiB" % (mypungi.size_srpms() / 1024 ** 2) - if opts.do_all or opts.do_createrepo: mypungi.doCreaterepo() diff --git a/src/pypungi/__init__.py b/src/pypungi/__init__.py index 39657e56..9e9d161d 100644 --- a/src/pypungi/__init__.py +++ b/src/pypungi/__init__.py @@ -23,6 +23,7 @@ import sys import gzip import pypungi.util import pprint +import lockfile import logging import urlgrabber.progress import subprocess @@ -35,6 +36,40 @@ import arch as arch_module import multilib +class ReentrantYumLock(object): + """ A lock that can be acquired multiple times by the same process. """ + + def __init__(self, lock, log): + self.lock = lock + self.log = log + self.count = 0 + + def __enter__(self): + if not self.count: + self.log.info("Waiting on %r" % self.lock.lock_file) + self.lock.acquire() + self.log.info("Got %r" % self.lock.lock_file) + self.count = self.count + 1 + self.log.info("Lock count upped to %i" % self.count) + + def __exit__(self, type, value, tb): + self.count = self.count - 1 + self.log.info("Lock count downed to %i" % self.count) + self.log.info("%r %r %r" % (type, value, tb)) + if not self.count: + self.lock.release() + self.log.info("Released %r" % self.lock.lock_file) + + +def yumlocked(method): + """ A locking decorator. """ + def wrapper(self, *args, **kwargs): + with self.yumlock: + return method(self, *args, **kwargs) + # TODO - replace argspec, signature, etc.. + return wrapper + + def is_debug(po): if "debuginfo" in po.name: return True @@ -163,10 +198,15 @@ class PungiYum(yum.YumBase): class Pungi(pypungi.PungiBase): def __init__(self, config, ksparser): pypungi.PungiBase.__init__(self, config) - + # Set our own logging name space self.logger = logging.getLogger('Pungi') + # Create a lock object for later use. + filename = self.config.get('pungi', 'cachedir') + "/yumlock" + lock = lockfile.LockFile(filename) + self.yumlock = ReentrantYumLock(lock, self.logger) + # Create the stdout/err streams and only send INFO+ stuff there formatter = logging.Formatter('%(name)s:%(levelname)s: %(message)s') console = logging.StreamHandler() @@ -294,6 +334,7 @@ class Pungi(pypungi.PungiBase): if os.path.exists(os.path.join(thisrepo.cachedir, 'repomd.xml')): os.remove(os.path.join(thisrepo.cachedir, 'repomd.xml')) + @yumlocked def _inityum(self): """Initialize the yum object. Only needed for certain actions.""" @@ -1070,6 +1111,7 @@ class Pungi(pypungi.PungiBase): self.logger.info('Finished downloading packages.') + @yumlocked def downloadPackages(self): """Download the package objects obtained in getPackageObjects().""" @@ -1118,6 +1160,7 @@ class Pungi(pypungi.PungiBase): #pypungi.util._doRunCommand(compsfilter, self.logger) + @yumlocked def downloadSRPMs(self): """Cycle through the list of srpms and find the package objects for them, Then download them.""" @@ -1125,6 +1168,7 @@ class Pungi(pypungi.PungiBase): # do the downloads self._downloadPackageList(self.srpm_po_list, os.path.join('source', 'SRPMS')) + @yumlocked def downloadDebuginfo(self): """Cycle through the list of debuginfo rpms and download them."""