almalinux/pungi
14
1
Fork 0
Code Issues Pull Requests Releases Activity

Use a lockfile around things that modify the cachedir.

Browse Source
This commit is contained in:
Ralph Bean 2014-04-01 15:56:29 -04:00 committed by Dennis Gilmore
parent 94235b093e
commit 224463030b
2 changed files with 90 additions and 43 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

87
src/bin/pungi.py
View File

@ -103,49 +103,52 @@ def main():
# Actually do work.
mypungi = pypungi.Pungi(config, ksparser)
with mypungi.yumlock:
if not opts.sourceisos:
if opts.do_all or opts.do_gather or opts.do_buildinstall:
mypungi._inityum() # initialize the yum object for things that need it
if opts.do_all or opts.do_gather:
mypungi.gather()
if opts.nodownload:
for line in mypungi.list_packages():
flags_str = ",".join(line["flags"])
if flags_str:
flags_str = "(%s)" % flags_str
sys.stdout.write("RPM%s: %s\n" % (flags_str, line["path"]))
sys.stdout.flush()
else:
mypungi.downloadPackages()
mypungi.makeCompsFile()
if not opts.nodebuginfo:
mypungi.getDebuginfoList()
if opts.nodownload:
for line in mypungi.list_debuginfo():
flags_str = ",".join(line["flags"])
if flags_str:
flags_str = "(%s)" % flags_str
sys.stdout.write("DEBUGINFO%s: %s\n" % (flags_str, line["path"]))
sys.stdout.flush()
else:
mypungi.downloadDebuginfo()
if not opts.nosource:
if opts.nodownload:
for line in mypungi.list_srpms():
flags_str = ",".join(line["flags"])
if flags_str:
flags_str = "(%s)" % flags_str
sys.stdout.write("SRPM%s: %s\n" % (flags_str, line["path"]))
sys.stdout.flush()
else:
mypungi.downloadSRPMs()
print "RPM size: %s MiB" % (mypungi.size_packages() / 1024 ** 2)
if not opts.nodebuginfo:
print "DEBUGINFO size: %s MiB" % (mypungi.size_debuginfo() / 1024 ** 2)
if not opts.nosource:
print "SRPM size: %s MiB" % (mypungi.size_srpms() / 1024 ** 2)
# Furthermore (but without the yumlock...)
if not opts.sourceisos:
if opts.do_all or opts.do_gather or opts.do_buildinstall:
mypungi._inityum() # initialize the yum object for things that need it
if opts.do_all or opts.do_gather:
mypungi.gather()
if opts.nodownload:
for line in mypungi.list_packages():
flags_str = ",".join(line["flags"])
if flags_str:
flags_str = "(%s)" % flags_str
sys.stdout.write("RPM%s: %s\n" % (flags_str, line["path"]))
sys.stdout.flush()
else:
mypungi.downloadPackages()
mypungi.makeCompsFile()
if not opts.nodebuginfo:
mypungi.getDebuginfoList()
if opts.nodownload:
for line in mypungi.list_debuginfo():
flags_str = ",".join(line["flags"])
if flags_str:
flags_str = "(%s)" % flags_str
sys.stdout.write("DEBUGINFO%s: %s\n" % (flags_str, line["path"]))
sys.stdout.flush()
else:
mypungi.downloadDebuginfo()
if not opts.nosource:
if opts.nodownload:
for line in mypungi.list_srpms():
flags_str = ",".join(line["flags"])
if flags_str:
flags_str = "(%s)" % flags_str
sys.stdout.write("SRPM%s: %s\n" % (flags_str, line["path"]))
sys.stdout.flush()
else:
mypungi.downloadSRPMs()
print "RPM size: %s MiB" % (mypungi.size_packages() / 1024 ** 2)
if not opts.nodebuginfo:
print "DEBUGINFO size: %s MiB" % (mypungi.size_debuginfo() / 1024 ** 2)
if not opts.nosource:
print "SRPM size: %s MiB" % (mypungi.size_srpms() / 1024 ** 2)
if opts.do_all or opts.do_createrepo:
mypungi.doCreaterepo()

44
src/pypungi/__init__.py
View File

@ -23,6 +23,7 @@ import sys
import gzip
import pypungi.util
import pprint
import lockfile
import logging
import urlgrabber.progress
import subprocess
@ -35,6 +36,40 @@ import arch as arch_module
import multilib
class ReentrantYumLock(object):
""" A lock that can be acquired multiple times by the same process. """
def __init__(self, lock, log):
self.lock = lock
self.log = log
self.count = 0
def __enter__(self):
if not self.count:
self.log.info("Waiting on %r" % self.lock.lock_file)
self.lock.acquire()
self.log.info("Got %r" % self.lock.lock_file)
self.count = self.count + 1
self.log.info("Lock count upped to %i" % self.count)
def __exit__(self, type, value, tb):
self.count = self.count - 1
self.log.info("Lock count downed to %i" % self.count)
self.log.info("%r %r %r" % (type, value, tb))
if not self.count:
self.lock.release()
self.log.info("Released %r" % self.lock.lock_file)
def yumlocked(method):
""" A locking decorator. """
def wrapper(self, *args, **kwargs):
with self.yumlock:
return method(self, *args, **kwargs)
# TODO - replace argspec, signature, etc..
return wrapper
def is_debug(po):
if "debuginfo" in po.name:
return True
@ -167,6 +202,11 @@ class Pungi(pypungi.PungiBase):
# Set our own logging name space
self.logger = logging.getLogger('Pungi')
# Create a lock object for later use.
filename = self.config.get('pungi', 'cachedir') + "/yumlock"
lock = lockfile.LockFile(filename)
self.yumlock = ReentrantYumLock(lock, self.logger)
# Create the stdout/err streams and only send INFO+ stuff there
formatter = logging.Formatter('%(name)s:%(levelname)s: %(message)s')
console = logging.StreamHandler()
@ -294,6 +334,7 @@ class Pungi(pypungi.PungiBase):
if os.path.exists(os.path.join(thisrepo.cachedir, 'repomd.xml')):
os.remove(os.path.join(thisrepo.cachedir, 'repomd.xml'))
@yumlocked
def _inityum(self):
"""Initialize the yum object. Only needed for certain actions."""
@ -1070,6 +1111,7 @@ class Pungi(pypungi.PungiBase):
self.logger.info('Finished downloading packages.')
@yumlocked
def downloadPackages(self):
"""Download the package objects obtained in getPackageObjects()."""
@ -1118,6 +1160,7 @@ class Pungi(pypungi.PungiBase):
#pypungi.util._doRunCommand(compsfilter, self.logger)
@yumlocked
def downloadSRPMs(self):
"""Cycle through the list of srpms and
find the package objects for them, Then download them."""
@ -1125,6 +1168,7 @@ class Pungi(pypungi.PungiBase):
# do the downloads
self._downloadPackageList(self.srpm_po_list, os.path.join('source', 'SRPMS'))
@yumlocked
def downloadDebuginfo(self):
"""Cycle through the list of debuginfo rpms and
download them."""