From 1f85a11293a566b434da9d49147694b00fb13944 Mon Sep 17 00:00:00 2001 From: "jkeating@localhost.localdomain" <> Date: Wed, 15 Aug 2007 19:19:13 -0400 Subject: [PATCH] Create a PungiBase class Make Gather and Pungi subclasses of this base class Adjust logging to use the facility in PungiBase Adjust logging levels to be appropriate Drop a note when the compose is finished. --- Changelog | 12 ++++++++ pungi | 2 ++ pypungi/__init__.py | 54 +++++++++++++++++++++++++++++++++++ pypungi/gather.py | 68 +++++++++++++++++++-------------------------- pypungi/pungi.py | 39 ++++++++++++++------------ 5 files changed, 118 insertions(+), 57 deletions(-) diff --git a/Changelog b/Changelog index 34a2a135..34c653d9 100644 --- a/Changelog +++ b/Changelog @@ -1,3 +1,15 @@ +* Wed Aug 15 2007 Jesse Keating +- Create a base pungi class that sets logging +- Inherit this class in Gather and Pungi +- Adjust logging to make use of new facility, use right levels +- Drop a note when all done with composing + +* Wed Aug 08 2007 Jesse Keating +- Make Gather() no longer a subclass of yum +- We set debuglevel when creating the yum object. +- Be verbose about what we clean +- Create a subclass of yum to work around logging fun + * Wed Aug 01 2007 Jesse Keating - Create a new yum object for source downloads as yum diff --git a/pungi b/pungi index d27d960e..13ba4109 100755 --- a/pungi +++ b/pungi @@ -142,6 +142,8 @@ def main(): if opts.do_all or opts.do_createiso: mypungi.doCreateIsos() + print "All done!" + if __name__ == '__main__': from optparse import OptionParser import sys diff --git a/pypungi/__init__.py b/pypungi/__init__.py index e69de29b..ed4d108f 100644 --- a/pypungi/__init__.py +++ b/pypungi/__init__.py @@ -0,0 +1,54 @@ +#!/usr/bin/python -tt +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; version 2 of the License. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Library General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. + +import logging +import os + +class PungiBase(): + """The base Pungi class. Set up config items and logging here""" + + def __init__(self, config): + self.config = config + self.doLoggerSetup() + + self.workdir = os.path.join(self.config.get('default', 'destdir'), + 'work', + self.config.get('default', 'flavor'), + self.config.get('default', 'arch')) + + + + def doLoggerSetup(self): + """Setup our logger""" + + logdir = os.path.join(self.config.get('default', 'destdir'), 'logs') + + if not os.path.exists(logdir): + os.makedirs(logdir) + + if self.config.get('default', 'flavor'): + logfile = os.path.join(logdir, '%s.%s.log' % (self.config.get('default', 'flavor'), + self.config.get('default', 'arch'))) + else: + logfile = os.path.join(logdir, '%s.log' % (self.config.get('default', 'arch'))) + + # Create the root logger, that will log to our file + logging.basicConfig(level=logging.DEBUG, + format='%(name)s.%(levelname)s: %(message)s', + filename=logfile) + + # Create the stdout/err streams and only send INFO+ stuff there + console = logging.StreamHandler() + console.setLevel(logging.INFO) + logging.getLogger('').addHandler(console) diff --git a/pypungi/gather.py b/pypungi/gather.py index e9c6fefa..dcfdb9ea 100755 --- a/pypungi/gather.py +++ b/pypungi/gather.py @@ -16,6 +16,8 @@ import yum import os import shutil import sys +import pypungi +import logging class PungiYum(yum.YumBase): """Subclass of Yum""" @@ -39,15 +41,18 @@ class PungiYum(yum.YumBase): yum.logging.basicConfig(level=yum.logging.DEBUG, filename=logfile) + def doFileLogSetup(self, uid, logfile): + # This function overrides a yum function, allowing pungi to control + # the logging. + pass -class Gather(): +class Gather(pypungi.PungiBase): def __init__(self, config, pkglist): - self.workdir = os.path.join(config.get('default', 'destdir'), - 'work', - config.get('default', 'flavor'), - config.get('default', 'arch')) + pypungi.PungiBase.__init__(self, config) + + # Set our own logging name space + self.logger = logging.getLogger('Pungi.Gather') - self.config = config self.pkglist = pkglist self.config.cachedir = os.path.join(self.workdir, 'yumcache') self.polist = [] @@ -82,8 +87,6 @@ class Gather(): #self.doSackSetup(archlist=arches) # work around temp break in yum api #self.doSackFilelistPopulate() self.ayum._getSacks(archlist=arches) - self.logger = yum.logging.getLogger("yum.verbose.pungi") - self.ayum.logger = yum.logging.getLogger("yum.verbose.pungi") def verifyCachePkg(self, po, path): # Stolen from yum """check the package checksum vs the cache @@ -106,8 +109,7 @@ class Gather(): transaction info""" - if not self.config.has_option('default', 'quiet'): - self.logger.info('Checking deps of %s.%s' % (po.name, po.arch)) + self.logger.info('Checking deps of %s.%s' % (po.name, po.arch)) reqs = po.requires provs = po.provides @@ -130,8 +132,7 @@ class Gather(): for dep in depsack.returnNewestByNameArch(): self.ayum.tsInfo.addInstall(dep) - if not self.config.has_option('default', 'quiet'): - self.logger.info('Added %s.%s for %s.%s' % (dep.name, dep.arch, po.name, po.arch)) + self.logger.info('Added %s.%s for %s.%s' % (dep.name, dep.arch, po.name, po.arch)) self.resolved_deps[req] = None @@ -207,22 +208,18 @@ class Gather(): for line in self.pkglist: line = line.strip() if line.startswith('#'): - if not self.config.has_option('default', 'quiet'): - self.logger.info('Skipping comment: %s' % line) + self.logger.debug('Skipping comment: %s' % line) continue if line.startswith('@'): - if not self.config.has_option('default', 'quiet'): - self.logger.info('Adding group: %s' % line) + self.logger.info('Adding group: %s' % line) grouplist.append(line.strip('@')) continue if line.startswith('-'): - if not self.config.has_option('default', 'quiet'): - self.logger.info('Adding exclude: %s' % line) + self.logger.info('Adding exclude: %s' % line) excludelist.append(line.strip('-')) continue else: - if not self.config.has_option('default', 'quiet'): - self.logger.info('Adding package: %s' % line) + self.logger.info('Adding package: %s' % line) addlist.append(line) # First remove the excludes @@ -251,8 +248,7 @@ class Gather(): mysack = yum.packageSack.ListPackageSack(matches) for match in mysack.returnNewestByNameArch(): self.ayum.tsInfo.addInstall(match) - if not self.config.has_option('default', 'quiet'): - self.logger.info('Found %s.%s' % (match.name, match.arch)) + self.logger.debug('Found %s.%s' % (match.name, match.arch)) for pkg in unmatched: if not pkg in matchdict.keys(): @@ -289,12 +285,11 @@ class Gather(): download them from their respective repos.""" - if not self.config.has_option('default', 'quiet'): - downloads = [] - for pkg in self.polist: - downloads.append('%s.%s' % (pkg.name, pkg.arch)) - downloads.sort() - self.logger.info("Download list: %s" % downloads) + downloads = [] + for pkg in self.polist: + downloads.append('%s.%s' % (pkg.name, pkg.arch)) + downloads.sort() + self.logger.info("Download list: %s" % downloads) # Package location within destdir, name subject to change/config pkgdir = os.path.join(self.config.get('default', 'destdir'), self.config.get('default', 'version'), @@ -312,9 +307,7 @@ class Gather(): local = os.path.basename(remote) local = os.path.join(self.config.get('default', 'cachedir'), local) if os.path.exists(local) and self.verifyCachePkg(pkg, local): - - if not self.config.has_option('default', 'quiet'): - self.logger.info("%s already exists and appears to be complete" % local) + self.logger.debug("%s already exists and appears to be complete" % local) target = os.path.join(pkgdir, os.path.basename(remote)) if os.path.exists(target): os.remove(target) # avoid traceback after interrupted download @@ -323,8 +316,7 @@ class Gather(): # Disable cache otherwise things won't download repo.cache = 0 - if not self.config.has_option('default', 'quiet'): - self.logger.info('Downloading %s' % os.path.basename(remote)) + self.logger.info('Downloading %s' % os.path.basename(remote)) pkg.localpath = local # Hack: to set the localpath to what we want. # do a little dance for file:// repos... @@ -385,20 +377,16 @@ class Gather(): local = os.path.basename(remote) local = os.path.join(self.config.get('default', 'cachedir'), local) if os.path.exists(local) and self.verifyCachePkg(pkg, local): - - if not self.config.has_option('default', 'quiet'): - self.logger.info("%s already exists and appears to be complete" % local) + self.logger.debug("%s already exists and appears to be complete" % local) if os.path.exists(os.path.join(pkgdir, os.path.basename(remote))) and self.verifyCachePkg(pkg, os.path.join(pkgdir, os.path.basename(remote))): - if not self.config.has_option('default', 'quiet'): - self.logger.info("%s already exists in tree and appears to be complete" % local) + self.logger.debug("%s already exists in tree and appears to be complete" % local) else: os.link(local, os.path.join(pkgdir, os.path.basename(remote))) continue # Disable cache otherwise things won't download repo.cache = 0 - if not self.config.has_option('default', 'quiet'): - self.logger.info('Downloading %s' % os.path.basename(remote)) + self.logger.info('Downloading %s' % os.path.basename(remote)) pkg.localpath = local # Hack: to set the localpath to what we want. # do a little dance for file:// repos... diff --git a/pypungi/pungi.py b/pypungi/pungi.py index dfc61ced..d8b4c9cb 100755 --- a/pypungi/pungi.py +++ b/pypungi/pungi.py @@ -20,12 +20,14 @@ sys.path.append('/usr/lib/anaconda-runtime') import splittree import shutil import re +import pypungi -log = logging.getLogger("pypungi.pungi") - -class Pungi: +class Pungi(pypungi.PungiBase): def __init__(self, config): - self.config = config + pypungi.PungiBase.__init__(self, config) + + self.logger = logging.getLogger('Pungi.Pungi') + self.prodpath = 'Fedora' # Probably should be defined elsewhere self.destdir = self.config.get('default', 'destdir') self.archdir = os.path.join(self.destdir, @@ -36,7 +38,7 @@ class Pungi: self.topdir = os.path.join(self.archdir, 'os') self.isodir = os.path.join(self.archdir, self.config.get('default','isodir')) - self.workdir = os.path.join(self.config.get('default', 'destdir'), + self.workdir = os.path.join(self.config.get('default', 'destdir'), 'work', self.config.get('default', 'flavor'), self.config.get('default', 'arch')) @@ -67,16 +69,16 @@ class Pungi: """Run a command and log the output. Error out if we get something on stderr""" - log.info("Running %s" % ' '.join(command)) + self.logger.info("Running %s" % ' '.join(command)) p1 = subprocess.Popen(command, cwd=rundir, stdout=output, stderr=error, universal_newlines=True) (out, err) = p1.communicate() - log.info(out) + self.logger.debug(out) if p1.returncode != 0: - log.error("Got an error from %s" % command[0]) - log.error(err) + self.logger.error("Got an error from %s" % command[0]) + self.logger.error(err) raise OSError, "Got an error from %s: %s" % (command[0], err) def doCreaterepo(self): @@ -185,18 +187,18 @@ class Pungi: (out, err) = subprocess.Popen(cpio, cwd=docsdir, stdin=p1.stdout, stdout=subprocess.PIPE, stderr=subprocess.PIPE, universal_newlines=True).communicate() except: - log.error("Got an error from rpm2cpio") - log.error(err) + self.logger.error("Got an error from rpm2cpio") + self.logger.error(err) raise - log.info(out) + self.logger.debug(out) # Walk the tree for our files for dirpath, dirname, filelist in os.walk(docsdir): for filename in filelist: for regex in fileres: if regex.match(filename) and not os.path.exists(os.path.join(self.topdir, filename)): - log.info("Copying release note file %s" % filename) + self.logger.info("Copying release note file %s" % filename) shutil.copy(os.path.join(dirpath, filename), os.path.join(self.topdir, filename)) self.common_files.append(filename) @@ -205,7 +207,7 @@ class Pungi: for directory in dirname: for regex in dirres: if regex.match(directory) and not os.path.exists(os.path.join(self.topdir, directory)): - log.info("Copying release note dir %s" % directory) + self.logger.info("Copying release note dir %s" % directory) shutil.copytree(os.path.join(dirpath, directory), os.path.join(self.topdir, directory)) @@ -232,8 +234,10 @@ class Pungi: timber.common_files = self.common_files #timber.reserve_size = + self.logger.info("Running splittree.") + output = timber.main() - log.info("Output from splittree: %s" % '\n'.join(output)) + self.logger.debug("Output from splittree: %s" % '\n'.join(output)) def doSplitSRPMs(self): """Use anaconda-runtime's splittree to split the srpms into appropriate @@ -270,8 +274,9 @@ class Pungi: "%s-disc%d" %(timber.dist_dir, i), timber.common_files) + self.logger.info("Splitting SRPMs") timber.splitSRPMS() - log.info("splitSRPMS complete") + self.logger.info("splitSRPMS complete") def doCreateSplitrepo(self): """Create the split metadata for the isos""" @@ -519,4 +524,4 @@ class Pungi: if directory.startswith('os-disc') or directory.startswith('SRPM-disc'): shutil.move(os.path.join(self.archdir, directory), os.path.join(self.workdir, directory)) - log.info("CreateIsos is done.") + self.logger.info("CreateIsos is done.")