almalinux/pungi
13
1
Fork 0
Code Issues Pull Requests Releases Activity

Create a PungiBase class

Browse Source 
Make Gather and Pungi subclasses of this base class
Adjust logging to use the facility in PungiBase
Adjust logging levels to be appropriate
Drop a note when the compose is finished.
This commit is contained in:
jkeating@localhost.localdomain 2007-08-15 19:19:13 -04:00 committed by Jesse Keating
parent 9cb130eb8d
commit 1f85a11293
5 changed files with 118 additions and 57 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

12
Changelog
View File

@ -1,3 +1,15 @@
* Wed Aug 15 2007 Jesse Keating <jkeating@redhat.com>
- Create a base pungi class that sets logging
- Inherit this class in Gather and Pungi
- Adjust logging to make use of new facility, use right levels
- Drop a note when all done with composing
* Wed Aug 08 2007 Jesse Keating <jkeating@redhat.com>
- Make Gather() no longer a subclass of yum
- We set debuglevel when creating the yum object.
- Be verbose about what we clean
- Create a subclass of yum to work around logging fun
* Wed Aug 01 2007 Jesse Keating <jkeating@redhat.com> * Wed Aug 01 2007 Jesse Keating <jkeating@redhat.com>
- Create a new yum object for source downloads as yum - Create a new yum object for source downloads as yum

2
pungi
View File

@ -142,6 +142,8 @@ def main():
if opts.do_all or opts.do_createiso: if opts.do_all or opts.do_createiso:
mypungi.doCreateIsos() mypungi.doCreateIsos()
print "All done!"
if __name__ == '__main__': if __name__ == '__main__':
from optparse import OptionParser from optparse import OptionParser
import sys import sys

54
pypungi/__init__.py
View File

@ -0,0 +1,54 @@
#!/usr/bin/python -tt
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation; version 2 of the License.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Library General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program; if not, write to the Free Software
# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
import logging
import os
class PungiBase():
"""The base Pungi class. Set up config items and logging here"""
def __init__(self, config):
self.config = config
self.doLoggerSetup()
self.workdir = os.path.join(self.config.get('default', 'destdir'),
'work',
self.config.get('default', 'flavor'),
self.config.get('default', 'arch'))
def doLoggerSetup(self):
"""Setup our logger"""
logdir = os.path.join(self.config.get('default', 'destdir'), 'logs')
if not os.path.exists(logdir):
os.makedirs(logdir)
if self.config.get('default', 'flavor'):
logfile = os.path.join(logdir, '%s.%s.log' % (self.config.get('default', 'flavor'),
self.config.get('default', 'arch')))
else:
logfile = os.path.join(logdir, '%s.log' % (self.config.get('default', 'arch')))
# Create the root logger, that will log to our file
logging.basicConfig(level=logging.DEBUG,
format='%(name)s.%(levelname)s: %(message)s',
filename=logfile)
# Create the stdout/err streams and only send INFO+ stuff there
console = logging.StreamHandler()
console.setLevel(logging.INFO)
logging.getLogger('').addHandler(console)

68
pypungi/gather.py
View File

@ -16,6 +16,8 @@ import yum
import os import os
import shutil import shutil
import sys import sys
import pypungi
import logging
class PungiYum(yum.YumBase): class PungiYum(yum.YumBase):
"""Subclass of Yum""" """Subclass of Yum"""
@ -39,15 +41,18 @@ class PungiYum(yum.YumBase):
yum.logging.basicConfig(level=yum.logging.DEBUG, filename=logfile) yum.logging.basicConfig(level=yum.logging.DEBUG, filename=logfile)
def doFileLogSetup(self, uid, logfile):
# This function overrides a yum function, allowing pungi to control
# the logging.
pass
class Gather(): class Gather(pypungi.PungiBase):
def __init__(self, config, pkglist): def __init__(self, config, pkglist):
self.workdir = os.path.join(config.get('default', 'destdir'), pypungi.PungiBase.__init__(self, config)
'work',
config.get('default', 'flavor'), # Set our own logging name space
config.get('default', 'arch')) self.logger = logging.getLogger('Pungi.Gather')
self.config = config
self.pkglist = pkglist self.pkglist = pkglist
self.config.cachedir = os.path.join(self.workdir, 'yumcache') self.config.cachedir = os.path.join(self.workdir, 'yumcache')
self.polist = [] self.polist = []
@ -82,8 +87,6 @@ class Gather():
#self.doSackSetup(archlist=arches) # work around temp break in yum api #self.doSackSetup(archlist=arches) # work around temp break in yum api
#self.doSackFilelistPopulate() #self.doSackFilelistPopulate()
self.ayum._getSacks(archlist=arches) self.ayum._getSacks(archlist=arches)
self.logger = yum.logging.getLogger("yum.verbose.pungi")
self.ayum.logger = yum.logging.getLogger("yum.verbose.pungi")
def verifyCachePkg(self, po, path): # Stolen from yum def verifyCachePkg(self, po, path): # Stolen from yum
"""check the package checksum vs the cache """check the package checksum vs the cache
@ -106,8 +109,7 @@ class Gather():
transaction info""" transaction info"""
if not self.config.has_option('default', 'quiet'): self.logger.info('Checking deps of %s.%s' % (po.name, po.arch))
self.logger.info('Checking deps of %s.%s' % (po.name, po.arch))
reqs = po.requires reqs = po.requires
provs = po.provides provs = po.provides
@ -130,8 +132,7 @@ class Gather():
for dep in depsack.returnNewestByNameArch(): for dep in depsack.returnNewestByNameArch():
self.ayum.tsInfo.addInstall(dep) self.ayum.tsInfo.addInstall(dep)
if not self.config.has_option('default', 'quiet'): self.logger.info('Added %s.%s for %s.%s' % (dep.name, dep.arch, po.name, po.arch))
self.logger.info('Added %s.%s for %s.%s' % (dep.name, dep.arch, po.name, po.arch))
self.resolved_deps[req] = None self.resolved_deps[req] = None
@ -207,22 +208,18 @@ class Gather():
for line in self.pkglist: for line in self.pkglist:
line = line.strip() line = line.strip()
if line.startswith('#'): if line.startswith('#'):
if not self.config.has_option('default', 'quiet'): self.logger.debug('Skipping comment: %s' % line)
self.logger.info('Skipping comment: %s' % line)
continue continue
if line.startswith('@'): if line.startswith('@'):
if not self.config.has_option('default', 'quiet'): self.logger.info('Adding group: %s' % line)
self.logger.info('Adding group: %s' % line)
grouplist.append(line.strip('@')) grouplist.append(line.strip('@'))
continue continue
if line.startswith('-'): if line.startswith('-'):
if not self.config.has_option('default', 'quiet'): self.logger.info('Adding exclude: %s' % line)
self.logger.info('Adding exclude: %s' % line)
excludelist.append(line.strip('-')) excludelist.append(line.strip('-'))
continue continue
else: else:
if not self.config.has_option('default', 'quiet'): self.logger.info('Adding package: %s' % line)
self.logger.info('Adding package: %s' % line)
addlist.append(line) addlist.append(line)
# First remove the excludes # First remove the excludes
@ -251,8 +248,7 @@ class Gather():
mysack = yum.packageSack.ListPackageSack(matches) mysack = yum.packageSack.ListPackageSack(matches)
for match in mysack.returnNewestByNameArch(): for match in mysack.returnNewestByNameArch():
self.ayum.tsInfo.addInstall(match) self.ayum.tsInfo.addInstall(match)
if not self.config.has_option('default', 'quiet'): self.logger.debug('Found %s.%s' % (match.name, match.arch))
self.logger.info('Found %s.%s' % (match.name, match.arch))
for pkg in unmatched: for pkg in unmatched:
if not pkg in matchdict.keys(): if not pkg in matchdict.keys():
@ -289,12 +285,11 @@ class Gather():
download them from their respective repos.""" download them from their respective repos."""
if not self.config.has_option('default', 'quiet'): downloads = []
downloads = [] for pkg in self.polist:
for pkg in self.polist: downloads.append('%s.%s' % (pkg.name, pkg.arch))
downloads.append('%s.%s' % (pkg.name, pkg.arch)) downloads.sort()
downloads.sort() self.logger.info("Download list: %s" % downloads)
self.logger.info("Download list: %s" % downloads)
# Package location within destdir, name subject to change/config # Package location within destdir, name subject to change/config
pkgdir = os.path.join(self.config.get('default', 'destdir'), self.config.get('default', 'version'), pkgdir = os.path.join(self.config.get('default', 'destdir'), self.config.get('default', 'version'),
@ -312,9 +307,7 @@ class Gather():
local = os.path.basename(remote) local = os.path.basename(remote)
local = os.path.join(self.config.get('default', 'cachedir'), local) local = os.path.join(self.config.get('default', 'cachedir'), local)
if os.path.exists(local) and self.verifyCachePkg(pkg, local): if os.path.exists(local) and self.verifyCachePkg(pkg, local):
self.logger.debug("%s already exists and appears to be complete" % local)
if not self.config.has_option('default', 'quiet'):
self.logger.info("%s already exists and appears to be complete" % local)
target = os.path.join(pkgdir, os.path.basename(remote)) target = os.path.join(pkgdir, os.path.basename(remote))
if os.path.exists(target): if os.path.exists(target):
os.remove(target) # avoid traceback after interrupted download os.remove(target) # avoid traceback after interrupted download
@ -323,8 +316,7 @@ class Gather():
# Disable cache otherwise things won't download # Disable cache otherwise things won't download
repo.cache = 0 repo.cache = 0
if not self.config.has_option('default', 'quiet'): self.logger.info('Downloading %s' % os.path.basename(remote))
self.logger.info('Downloading %s' % os.path.basename(remote))
pkg.localpath = local # Hack: to set the localpath to what we want. pkg.localpath = local # Hack: to set the localpath to what we want.
# do a little dance for file:// repos... # do a little dance for file:// repos...
@ -385,20 +377,16 @@ class Gather():
local = os.path.basename(remote) local = os.path.basename(remote)
local = os.path.join(self.config.get('default', 'cachedir'), local) local = os.path.join(self.config.get('default', 'cachedir'), local)
if os.path.exists(local) and self.verifyCachePkg(pkg, local): if os.path.exists(local) and self.verifyCachePkg(pkg, local):
self.logger.debug("%s already exists and appears to be complete" % local)
if not self.config.has_option('default', 'quiet'):
self.logger.info("%s already exists and appears to be complete" % local)
if os.path.exists(os.path.join(pkgdir, os.path.basename(remote))) and self.verifyCachePkg(pkg, os.path.join(pkgdir, os.path.basename(remote))): if os.path.exists(os.path.join(pkgdir, os.path.basename(remote))) and self.verifyCachePkg(pkg, os.path.join(pkgdir, os.path.basename(remote))):
if not self.config.has_option('default', 'quiet'): self.logger.debug("%s already exists in tree and appears to be complete" % local)
self.logger.info("%s already exists in tree and appears to be complete" % local)
else: else:
os.link(local, os.path.join(pkgdir, os.path.basename(remote))) os.link(local, os.path.join(pkgdir, os.path.basename(remote)))
continue continue
# Disable cache otherwise things won't download # Disable cache otherwise things won't download
repo.cache = 0 repo.cache = 0
if not self.config.has_option('default', 'quiet'): self.logger.info('Downloading %s' % os.path.basename(remote))
self.logger.info('Downloading %s' % os.path.basename(remote))
pkg.localpath = local # Hack: to set the localpath to what we want. pkg.localpath = local # Hack: to set the localpath to what we want.
# do a little dance for file:// repos... # do a little dance for file:// repos...

39
pypungi/pungi.py
View File

@ -20,12 +20,14 @@ sys.path.append('/usr/lib/anaconda-runtime')
import splittree import splittree
import shutil import shutil
import re import re
import pypungi
log = logging.getLogger("pypungi.pungi") class Pungi(pypungi.PungiBase):
class Pungi:
def __init__(self, config): def __init__(self, config):
self.config = config pypungi.PungiBase.__init__(self, config)
self.logger = logging.getLogger('Pungi.Pungi')
self.prodpath = 'Fedora' # Probably should be defined elsewhere self.prodpath = 'Fedora' # Probably should be defined elsewhere
self.destdir = self.config.get('default', 'destdir') self.destdir = self.config.get('default', 'destdir')
self.archdir = os.path.join(self.destdir, self.archdir = os.path.join(self.destdir,
@ -36,7 +38,7 @@ class Pungi:
self.topdir = os.path.join(self.archdir, 'os') self.topdir = os.path.join(self.archdir, 'os')
self.isodir = os.path.join(self.archdir, self.config.get('default','isodir')) self.isodir = os.path.join(self.archdir, self.config.get('default','isodir'))
self.workdir = os.path.join(self.config.get('default', 'destdir'), self.workdir = os.path.join(self.config.get('default', 'destdir'),
'work', 'work',
self.config.get('default', 'flavor'), self.config.get('default', 'flavor'),
self.config.get('default', 'arch')) self.config.get('default', 'arch'))
@ -67,16 +69,16 @@ class Pungi:
"""Run a command and log the output. Error out if we get something on stderr""" """Run a command and log the output. Error out if we get something on stderr"""
log.info("Running %s" % ' '.join(command)) self.logger.info("Running %s" % ' '.join(command))
p1 = subprocess.Popen(command, cwd=rundir, stdout=output, stderr=error, universal_newlines=True) p1 = subprocess.Popen(command, cwd=rundir, stdout=output, stderr=error, universal_newlines=True)
(out, err) = p1.communicate() (out, err) = p1.communicate()
log.info(out) self.logger.debug(out)
if p1.returncode != 0: if p1.returncode != 0:
log.error("Got an error from %s" % command[0]) self.logger.error("Got an error from %s" % command[0])
log.error(err) self.logger.error(err)
raise OSError, "Got an error from %s: %s" % (command[0], err) raise OSError, "Got an error from %s: %s" % (command[0], err)
def doCreaterepo(self): def doCreaterepo(self):
@ -185,18 +187,18 @@ class Pungi:
(out, err) = subprocess.Popen(cpio, cwd=docsdir, stdin=p1.stdout, stdout=subprocess.PIPE, (out, err) = subprocess.Popen(cpio, cwd=docsdir, stdin=p1.stdout, stdout=subprocess.PIPE,
stderr=subprocess.PIPE, universal_newlines=True).communicate() stderr=subprocess.PIPE, universal_newlines=True).communicate()
except: except:
log.error("Got an error from rpm2cpio") self.logger.error("Got an error from rpm2cpio")
log.error(err) self.logger.error(err)
raise raise
log.info(out) self.logger.debug(out)
# Walk the tree for our files # Walk the tree for our files
for dirpath, dirname, filelist in os.walk(docsdir): for dirpath, dirname, filelist in os.walk(docsdir):
for filename in filelist: for filename in filelist:
for regex in fileres: for regex in fileres:
if regex.match(filename) and not os.path.exists(os.path.join(self.topdir, filename)): if regex.match(filename) and not os.path.exists(os.path.join(self.topdir, filename)):
log.info("Copying release note file %s" % filename) self.logger.info("Copying release note file %s" % filename)
shutil.copy(os.path.join(dirpath, filename), os.path.join(self.topdir, filename)) shutil.copy(os.path.join(dirpath, filename), os.path.join(self.topdir, filename))
self.common_files.append(filename) self.common_files.append(filename)
@ -205,7 +207,7 @@ class Pungi:
for directory in dirname: for directory in dirname:
for regex in dirres: for regex in dirres:
if regex.match(directory) and not os.path.exists(os.path.join(self.topdir, directory)): if regex.match(directory) and not os.path.exists(os.path.join(self.topdir, directory)):
log.info("Copying release note dir %s" % directory) self.logger.info("Copying release note dir %s" % directory)
shutil.copytree(os.path.join(dirpath, directory), os.path.join(self.topdir, directory)) shutil.copytree(os.path.join(dirpath, directory), os.path.join(self.topdir, directory))
@ -232,8 +234,10 @@ class Pungi:
timber.common_files = self.common_files timber.common_files = self.common_files
#timber.reserve_size = #timber.reserve_size =
self.logger.info("Running splittree.")
output = timber.main() output = timber.main()
log.info("Output from splittree: %s" % '\n'.join(output)) self.logger.debug("Output from splittree: %s" % '\n'.join(output))
def doSplitSRPMs(self): def doSplitSRPMs(self):
"""Use anaconda-runtime's splittree to split the srpms into appropriate """Use anaconda-runtime's splittree to split the srpms into appropriate
@ -270,8 +274,9 @@ class Pungi:
"%s-disc%d" %(timber.dist_dir, i), "%s-disc%d" %(timber.dist_dir, i),
timber.common_files) timber.common_files)
self.logger.info("Splitting SRPMs")
timber.splitSRPMS() timber.splitSRPMS()
log.info("splitSRPMS complete") self.logger.info("splitSRPMS complete")
def doCreateSplitrepo(self): def doCreateSplitrepo(self):
"""Create the split metadata for the isos""" """Create the split metadata for the isos"""
@ -519,4 +524,4 @@ class Pungi:
if directory.startswith('os-disc') or directory.startswith('SRPM-disc'): if directory.startswith('os-disc') or directory.startswith('SRPM-disc'):
shutil.move(os.path.join(self.archdir, directory), os.path.join(self.workdir, directory)) shutil.move(os.path.join(self.archdir, directory), os.path.join(self.workdir, directory))
log.info("CreateIsos is done.") self.logger.info("CreateIsos is done.")