From b45c700be80e89d3e039d2000628698bf509a2a1 Mon Sep 17 00:00:00 2001 From: Jesse Keating Date: Thu, 12 Jun 2008 08:55:42 -0400 Subject: [PATCH] Collapse all of Gather and Pungi into one class/file. --- src/pypungi/pungi.py | 402 ++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 393 insertions(+), 9 deletions(-) diff --git a/src/pypungi/pungi.py b/src/pypungi/pungi.py index 1b224181..f92fb476 100755 --- a/src/pypungi/pungi.py +++ b/src/pypungi/pungi.py @@ -12,22 +12,60 @@ # along with this program; if not, write to the Free Software # Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. -import subprocess -import logging +import yum import os +import re +import shutil import sys +import gzip +import pypungi +import logging +import urlgrabber.progress +import subprocess sys.path.append('/usr/lib/anaconda-runtime') import splittree -import shutil -import re -import pypungi import createrepo -class Pungi(pypungi.PungiBase): - def __init__(self, config): - pypungi.PungiBase.__init__(self, config) - self.logger = logging.getLogger('Pungi.Pungi') +class CallBack(urlgrabber.progress.TextMeter): + """A call back function used with yum.""" + + def progressbar(self, current, total, name=None): + return + +class PungiYum(yum.YumBase): + """Subclass of Yum""" + + def __init__(self, config): + self.pungiconfig = config + yum.YumBase.__init__(self) + + def doLoggingSetup(self, debuglevel, errorlevel): + """Setup the logging facility.""" + + logdir = os.path.join(self.pungiconfig.get('default', 'destdir'), 'logs') + if not os.path.exists(logdir): + os.makedirs(logdir) + if self.pungiconfig.get('default', 'flavor'): + logfile = os.path.join(logdir, '%s.%s.log' % (self.pungiconfig.get('default', 'flavor'), + self.pungiconfig.get('default', 'arch'))) + else: + logfile = os.path.join(logdir, '%s.log' % (self.pungiconfig.get('default', 'arch'))) + + yum.logging.basicConfig(level=yum.logging.DEBUG, filename=logfile) + self.logger.error('foobar') + + def doFileLogSetup(self, uid, logfile): + # This function overrides a yum function, allowing pungi to control + # the logging. + pass + +class Pungi(pypungi.PungiBase): + def __init__(self, config, ksparser): + pypungi.PungiBase.__init__(self, config) + + # Set our own logging name space + self.logger = logging.getLogger('Pungi') # Create the stdout/err streams and only send INFO+ stuff there formatter = logging.Formatter('%(name)s:%(levelname)s: %(message)s') @@ -52,6 +90,352 @@ class Pungi(pypungi.PungiBase): self.config.get('default', 'version'), '.composeinfo') + self.ksparser = ksparser + self.polist = [] + self.srpmlist = [] + self.resolved_deps = {} # list the deps we've already resolved, short circuit. + + # Create a yum object to use + self.ayum = PungiYum(config) + self.ayum.doLoggingSetup(6, 6) + yumconf = yum.config.YumConf() + yumconf.debuglevel = 6 + yumconf.errorlevel = 6 + yumconf.cachedir = self.config.get('default', 'cachedir') + yumconf.persistdir = os.path.join(self.workdir, 'yumlib') + yumconf.installroot = os.path.join(self.workdir, 'yumroot') + yumconf.uid = os.geteuid() + yumconf.cache = 0 + yumconf.failovermethod = 'priority' + yumvars = yum.config._getEnvVar() + yumvars['releasever'] = self.config.get('default', 'version') + yumvars['basearch'] = yum.rpmUtils.arch.getBaseArch(myarch=self.config.get('default', 'arch')) + yumconf.yumvar = yumvars + self.ayum._conf = yumconf + self.ayum.repos.setCacheDir(self.ayum.conf.cachedir) + + arch = self.config.get('default', 'arch') + if arch == 'i386': + yumarch = 'athlon' + elif arch == 'ppc': + yumarch = 'ppc64' + elif arch == 'sparc': + yumarch = 'sparc64v' + else: + yumarch = arch + + self.ayum.compatarch = yumarch + arches = yum.rpmUtils.arch.getArchList(yumarch) + arches.append('src') # throw source in there, filter it later + + # deal with our repos + try: + ksparser.handler.repo.methodToRepo() + except: + pass + + for repo in ksparser.handler.repo.repoList: + self.logger.info('Adding repo %s' % repo.name) + thisrepo = yum.yumRepo.YumRepository(repo.name) + thisrepo.name = repo.name + # add excludes and such here when pykickstart gets them + if repo.mirrorlist: + thisrepo.mirrorlist = yum.parser.varReplace(repo.mirrorlist, self.ayum.conf.yumvar) + self.logger.info('Mirrorlist for repo %s is %s' % (thisrepo.name, thisrepo.mirrorlist)) + else: + thisrepo.baseurl = yum.parser.varReplace(repo.baseurl, self.ayum.conf.yumvar) + self.logger.info('URL for repo %s is %s' % (thisrepo.name, thisrepo.baseurl)) + thisrepo.basecachedir = self.ayum.conf.cachedir + thisrepo.enablegroups = True + thisrepo.failovermethod = 'priority' # This is until yum uses this failover by default + thisrepo.exclude = repo.excludepkgs + thisrepo.includepkgs = repo.includepkgs + if repo.cost: + thisrepo.cost = repo.cost + self.ayum.repos.add(thisrepo) + self.ayum.repos.enableRepo(thisrepo.id) + self.ayum._getRepos(thisrepo=thisrepo.id, doSetup = True) + + self.ayum.repos.setProgressBar(CallBack()) + self.ayum.repos.callback = CallBack() + + # Set the metadata and mirror list to be expired so we always get new ones. + for repo in self.ayum.repos.listEnabled(): + repo.metadata_expire = 0 + repo.mirrorlist_expire = 0 + if os.path.exists(os.path.join(repo.cachedir, 'repomd.xml')): + os.remove(os.path.join(repo.cachedir, 'repomd.xml')) + + self.logger.info('Getting sacks for arches %s' % arches) + self.ayum._getSacks(archlist=arches) + + def _filtersrc(self, po): + """Filter out package objects that are of 'src' arch.""" + + if po.arch == 'src': + return False + + return True + + def verifyCachePkg(self, po, path): # Stolen from yum + """check the package checksum vs the cache + return True if pkg is good, False if not""" + + (csum_type, csum) = po.returnIdSum() + + try: + filesum = yum.misc.checksum(csum_type, path) + except yum.Errors.MiscError: + return False + + if filesum != csum: + return False + + return True + + def getPackageDeps(self, po): + """Add the dependencies for a given package to the + transaction info""" + + self.logger.info('Checking deps of %s.%s' % (po.name, po.arch)) + + reqs = po.requires + provs = po.provides + + for req in reqs: + if self.resolved_deps.has_key(req): + continue + (r,f,v) = req + if r.startswith('rpmlib(') or r.startswith('config('): + continue + if req in provs: + continue + + deps = self.ayum.whatProvides(r, f, v).returnPackages() + if not deps: + self.logger.warn("Unresolvable dependency %s in %s.%s" % (r, po.name, po.arch)) + continue + + depsack = yum.packageSack.ListPackageSack(deps) + + for dep in depsack.returnNewestByNameArch(): + self.ayum.tsInfo.addInstall(dep) + self.logger.info('Added %s.%s for %s.%s' % (dep.name, dep.arch, po.name, po.arch)) + + self.resolved_deps[req] = None + + def getPackagesFromGroup(self, group): + """Get a list of package names from a ksparser group object + + Returns a list of package names""" + + packages = [] + + # Check if we have the group + if not self.ayum.comps.has_group(group.name): + self.logger.error("Group %s not found in comps!" % group) + return packages + + # Get the group object to work with + groupobj = self.ayum.comps.return_group(group.name) + + # Add the mandatory packages + packages.extend(groupobj.mandatory_packages.keys()) + + # Add the default packages unless we don't want them + if group.include == 1: + packages.extend(groupobj.default_packages.keys()) + + # Add the optional packages if we want them + if group.include == 2: + packages.extend(groupobj.default_packages.keys()) + packages.extend(groupobj.optional_packages.keys()) + + # Deal with conditional packages + # Populate a dict with the name of the required package and value + # of the package objects it would bring in. To be used later if + # we match the conditional. + for condreq, cond in groupobj.conditional_packages.iteritems(): + pkgs = self.ayum.pkgSack.searchNevra(name=condreq) + if pkgs: + pkgs = self.ayum.bestPackagesFromList(pkgs, arch=self.ayum.compatarch) + if self.ayum.tsInfo.conditionals.has_key(cond): + self.ayum.tsInfo.conditionals[cond].extend(pkgs) + else: + self.ayum.tsInfo.conditionals[cond] = pkgs + + return packages + + def getPackageObjects(self): + """Cycle through the list of packages, get package object + matches, and resolve deps. + + Returns a list of package objects""" + + final_pkgobjs = {} # The final list of package objects + searchlist = [] # The list of package names/globs to search for + matchdict = {} # A dict of objects to names + + # First remove the excludes + self.ayum.conf.exclude.extend(self.ksparser.handler.packages.excludedList) + self.ayum.excludePackages() + + # Always add the core groiup + self.ksparser.handler.packages.add(['@core']) + + # Check to see if we need the base group + if self.ksparser.handler.packages.addBase: + self.ksparser.handler.packages.add(['@base']) + + # Get a list of packages from groups + for group in self.ksparser.handler.packages.groupList: + searchlist.extend(self.getPackagesFromGroup(group)) + + # Add the adds + searchlist.extend(self.ksparser.handler.packages.packageList) + + # Make the search list unique + searchlist = yum.misc.unique(searchlist) + + # Search repos for things in our searchlist, supports globs + (exactmatched, matched, unmatched) = yum.packages.parsePackages(self.ayum.pkgSack.returnPackages(), searchlist, casematch=1) + matches = filter(self._filtersrc, exactmatched + matched) + + # Populate a dict of package objects to their names + for match in matches: + matchdict[match.name] = match + + # Get the newest results from the search + mysack = yum.packageSack.ListPackageSack(matches) + for match in mysack.returnNewestByNameArch(): + self.ayum.tsInfo.addInstall(match) + self.logger.debug('Found %s.%s' % (match.name, match.arch)) + + for pkg in unmatched: + if not pkg in matchdict.keys(): + self.logger.warn('Could not find a match for %s in any configured repo' % pkg) + + if len(self.ayum.tsInfo) == 0: + raise yum.Errors.MiscError, 'No packages found to download.' + + moretoprocess = True + while moretoprocess: # Our fun loop + moretoprocess = False + for txmbr in self.ayum.tsInfo: + if not final_pkgobjs.has_key(txmbr.po): + final_pkgobjs[txmbr.po] = None # Add the pkg to our final list + self.getPackageDeps(txmbr.po) # Get the deps of our package + moretoprocess = True + + self.polist = final_pkgobjs.keys() + self.logger.info('Finished gathering package objects.') + + def getSRPMList(self): + """Cycle through the list of package objects and + find the sourcerpm for them. Requires yum still + configured and a list of package objects""" + + for po in self.polist: + srpm = po.sourcerpm.split('.src.rpm')[0] + if not srpm in self.srpmlist: + self.srpmlist.append(srpm) + + def _downloadPackageList(self, polist, relpkgdir): + """Cycle through the list of package objects and + download them from their respective repos.""" + + downloads = [] + for pkg in polist: + downloads.append('%s.%s' % (pkg.name, pkg.arch)) + downloads.sort() + self.logger.info("Download list: %s" % downloads) + + pkgdir = os.path.join(self.config.get('default', 'destdir'), + self.config.get('default', 'version'), + self.config.get('default', 'flavor'), + relpkgdir) + + # Ensure the pkgdir exists, force if requested, and make sure we clean it out + if relpkgdir.endswith('SRPMS'): + # Since we share source dirs with other arches don't clean, but do allow us to use it + pypungi._ensuredir(pkgdir, self.logger, force=True, clean=False) + else: + pypungi._ensuredir(pkgdir, self.logger, force=self.config.getboolean('default', 'force'), clean=True) + + probs = self.ayum.downloadPkgs(polist) + + if len(probs.keys()) > 0: + self.logger.error("Errors were encountered while downloading packages.") + for key in probs.keys(): + errors = yum.misc.unique(probs[key]) + for error in errors: + self.logger.error("%s: %s" % (key, error)) + sys.exit(1) + + for po in polist: + basename = os.path.basename(po.relativepath) + + local = po.localPkg() + target = os.path.join(pkgdir, basename) + + # Link downloaded package in (or link package from file repo) + try: + pypungi._link(local, target, self.logger, force=True) + continue + except: + self.logger.error("Unable to link %s from the yum cache." % po.name) + sys.exit(1) + + self.logger.info('Finished downloading packages.') + + def downloadPackages(self): + """Download the package objects obtained in getPackageObjects().""" + + self._downloadPackageList(self.polist, + os.path.join(self.config.get('default', 'arch'), + self.config.get('default', 'osdir'), + self.config.get('default', 'product_path'))) + + def makeCompsFile(self): + """Gather any comps files we can from repos and merge them into one.""" + + ourcompspath = os.path.join(self.workdir, '%s-%s-comps.xml' % (self.config.get('default', 'name'), self.config.get('default', 'version'))) + + ourcomps = open(ourcompspath, 'w') + + ourcomps.write(self.ayum.comps.xml()) + + ourcomps.close() + + # Disable this until https://bugzilla.redhat.com/show_bug.cgi?id=442097 is fixed. + # Run the xslt filter over our comps file + #compsfilter = ['/usr/bin/xsltproc', '--novalid'] + #compsfilter.append('-o') + #compsfilter.append(ourcompspath) + #compsfilter.append('/usr/share/pungi/comps-cleanup.xsl') + #compsfilter.append(ourcompspath) + + #pypungi._doRunCommand(compsfilter, self.logger) + + def downloadSRPMs(self): + """Cycle through the list of srpms and + find the package objects for them, Then download them.""" + + srpmpolist = [] + + for srpm in self.srpmlist: + (sname, sver, srel) = srpm.rsplit('-', 2) + try: + srpmpo = self.ayum.pkgSack.searchNevra(name=sname, ver=sver, rel=srel, arch='src')[0] + if not srpmpo in srpmpolist: + srpmpolist.append(srpmpo) + except IndexError: + print >> sys.stderr, "Error: Cannot find a source rpm for %s" % srpm + sys.exit(1) + + # do the downloads + self._downloadPackageList(srpmpolist, os.path.join('source', 'SRPMS')) + def writeinfo(self, line): """Append a line to the infofile in self.infofile"""