almalinux/pungi
15
2
Fork 0
Code Issues Pull Requests Releases Activity

Major rewrite to gain performance boost in nogreedy mode.

Browse Source
This commit is contained in:
Daniel Mach 2012-12-18 07:38:26 -05:00 committed by Dennis Gilmore
parent 60803f32f3
commit 17221a33f3
2 changed files with 288 additions and 230 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

37
src/bin/pungi.py
View File

@ -15,7 +15,6 @@
import os import os
import pypungi import pypungi
import pypungi.config import pypungi.config
import yum
import pykickstart.parser import pykickstart.parser
import pykickstart.version import pykickstart.version
import subprocess import subprocess
@ -91,6 +90,10 @@ def main():
config.set('pungi', 'multilib', " ".join(opts.multilib)) config.set('pungi', 'multilib', " ".join(opts.multilib))
if opts.lookaside_repos: if opts.lookaside_repos:
config.set('pungi', 'lookaside_repos', " ".join(opts.lookaside_repos)) config.set('pungi', 'lookaside_repos', " ".join(opts.lookaside_repos))
config.set("pungi", "fulltree", str(bool(opts.fulltree)))
config.set("pungi", "selfhosting", str(bool(opts.selfhosting)))
config.set("pungi", "nosource", str(bool(opts.nosource)))
config.set("pungi", "nodebuginfo", str(bool(opts.nodebuginfo)))
# Actually do work. # Actually do work.
mypungi = pypungi.Pungi(config, ksparser) mypungi = pypungi.Pungi(config, ksparser)
@ -99,27 +102,9 @@ def main():
if opts.do_all or opts.do_gather or opts.do_buildinstall: if opts.do_all or opts.do_gather or opts.do_buildinstall:
mypungi._inityum() # initialize the yum object for things that need it mypungi._inityum() # initialize the yum object for things that need it
if opts.do_all or opts.do_gather: if opts.do_all or opts.do_gather:
mypungi.getPackageObjects() mypungi.gather()
if not opts.nosource or opts.selfhosting or opts.fulltree:
mypungi.createSourceHashes()
mypungi.getSRPMList()
if opts.selfhosting:
mypungi.resolvePackageBuildDeps()
if opts.fulltree:
mypungi.completePackageSet()
if opts.selfhosting and opts.fulltree:
# OUCH.
while 1:
plen = len(mypungi.srpmpolist)
mypungi.resolvePackageBuildDeps()
if plen == len(mypungi.srpmpolist):
break
plen = len(mypungi.srpmpolist)
mypungi.completePackageSet()
if plen == len(mypungi.srpmpolist):
break
if opts.nodownload: if opts.nodownload:
for line in mypungi.listPackages(): for line in mypungi.list_packages():
sys.stdout.write("RPM: %s\n" % line) sys.stdout.write("RPM: %s\n" % line)
sys.stdout.flush() sys.stdout.flush()
else: else:
@ -128,19 +113,25 @@ def main():
if not opts.nodebuginfo: if not opts.nodebuginfo:
mypungi.getDebuginfoList() mypungi.getDebuginfoList()
if opts.nodownload: if opts.nodownload:
for line in mypungi.listDebuginfo(): for line in mypungi.list_debuginfo():
sys.stdout.write("DEBUGINFO: %s\n" % line) sys.stdout.write("DEBUGINFO: %s\n" % line)
sys.stdout.flush() sys.stdout.flush()
else: else:
mypungi.downloadDebuginfo() mypungi.downloadDebuginfo()
if not opts.nosource: if not opts.nosource:
if opts.nodownload: if opts.nodownload:
for line in mypungi.listSRPMs(): for line in mypungi.list_srpms():
sys.stdout.write("SRPM: %s\n" % line) sys.stdout.write("SRPM: %s\n" % line)
sys.stdout.flush() sys.stdout.flush()
else: else:
mypungi.downloadSRPMs() mypungi.downloadSRPMs()
print "RPM size: %s MiB" % (mypungi.size_packages() / 1024 ** 2)
if not opts.nodebuginfo:
print "DEBUGINFO size: %s MiB" % (mypungi.size_debuginfo() / 1024 ** 2)
if not opts.nosource:
print "SRPM size: %s MiB" % (mypungi.size_srpms() / 1024 ** 2)
if opts.do_all or opts.do_createrepo: if opts.do_all or opts.do_createrepo:
mypungi.doCreaterepo() mypungi.doCreaterepo()

481
src/pypungi/__init__.py
View File

@ -75,16 +75,13 @@ class PungiBase(object):
# ARCH setup # ARCH setup
self.tree_arch = self.config.get('pungi', 'arch') self.tree_arch = self.config.get('pungi', 'arch')
self.yum_arch = arch_module.tree_arch_to_yum_arch(arch) self.yum_arch = arch_module.tree_arch_to_yum_arch(self.tree_arch)
full_archlist = self.config.getboolean('pungi', 'full_archlist') full_archlist = self.config.getboolean('pungi', 'full_archlist')
self.valid_arches = arch_module.get_valid_arches(self.tree_arch, multilib=full_archlist) self.valid_arches = arch_module.get_valid_arches(self.tree_arch, multilib=full_archlist)
self.valid_arches.append("src") # throw source in there, filter it later self.valid_arches.append("src") # throw source in there, filter it later
self.valid_native_arches = arch_module.get_valid_arches(self.tree_arch, multilib=False) self.valid_native_arches = arch_module.get_valid_arches(self.tree_arch, multilib=False)
self.valid_multilib_arches = arch_module.get_valid_multilib_arches(self.tree_arch) self.valid_multilib_arches = arch_module.get_valid_multilib_arches(self.tree_arch)
# --nogreedy
self.greedy = config.getboolean('pungi', 'alldeps')
# arch: compatible arches # arch: compatible arches
self.compatible_arches = {} self.compatible_arches = {}
for i in self.valid_arches: for i in self.valid_arches:
@ -131,11 +128,6 @@ class PungiYum(yum.YumBase):
self.pungiconfig = config self.pungiconfig = config
yum.YumBase.__init__(self) yum.YumBase.__init__(self)
def _checkInstall(self, txmbr):
# overriding this method allows us to ignore installed packages
# and always prefer native packages over those who pull less deps into a transaction
return []
def doLoggingSetup(self, debuglevel, errorlevel, syslog_ident=None, syslog_facility=None): def doLoggingSetup(self, debuglevel, errorlevel, syslog_ident=None, syslog_facility=None):
"""Setup the logging facility.""" """Setup the logging facility."""
@ -155,6 +147,17 @@ class PungiYum(yum.YumBase):
# the logging. # the logging.
pass pass
def _compare_providers(self, *args, **kwargs):
# HACK: always prefer 64bit over 32bit packages
result = yum.YumBase._compare_providers(self, *args, **kwargs)
if len(result) >= 2:
pkg1 = result[0][0]
pkg2 = result[1][0]
if pkg1.name == pkg2.name:
best_arch = self.arch.get_best_arch_from_list([pkg1.arch, pkg2.arch], self.arch.canonarch)
if best_arch != "noarch" and best_arch != pkg1.arch:
result[0:1] = result[0:1:-1]
return result
class Pungi(pypungi.PungiBase): class Pungi(pypungi.PungiBase):
def __init__(self, config, ksparser): def __init__(self, config, ksparser):
@ -188,12 +191,7 @@ class Pungi(pypungi.PungiBase):
self.ksparser = ksparser self.ksparser = ksparser
self.polist = []
self.srpmpolist = []
self.debuginfolist = []
self.srpms_build = []
self.srpms_fulltree = []
self.last_po = 0
self.resolved_deps = {} # list the deps we've already resolved, short circuit. self.resolved_deps = {} # list the deps we've already resolved, short circuit.
self.excluded_pkgs = {} # list the packages we've already excluded. self.excluded_pkgs = {} # list the packages we've already excluded.
self.seen_pkgs = {} # list the packages we've already seen so we can check all deps only once self.seen_pkgs = {} # list the packages we've already seen so we can check all deps only once
@ -201,28 +199,45 @@ class Pungi(pypungi.PungiBase):
self.lookaside_repos = self.config.get('pungi', 'lookaside_repos').split(" ") self.lookaside_repos = self.config.get('pungi', 'lookaside_repos').split(" ")
self.sourcerpm_arch_map = {} # {sourcerpm: set[arches]} - used for gathering debuginfo self.sourcerpm_arch_map = {} # {sourcerpm: set[arches]} - used for gathering debuginfo
# package object lists
self.po_list = set()
self.srpm_po_list = set()
self.debuginfo_po_list = set()
# get_srpm_po() cache
self.sourcerpm_srpmpo_map = {}
# already processed packages
self.completed_add_srpms = set() # srpms
self.completed_debuginfo = set() # rpms
self.completed_depsolve = set() # rpms
self.completed_langpacks = set() # rpms
self.completed_multilib = set() # rpms
self.completed_fulltree = set() # srpms
self.completed_selfhosting = set() # srpms
self.is_fulltree = self.config.getboolean("pungi", "fulltree")
self.is_selfhosting = self.config.getboolean("pungi", "selfhosting")
self.is_sources = not self.config.getboolean("pungi", "nosource")
self.is_debuginfo = not self.config.getboolean("pungi", "nodebuginfo")
self.is_greedy = self.config.getboolean("pungi", "alldeps")
self.is_resolve_deps = True # TODO: implement --nodepsolve
def _add_yum_repo(self, name, url, mirrorlist=False, groups=True, def _add_yum_repo(self, name, url, mirrorlist=False, groups=True,
cost=1000, includepkgs=[], excludepkgs=[], cost=1000, includepkgs=None, excludepkgs=None,
proxy=None): proxy=None):
"""This function adds a repo to the yum object. """This function adds a repo to the yum object.
name: Name of the repo name: Name of the repo
url: Full url to the repo url: Full url to the repo
mirrorlist: Bool for whether or not url is a mirrorlist mirrorlist: Bool for whether or not url is a mirrorlist
groups: Bool for whether or not to use groupdata from this repo groups: Bool for whether or not to use groupdata from this repo
cost: an optional int representing the cost of a repo cost: an optional int representing the cost of a repo
includepkgs: An optional list of includes to use includepkgs: An optional list of includes to use
excludepkgs: An optional list of excludes to use excludepkgs: An optional list of excludes to use
proxy: An optional proxy to use proxy: An optional proxy to use
""" """
includepkgs = includepkgs or []
excludepkgs = excludepkgs or []
self.logger.info('Adding repo %s' % name) self.logger.info('Adding repo %s' % name)
thisrepo = yum.yumRepo.YumRepository(name) thisrepo = yum.yumRepo.YumRepository(name)
@ -288,6 +303,8 @@ class Pungi(pypungi.PungiBase):
del self.ayum.prerepoconf del self.ayum.prerepoconf
self.ayum.repos.setCacheDir(self.ayum.conf.cachedir) self.ayum.repos.setCacheDir(self.ayum.conf.cachedir)
self.ayum.arch.setup_arch(self.yum_arch)
# deal with our repos # deal with our repos
try: try:
self.ksparser.handler.repo.methodToRepo() self.ksparser.handler.repo.methodToRepo()
@ -331,8 +348,8 @@ class Pungi(pypungi.PungiBase):
raise ValueError("Not a binary package: %s" % po) raise ValueError("Not a binary package: %s" % po)
if msg: if msg:
self.logger.info(msg) self.logger.info(msg)
if po not in self.polist: if po not in self.po_list:
self.polist.append(po) self.po_list.add(po)
self.ayum.install(po) self.ayum.install(po)
self.sourcerpm_arch_map.setdefault(po.sourcerpm, set()).add(po.arch) self.sourcerpm_arch_map.setdefault(po.sourcerpm, set()).add(po.arch)
@ -341,16 +358,16 @@ class Pungi(pypungi.PungiBase):
raise ValueError("Not a debuginfog package: %s" % po) raise ValueError("Not a debuginfog package: %s" % po)
if msg: if msg:
self.logger.info(msg) self.logger.info(msg)
if po not in self.debuginfolist: if po not in self.debuginfo_po_list:
self.debuginfolist.append(po) self.debuginfo_po_list.add(po)
def add_source(self, po, msg=None): def add_source(self, po, msg=None):
if not is_source(po): if not is_source(po):
raise ValueError("Not a source package: %s" % po) raise ValueError("Not a source package: %s" % po)
if msg: if msg:
self.logger.info(msg) self.logger.info(msg)
if po not in self.srpmpolist: if po not in self.srpm_po_list:
self.srpmpolist.append(po) self.srpm_po_list.add(po)
def verifyCachePkg(self, po, path): # Stolen from yum def verifyCachePkg(self, po, path): # Stolen from yum
"""check the package checksum vs the cache """check the package checksum vs the cache
@ -397,21 +414,18 @@ class Pungi(pypungi.PungiBase):
return pkg_sack return pkg_sack
def getPackageDeps(self, po): def get_package_deps(self, po):
"""Add the dependencies for a given package to the """Add the dependencies for a given package to the
transaction info""" transaction info"""
if po in self.seen_pkgs: added = set()
return if po in self.completed_depsolve:
self.seen_pkgs[po] = None return added
self.completed_depsolve.add(po)
self.logger.info('Checking deps of %s.%s' % (po.name, po.arch)) self.logger.info('Checking deps of %s.%s' % (po.name, po.arch))
reqs = po.requires reqs = po.requires
provs = po.provides provs = po.provides
added = set()
added.update(self.getLangpacks([po]))
added.update(self.getMultilib([po]))
for req in reqs: for req in reqs:
if req in self.resolved_deps: if req in self.resolved_deps:
@ -429,12 +443,12 @@ class Pungi(pypungi.PungiBase):
self.logger.warn("Unresolvable dependency %s in %s.%s" % (r, po.name, po.arch)) self.logger.warn("Unresolvable dependency %s in %s.%s" % (r, po.name, po.arch))
continue continue
if self.greedy: if self.is_greedy:
deps = yum.packageSack.ListPackageSack(deps).returnNewestByNameArch() deps = yum.packageSack.ListPackageSack(deps).returnNewestByNameArch()
else: else:
found = False found = False
for dep in deps: for dep in deps:
if dep in self.polist: if dep in self.po_list:
found = True found = True
break break
if found: if found:
@ -454,20 +468,27 @@ class Pungi(pypungi.PungiBase):
self.resolved_deps[req] = None self.resolved_deps[req] = None
for add in added: for add in added:
self.getPackageDeps(add) self.get_package_deps(add)
return added
def getLangpacks(self, po_list): def add_langpacks(self, po_list=None):
added = [] po_list = po_list or self.po_list
added = set()
for po in sorted(po_list):
if po in self.completed_langpacks:
continue
for po in po_list:
# get all langpacks matching the package name # get all langpacks matching the package name
langpacks = [ i for i in self.langpacks if i["name"] == po.name ] langpacks = [ i for i in self.langpacks if i["name"] == po.name ]
if not langpacks: if not langpacks:
continue continue
self.completed_langpacks.add(po)
for langpack in langpacks: for langpack in langpacks:
pattern = langpack["install"] % "*" # replace '%s' with '*' pattern = langpack["install"] % "*" # replace '%s' with '*'
exactmatched, matched, unmatched = yum.packages.parsePackages(self.pkgs, [pattern], casematch=1, pkgdict=self.pkg_refs.copy()) exactmatched, matched, unmatched = yum.packages.parsePackages(self.all_pkgs, [pattern], casematch=1, pkgdict=self.pkg_refs.copy())
matches = filter(self._filtersrcdebug, exactmatched + matched) matches = filter(self._filtersrcdebug, exactmatched + matched)
matches = [ i for i in matches if not i.name.endswith("-devel") and not i.name.endswith("-static") and i.name != "man-pages-overrides" ] matches = [ i for i in matches if not i.name.endswith("-devel") and not i.name.endswith("-static") and i.name != "man-pages-overrides" ]
matches = [ i for i in matches if fnmatch(i.name, pattern) ] matches = [ i for i in matches if fnmatch(i.name, pattern) ]
@ -481,23 +502,30 @@ class Pungi(pypungi.PungiBase):
match = self.ayum._bestPackageFromList(pkg_sack) match = self.ayum._bestPackageFromList(pkg_sack)
msg = 'Added langpack %s.%s for package %s (pattern: %s)' % (match.name, match.arch, po.name, pattern) msg = 'Added langpack %s.%s for package %s (pattern: %s)' % (match.name, match.arch, po.name, pattern)
self.add_package(match, msg) self.add_package(match, msg)
added.append(match) self.completed_langpacks.add(match) # assuming langpack doesn't have langpacks
added.add(match)
return added return added
def getMultilib(self, po_list): def add_multilib(self, po_list=None):
added = [] po_list = po_list or self.po_list
added = set()
if not self.multilib_methods: if not self.multilib_methods:
return added return added
for po in po_list: for po in sorted(po_list):
if po in self.completed_multilib:
continue
if po.arch in ("noarch", "src", "nosrc"): if po.arch in ("noarch", "src", "nosrc"):
continue continue
if po.arch in self.valid_multilib_arches: if po.arch in self.valid_multilib_arches:
continue continue
self.completed_multilib.add(po)
matches = self.ayum.pkgSack.searchNevra(name=po.name, ver=po.version, rel=po.release) matches = self.ayum.pkgSack.searchNevra(name=po.name, ver=po.version, rel=po.release)
matches = [i for i in matches if i.arch in self.valid_multilib_arches] matches = [i for i in matches if i.arch in self.valid_multilib_arches]
if not matches: if not matches:
@ -511,7 +539,8 @@ class Pungi(pypungi.PungiBase):
continue continue
msg = "Added multilib package %s.%s for package %s.%s (method: %s)" % (match.name, match.arch, po.name, po.arch, method) msg = "Added multilib package %s.%s for package %s.%s (method: %s)" % (match.name, match.arch, po.name, po.arch, method)
self.add_package(match, msg) self.add_package(match, msg)
added.append(match) self.completed_multilib.add(match)
added.add(match)
return added return added
def getPackagesFromGroup(self, group): def getPackagesFromGroup(self, group):
@ -546,19 +575,18 @@ class Pungi(pypungi.PungiBase):
# of the package objects it would bring in. To be used later if # of the package objects it would bring in. To be used later if
# we match the conditional. # we match the conditional.
for condreq, cond in groupobj.conditional_packages.iteritems(): for condreq, cond in groupobj.conditional_packages.iteritems():
pkgs = self.ayum.pkgSack.searchNevra(name=condreq) matches = self.ayum.pkgSack.searchNevra(name=condreq)
if pkgs: if matches:
pkgs = self.ayum.bestPackagesFromList(pkgs, arch=self.ayum.compatarch) if not self.is_greedy:
if self.ayum.tsInfo.conditionals.has_key(cond): matches = [self.ayum._bestPackageFromList(matches)]
self.ayum.tsInfo.conditionals[cond].extend(pkgs) self.ayum.tsInfo.conditionals.setdefault(cond, []).extend(matches)
else:
self.ayum.tsInfo.conditionals[cond] = pkgs
return packages return packages
def _addDefaultGroups(self, excludeGroups=[]): def _addDefaultGroups(self, excludeGroups=None):
"""Cycle through the groups and return at list of the ones that ara """Cycle through the groups and return at list of the ones that ara
default.""" default."""
excludeGroups = excludeGroups or []
# This is mostly stolen from anaconda. # This is mostly stolen from anaconda.
groups = map(lambda x: x.groupid, groups = map(lambda x: x.groupid,
@ -569,21 +597,7 @@ class Pungi(pypungi.PungiBase):
self.logger.debug('Add default groups %s' % groups) self.logger.debug('Add default groups %s' % groups)
return groups return groups
def getPackageObjects(self): def get_langpacks(self):
"""Cycle through the list of packages, get package object
matches, and resolve deps.
Returns a list of package objects"""
final_pkgobjs = {} # The final list of package objects
searchlist = [] # The list of package names/globs to search for
matchdict = {} # A dict of objects to names
excludeGroups = [] #A list of groups for removal defined in the ks file
# precompute pkgs and pkg_refs to speed things up
self.pkgs = self.ayum.pkgSack.returnPackages()
self.pkg_refs = yum.packages.buildPkgRefDict(self.pkgs, casematch=True)
try: try:
self.langpacks = list(self.ayum.comps.langpacks) self.langpacks = list(self.ayum.comps.langpacks)
except AttributeError: except AttributeError:
@ -595,6 +609,19 @@ class Pungi(pypungi.PungiBase):
self.logger.warning("Could not get langpacks due to missing comps in repodata or --ignoregroups=true option.") self.logger.warning("Could not get langpacks due to missing comps in repodata or --ignoregroups=true option.")
self.langpacks = [] self.langpacks = []
def getPackageObjects(self):
"""Cycle through the list of packages and get package object matches."""
searchlist = [] # The list of package names/globs to search for
matchdict = {} # A dict of objects to names
excludeGroups = [] # A list of groups for removal defined in the ks file
# precompute pkgs and pkg_refs to speed things up
self.all_pkgs = self.ayum.pkgSack.returnPackages()
self.pkg_refs = yum.packages.buildPkgRefDict(self.all_pkgs, casematch=True)
self.get_langpacks()
# First remove the excludes # First remove the excludes
self.ayum.excludePackages() self.ayum.excludePackages()
@ -634,16 +661,16 @@ class Pungi(pypungi.PungiBase):
name = name[:-2] name = name[:-2]
multilib = True multilib = True
if self.greedy and name == "system-release": if self.is_greedy and name == "system-release":
# HACK: handles a special case, when system-release virtual provide is specified in the greedy mode # HACK: handles a special case, when system-release virtual provide is specified in the greedy mode
matches = self.ayum.whatProvides(name, None, None).returnPackages() matches = self.ayum.whatProvides(name, None, None).returnPackages()
else: else:
exactmatched, matched, unmatched = yum.packages.parsePackages(self.pkgs, [name], casematch=1, pkgdict=self.pkg_refs.copy()) exactmatched, matched, unmatched = yum.packages.parsePackages(self.all_pkgs, [name], casematch=1, pkgdict=self.pkg_refs.copy())
matches = exactmatched + matched matches = exactmatched + matched
matches = filter(self._filtersrcdebug, matches) matches = filter(self._filtersrcdebug, matches)
if multilib and not self.greedy: if multilib and not self.is_greedy:
matches = [ po for po in matches if po.arch in self.valid_multilib_arches ] matches = [ po for po in matches if po.arch in self.valid_multilib_arches ]
if not matches: if not matches:
@ -656,7 +683,7 @@ class Pungi(pypungi.PungiBase):
for name, packages in packages_by_name.iteritems(): for name, packages in packages_by_name.iteritems():
packages = self.excludePackages(packages) packages = self.excludePackages(packages)
if self.greedy: if self.is_greedy:
packages = yum.packageSack.ListPackageSack(packages).returnNewestByNameArch() packages = yum.packageSack.ListPackageSack(packages).returnNewestByNameArch()
else: else:
packages = [self.ayum._bestPackageFromList(packages)] packages = [self.ayum._bestPackageFromList(packages)]
@ -665,33 +692,75 @@ class Pungi(pypungi.PungiBase):
msg = 'Found %s.%s' % (po.name, po.arch) msg = 'Found %s.%s' % (po.name, po.arch)
self.add_package(po, msg) self.add_package(po, msg)
if len(self.ayum.tsInfo) == 0: if not self.po_list:
raise yum.Errors.MiscError, 'No packages found to download.' raise RuntimeError("No packages found")
moretoprocess = True
while moretoprocess: # Our fun loop
moretoprocess = False
for txmbr in self.ayum.tsInfo:
if not final_pkgobjs.has_key(txmbr.po):
final_pkgobjs[txmbr.po] = None # Add the pkg to our final list
self.getPackageDeps(txmbr.po) # Get the deps of our package
moretoprocess = True
self.polist = final_pkgobjs.keys()
self.logger.info('Finished gathering package objects.') self.logger.info('Finished gathering package objects.')
def getSRPMPo(self, po): def gather(self):
"""Given a package object, get a package object for the
corresponding source rpm. Requires yum still configured # get package objects according to the input list
and a valid package object.""" self.getPackageObjects()
self.createSourceHashes()
pass_num = 0
added = set()
while 1:
if pass_num > 0 and not added:
break
added = set()
pass_num += 1
self.logger.info("Pass #%s" % pass_num)
for txmbr in self.ayum.tsInfo:
if not txmbr.po in self.po_list:
self.add_package(txmbr.po)
# resolve deps
if self.is_resolve_deps:
for po in sorted(self.po_list):
added.update(self.get_package_deps(po))
added_srpms = self.add_srpms()
added.update(added_srpms)
if self.is_selfhosting:
for srpm_po in sorted(added_srpms):
added.update(self.get_package_deps(srpm_po))
if self.is_fulltree:
added.update(self.add_fulltree())
if added:
continue
# add langpacks
added.update(self.add_langpacks(self.po_list))
if added:
continue
# add multilib packages
added.update(self.add_multilib(self.po_list))
if added:
continue
def get_srpm_po(self, po):
"""Given a package object, get a package object for the corresponding source rpm."""
# return srpm_po from cache if available
srpm_po = self.sourcerpm_srpmpo_map.get(po.sourcerpm, None)
if srpm_po is not None:
return srpm_po
# arch can be "src" or "nosrc"
nvr, arch, _ = po.sourcerpm.rsplit(".", 2) nvr, arch, _ = po.sourcerpm.rsplit(".", 2)
name, ver, rel = nvr.rsplit('-', 2) name, ver, rel = nvr.rsplit('-', 2)
try:
srpmpo = self.ayum.pkgSack.searchNevra(name=name, ver=ver, rel=rel, arch='src')[0] # ... but even "nosrc" packages are stored as "src" in repodata
return srpmpo srpm_po_list = self.ayum.pkgSack.searchNevra(name=name, ver=ver, rel=rel, arch="src")
except IndexError: if not srpm_po_list:
print >> sys.stderr, "Error: Cannot find a source rpm for %s" % srpm raise RuntimeError("Cannot find a source rpm for %s" % po.sourcerpm)
sys.exit(1) srpm_po = srpm_po_list[0]
self.sourcerpm_srpmpo_map[po.sourcerpm] = srpm_po
return srpm_po
def createSourceHashes(self): def createSourceHashes(self):
"""Create two dicts - one that maps binary POs to source POs, and """Create two dicts - one that maps binary POs to source POs, and
@ -700,127 +769,109 @@ class Pungi(pypungi.PungiBase):
self.src_by_bin = {} self.src_by_bin = {}
self.bin_by_src = {} self.bin_by_src = {}
self.logger.info("Generating source <-> binary package mappings") self.logger.info("Generating source <-> binary package mappings")
(dummy1, everything, dummy2) = yum.packages.parsePackages(self.pkgs, ['*'], pkgdict=self.pkg_refs.copy()) #(dummy1, everything, dummy2) = yum.packages.parsePackages(self.all_pkgs, ['*'], pkgdict=self.pkg_refs.copy())
for po in everything: for po in self.all_pkgs:
if po.arch == 'src': if is_source(po):
continue continue
srpmpo = self.getSRPMPo(po) srpmpo = self.get_srpm_po(po)
self.src_by_bin[po] = srpmpo self.src_by_bin[po] = srpmpo
if self.bin_by_src.has_key(srpmpo): if self.bin_by_src.has_key(srpmpo):
self.bin_by_src[srpmpo].append(po) self.bin_by_src[srpmpo].append(po)
else: else:
self.bin_by_src[srpmpo] = [po] self.bin_by_src[srpmpo] = [po]
def getSRPMList(self): def add_srpms(self, po_list=None):
"""Cycle through the list of package objects and """Cycle through the list of package objects and
find the sourcerpm for them. Requires yum still find the sourcerpm for them. Requires yum still
configured and a list of package objects""" configured and a list of package objects"""
for po in self.polist[self.last_po:]:
srpm_po = self.src_by_bin[po]
if not srpm_po in self.srpmpolist:
msg = "Adding source package %s.%s" % (srpm_po.name, srpm_po.arch)
self.add_source(srpm_po)
self.last_po = len(self.polist)
def resolvePackageBuildDeps(self): srpms = set()
"""Make the package lists self hosting. Requires yum po_list = po_list or self.po_list
still configured, a list of package objects, and a for po in sorted(po_list):
a list of source rpms.""" srpm_po = self.sourcerpm_srpmpo_map[po.sourcerpm]
deppass = 1 if srpm_po in self.completed_add_srpms:
while 1: continue
self.logger.info("Resolving build dependencies, pass %d" % (deppass)) msg = "Adding source package %s.%s" % (srpm_po.name, srpm_po.arch)
prev = list(self.ayum.tsInfo.getMembers()) self.add_source(srpm_po, msg)
for srpm in self.srpmpolist[len(self.srpms_build):]: self.completed_add_srpms.add(srpm_po)
self.getPackageDeps(srpm) srpms.add(srpm_po)
for txmbr in self.ayum.tsInfo: return srpms
if txmbr.po.arch != 'src' and txmbr.po not in self.polist:
self.polist.append(txmbr.po)
self.getPackageDeps(txmbr.po)
self.srpms_build = list(self.srpmpolist)
# Now that we've resolved deps, refresh the source rpm list
self.getSRPMList()
deppass = deppass + 1
if len(prev) == len(self.ayum.tsInfo.getMembers()):
break
def completePackageSet(self): def add_fulltree(self, srpm_po_list=None):
"""Cycle through all package objects, and add any """Cycle through all package objects, and add any
that correspond to a source rpm that we are including. that correspond to a source rpm that we are including.
Requires yum still configured and a list of package Requires yum still configured and a list of package
objects.""" objects."""
thepass = 1
while 1:
prevlen = len(self.srpmpolist)
self.logger.info("Completing package set, pass %d" % (thepass,))
for srpm in self.srpmpolist[len(self.srpms_fulltree):]:
include_native = False self.logger.info("Completing package set")
include_multilib = False
has_native = False srpm_po_list = srpm_po_list or self.srpm_po_list
has_multilib = False srpms = []
for po in self.excludePackages(self.bin_by_src[srpm]): for srpm_po in srpm_po_list:
if not is_package(po): if srpm_po in self.completed_fulltree:
continue continue
if po.arch == "noarch": srpms.append(srpm_po)
continue self.completed_fulltree.add(srpm_po)
if po not in self.polist:
# process only already included packages added = set()
if po.arch in self.valid_multilib_arches: for srpm_po in srpms:
has_multilib = True include_native = False
elif po.arch in self.valid_native_arches: include_multilib = False
has_native = True has_native = False
continue has_multilib = False
for po in self.excludePackages(self.bin_by_src[srpm_po]):
if not is_package(po):
continue
if po.arch == "noarch":
continue
if po not in self.po_list:
# process only already included packages
if po.arch in self.valid_multilib_arches: if po.arch in self.valid_multilib_arches:
include_multilib = True has_multilib = True
elif po.arch in self.valid_native_arches: elif po.arch in self.valid_native_arches:
include_native = True has_native = True
continue
if po.arch in self.valid_multilib_arches:
include_multilib = True
elif po.arch in self.valid_native_arches:
include_native = True
# XXX: this is very fragile! # XXX: this is very fragile!
# Do not make any changes unless you really know what you're doing! # Do not make any changes unless you really know what you're doing!
if not include_native: if not include_native:
# if there's no native package already pulled in... # if there's no native package already pulled in...
if has_native and not include_multilib: if has_native and not include_multilib:
# include all native packages, but only if we're not pulling multilib already # include all native packages, but only if we're not pulling multilib already
# SCENARIO: a noarch package was already pulled in and there are x86_64 and i686 packages -> we want x86_64 in to complete the package set # SCENARIO: a noarch package was already pulled in and there are x86_64 and i686 packages -> we want x86_64 in to complete the package set
include_native = True include_native = True
elif has_multilib: elif has_multilib:
# SCENARIO: a noarch package was already pulled in and there are no x86_64 packages; we want i686 in to complete the package set # SCENARIO: a noarch package was already pulled in and there are no x86_64 packages; we want i686 in to complete the package set
include_multilib = True include_multilib = True
for po in self.excludePackages(self.bin_by_src[srpm]):
if not is_package(po):
continue
if po in self.polist:
continue
if po.arch != "noarch":
if po.arch in self.valid_multilib_arches:
if not include_multilib:
continue
if po.arch in self.valid_native_arches:
if not include_native:
continue
msg = "Adding %s.%s to complete package set" % (po.name, po.arch)
self.add_package(po, msg)
self.getPackageDeps(po)
for txmbr in self.ayum.tsInfo:
if txmbr.po.arch != 'src' and txmbr.po not in self.polist:
self.polist.append(txmbr.po)
self.getPackageDeps(po)
self.srpms_fulltree = list(self.srpmpolist)
# Now that we've resolved deps, refresh the source rpm list
self.getSRPMList()
if len(self.srpmpolist) == prevlen:
self.logger.info("Completion finished in %d passes" % (thepass,))
break
thepass = thepass + 1
for po in self.excludePackages(self.bin_by_src[srpm_po]):
if not is_package(po):
continue
if po in self.po_list:
continue
if po.arch != "noarch":
if po.arch in self.valid_multilib_arches:
if not include_multilib:
continue
if po.arch in self.valid_native_arches:
if not include_native:
continue
msg = "Adding %s.%s to complete package set" % (po.name, po.arch)
self.add_package(po, msg)
return added
def getDebuginfoList(self): def getDebuginfoList(self):
"""Cycle through the list of package objects and find """Cycle through the list of package objects and find
debuginfo rpms for them. Requires yum still debuginfo rpms for them. Requires yum still
configured and a list of package objects""" configured and a list of package objects"""
for po in self.pkgs: added = set()
for po in self.all_pkgs:
if not is_debug(po): if not is_debug(po):
continue continue
@ -833,6 +884,8 @@ class Pungi(pypungi.PungiBase):
continue continue
msg = 'Added debuginfo %s.%s' % (po.name, po.arch) msg = 'Added debuginfo %s.%s' % (po.name, po.arch)
self.add_debuginfo(po, msg) self.add_debuginfo(po, msg)
added.add(po)
return added
def _downloadPackageList(self, polist, relpkgdir): def _downloadPackageList(self, polist, relpkgdir):
"""Cycle through the list of package objects and """Cycle through the list of package objects and
@ -891,7 +944,7 @@ class Pungi(pypungi.PungiBase):
def downloadPackages(self): def downloadPackages(self):
"""Download the package objects obtained in getPackageObjects().""" """Download the package objects obtained in getPackageObjects()."""
self._downloadPackageList(self.polist, self._downloadPackageList(self.po_list,
os.path.join(self.tree_arch, os.path.join(self.tree_arch,
self.config.get('pungi', 'osdir'), self.config.get('pungi', 'osdir'),
self.config.get('pungi', 'product_path'))) self.config.get('pungi', 'product_path')))
@ -941,30 +994,44 @@ class Pungi(pypungi.PungiBase):
find the package objects for them, Then download them.""" find the package objects for them, Then download them."""
# do the downloads # do the downloads
self._downloadPackageList(self.srpmpolist, os.path.join('source', 'SRPMS')) self._downloadPackageList(self.srpm_po_list, os.path.join('source', 'SRPMS'))
def downloadDebuginfo(self): def downloadDebuginfo(self):
"""Cycle through the list of debuginfo rpms and """Cycle through the list of debuginfo rpms and
download them.""" download them."""
# do the downloads # do the downloads
self._downloadPackageList(self.debuginfolist, os.path.join(self.tree_arch, 'debug')) self._downloadPackageList(self.debuginfo_po_list, os.path.join(self.tree_arch, 'debug'))
def _listPackages(self, polist): def _list_packages(self, po_list):
"""Cycle through the list of packages and return their paths.""" """Cycle through the list of packages and return their paths."""
return [ os.path.join(pkg.basepath or "", pkg.relativepath) for pkg in polist if pkg.repoid not in self.lookaside_repos ] result = [ os.path.join(po.basepath or "", po.relativepath) for po in po_list if po.repoid not in self.lookaside_repos ]
result.sort()
return result
def listPackages(self): def list_packages(self):
"""Cycle through the list of RPMs and return their paths.""" """Cycle through the list of RPMs and return their paths."""
return self._listPackages(self.polist) return self._list_packages(self.po_list)
def listSRPMs(self): def list_srpms(self):
"""Cycle through the list of SRPMs and return their paths.""" """Cycle through the list of SRPMs and return their paths."""
return self._listPackages(self.srpmpolist) return self._list_packages(self.srpm_po_list)
def listDebuginfo(self): def list_debuginfo(self):
"""Cycle through the list of DEBUGINFO RPMs and return their paths.""" """Cycle through the list of DEBUGINFO RPMs and return their paths."""
return self._listPackages(self.debuginfolist) return self._list_packages(self.debuginfo_po_list)
def _size_packages(self, po_list):
return sum([ po.size for po in po_list if po.repoid not in self.lookaside_repos ])
def size_packages(self):
return self._size_packages(self.po_list)
def size_srpms(self):
return self._size_packages(self.srpm_po_list)
def size_debuginfo(self):
return self._size_packages(self.debuginfo_po_list)
def writeinfo(self, line): def writeinfo(self, line):
"""Append a line to the infofile in self.infofile""" """Append a line to the infofile in self.infofile"""
@ -996,7 +1063,7 @@ class Pungi(pypungi.PungiBase):
conf.directory = path conf.directory = path
conf.database = True conf.database = True
if comps: if comps:
conf.groupfile = comps conf.groupfile = comps
if basedir: if basedir:
conf.basedir = basedir conf.basedir = basedir
if baseurl: if baseurl:
@ -1241,7 +1308,7 @@ class Pungi(pypungi.PungiBase):
self.logger.info("ARCH: arm, not doing doCreateIsos().") self.logger.info("ARCH: arm, not doing doCreateIsos().")
return return
isolist=[] isolist = []
ppcbootinfo = '/usr/share/lorax/config_files/ppc' ppcbootinfo = '/usr/share/lorax/config_files/ppc'
pypungi.util._ensuredir(self.isodir, self.logger, pypungi.util._ensuredir(self.isodir, self.logger,