diff --git a/src/bin/pungi.py b/src/bin/pungi.py index 1d72ae3c..c1e71bc0 100755 --- a/src/bin/pungi.py +++ b/src/bin/pungi.py @@ -15,7 +15,6 @@ import os import pypungi import pypungi.config -import yum import pykickstart.parser import pykickstart.version import subprocess @@ -91,6 +90,10 @@ def main(): config.set('pungi', 'multilib', " ".join(opts.multilib)) if opts.lookaside_repos: config.set('pungi', 'lookaside_repos', " ".join(opts.lookaside_repos)) + config.set("pungi", "fulltree", str(bool(opts.fulltree))) + config.set("pungi", "selfhosting", str(bool(opts.selfhosting))) + config.set("pungi", "nosource", str(bool(opts.nosource))) + config.set("pungi", "nodebuginfo", str(bool(opts.nodebuginfo))) # Actually do work. mypungi = pypungi.Pungi(config, ksparser) @@ -99,27 +102,9 @@ def main(): if opts.do_all or opts.do_gather or opts.do_buildinstall: mypungi._inityum() # initialize the yum object for things that need it if opts.do_all or opts.do_gather: - mypungi.getPackageObjects() - if not opts.nosource or opts.selfhosting or opts.fulltree: - mypungi.createSourceHashes() - mypungi.getSRPMList() - if opts.selfhosting: - mypungi.resolvePackageBuildDeps() - if opts.fulltree: - mypungi.completePackageSet() - if opts.selfhosting and opts.fulltree: - # OUCH. - while 1: - plen = len(mypungi.srpmpolist) - mypungi.resolvePackageBuildDeps() - if plen == len(mypungi.srpmpolist): - break - plen = len(mypungi.srpmpolist) - mypungi.completePackageSet() - if plen == len(mypungi.srpmpolist): - break + mypungi.gather() if opts.nodownload: - for line in mypungi.listPackages(): + for line in mypungi.list_packages(): sys.stdout.write("RPM: %s\n" % line) sys.stdout.flush() else: @@ -128,19 +113,25 @@ def main(): if not opts.nodebuginfo: mypungi.getDebuginfoList() if opts.nodownload: - for line in mypungi.listDebuginfo(): + for line in mypungi.list_debuginfo(): sys.stdout.write("DEBUGINFO: %s\n" % line) sys.stdout.flush() else: mypungi.downloadDebuginfo() if not opts.nosource: if opts.nodownload: - for line in mypungi.listSRPMs(): + for line in mypungi.list_srpms(): sys.stdout.write("SRPM: %s\n" % line) sys.stdout.flush() else: mypungi.downloadSRPMs() + print "RPM size: %s MiB" % (mypungi.size_packages() / 1024 ** 2) + if not opts.nodebuginfo: + print "DEBUGINFO size: %s MiB" % (mypungi.size_debuginfo() / 1024 ** 2) + if not opts.nosource: + print "SRPM size: %s MiB" % (mypungi.size_srpms() / 1024 ** 2) + if opts.do_all or opts.do_createrepo: mypungi.doCreaterepo() diff --git a/src/pypungi/__init__.py b/src/pypungi/__init__.py index 67aff859..e811b9d5 100644 --- a/src/pypungi/__init__.py +++ b/src/pypungi/__init__.py @@ -75,16 +75,13 @@ class PungiBase(object): # ARCH setup self.tree_arch = self.config.get('pungi', 'arch') - self.yum_arch = arch_module.tree_arch_to_yum_arch(arch) + self.yum_arch = arch_module.tree_arch_to_yum_arch(self.tree_arch) full_archlist = self.config.getboolean('pungi', 'full_archlist') self.valid_arches = arch_module.get_valid_arches(self.tree_arch, multilib=full_archlist) self.valid_arches.append("src") # throw source in there, filter it later self.valid_native_arches = arch_module.get_valid_arches(self.tree_arch, multilib=False) self.valid_multilib_arches = arch_module.get_valid_multilib_arches(self.tree_arch) - # --nogreedy - self.greedy = config.getboolean('pungi', 'alldeps') - # arch: compatible arches self.compatible_arches = {} for i in self.valid_arches: @@ -131,11 +128,6 @@ class PungiYum(yum.YumBase): self.pungiconfig = config yum.YumBase.__init__(self) - def _checkInstall(self, txmbr): - # overriding this method allows us to ignore installed packages - # and always prefer native packages over those who pull less deps into a transaction - return [] - def doLoggingSetup(self, debuglevel, errorlevel, syslog_ident=None, syslog_facility=None): """Setup the logging facility.""" @@ -155,6 +147,17 @@ class PungiYum(yum.YumBase): # the logging. pass + def _compare_providers(self, *args, **kwargs): + # HACK: always prefer 64bit over 32bit packages + result = yum.YumBase._compare_providers(self, *args, **kwargs) + if len(result) >= 2: + pkg1 = result[0][0] + pkg2 = result[1][0] + if pkg1.name == pkg2.name: + best_arch = self.arch.get_best_arch_from_list([pkg1.arch, pkg2.arch], self.arch.canonarch) + if best_arch != "noarch" and best_arch != pkg1.arch: + result[0:1] = result[0:1:-1] + return result class Pungi(pypungi.PungiBase): def __init__(self, config, ksparser): @@ -188,12 +191,7 @@ class Pungi(pypungi.PungiBase): self.ksparser = ksparser - self.polist = [] - self.srpmpolist = [] - self.debuginfolist = [] - self.srpms_build = [] - self.srpms_fulltree = [] - self.last_po = 0 + self.resolved_deps = {} # list the deps we've already resolved, short circuit. self.excluded_pkgs = {} # list the packages we've already excluded. self.seen_pkgs = {} # list the packages we've already seen so we can check all deps only once @@ -201,28 +199,45 @@ class Pungi(pypungi.PungiBase): self.lookaside_repos = self.config.get('pungi', 'lookaside_repos').split(" ") self.sourcerpm_arch_map = {} # {sourcerpm: set[arches]} - used for gathering debuginfo + # package object lists + self.po_list = set() + self.srpm_po_list = set() + self.debuginfo_po_list = set() + + # get_srpm_po() cache + self.sourcerpm_srpmpo_map = {} + + # already processed packages + self.completed_add_srpms = set() # srpms + self.completed_debuginfo = set() # rpms + self.completed_depsolve = set() # rpms + self.completed_langpacks = set() # rpms + self.completed_multilib = set() # rpms + self.completed_fulltree = set() # srpms + self.completed_selfhosting = set() # srpms + + self.is_fulltree = self.config.getboolean("pungi", "fulltree") + self.is_selfhosting = self.config.getboolean("pungi", "selfhosting") + self.is_sources = not self.config.getboolean("pungi", "nosource") + self.is_debuginfo = not self.config.getboolean("pungi", "nodebuginfo") + self.is_greedy = self.config.getboolean("pungi", "alldeps") + self.is_resolve_deps = True # TODO: implement --nodepsolve + def _add_yum_repo(self, name, url, mirrorlist=False, groups=True, - cost=1000, includepkgs=[], excludepkgs=[], + cost=1000, includepkgs=None, excludepkgs=None, proxy=None): """This function adds a repo to the yum object. - name: Name of the repo - url: Full url to the repo - mirrorlist: Bool for whether or not url is a mirrorlist - groups: Bool for whether or not to use groupdata from this repo - cost: an optional int representing the cost of a repo - includepkgs: An optional list of includes to use - excludepkgs: An optional list of excludes to use - proxy: An optional proxy to use - """ + includepkgs = includepkgs or [] + excludepkgs = excludepkgs or [] self.logger.info('Adding repo %s' % name) thisrepo = yum.yumRepo.YumRepository(name) @@ -288,6 +303,8 @@ class Pungi(pypungi.PungiBase): del self.ayum.prerepoconf self.ayum.repos.setCacheDir(self.ayum.conf.cachedir) + self.ayum.arch.setup_arch(self.yum_arch) + # deal with our repos try: self.ksparser.handler.repo.methodToRepo() @@ -331,8 +348,8 @@ class Pungi(pypungi.PungiBase): raise ValueError("Not a binary package: %s" % po) if msg: self.logger.info(msg) - if po not in self.polist: - self.polist.append(po) + if po not in self.po_list: + self.po_list.add(po) self.ayum.install(po) self.sourcerpm_arch_map.setdefault(po.sourcerpm, set()).add(po.arch) @@ -341,16 +358,16 @@ class Pungi(pypungi.PungiBase): raise ValueError("Not a debuginfog package: %s" % po) if msg: self.logger.info(msg) - if po not in self.debuginfolist: - self.debuginfolist.append(po) + if po not in self.debuginfo_po_list: + self.debuginfo_po_list.add(po) def add_source(self, po, msg=None): if not is_source(po): raise ValueError("Not a source package: %s" % po) if msg: self.logger.info(msg) - if po not in self.srpmpolist: - self.srpmpolist.append(po) + if po not in self.srpm_po_list: + self.srpm_po_list.add(po) def verifyCachePkg(self, po, path): # Stolen from yum """check the package checksum vs the cache @@ -397,21 +414,18 @@ class Pungi(pypungi.PungiBase): return pkg_sack - def getPackageDeps(self, po): + def get_package_deps(self, po): """Add the dependencies for a given package to the transaction info""" - if po in self.seen_pkgs: - return - self.seen_pkgs[po] = None + added = set() + if po in self.completed_depsolve: + return added + self.completed_depsolve.add(po) self.logger.info('Checking deps of %s.%s' % (po.name, po.arch)) reqs = po.requires provs = po.provides - added = set() - - added.update(self.getLangpacks([po])) - added.update(self.getMultilib([po])) for req in reqs: if req in self.resolved_deps: @@ -429,12 +443,12 @@ class Pungi(pypungi.PungiBase): self.logger.warn("Unresolvable dependency %s in %s.%s" % (r, po.name, po.arch)) continue - if self.greedy: + if self.is_greedy: deps = yum.packageSack.ListPackageSack(deps).returnNewestByNameArch() else: found = False for dep in deps: - if dep in self.polist: + if dep in self.po_list: found = True break if found: @@ -454,20 +468,27 @@ class Pungi(pypungi.PungiBase): self.resolved_deps[req] = None for add in added: - self.getPackageDeps(add) + self.get_package_deps(add) + return added - def getLangpacks(self, po_list): - added = [] + def add_langpacks(self, po_list=None): + po_list = po_list or self.po_list + added = set() + + for po in sorted(po_list): + if po in self.completed_langpacks: + continue - for po in po_list: # get all langpacks matching the package name langpacks = [ i for i in self.langpacks if i["name"] == po.name ] if not langpacks: continue + self.completed_langpacks.add(po) + for langpack in langpacks: pattern = langpack["install"] % "*" # replace '%s' with '*' - exactmatched, matched, unmatched = yum.packages.parsePackages(self.pkgs, [pattern], casematch=1, pkgdict=self.pkg_refs.copy()) + exactmatched, matched, unmatched = yum.packages.parsePackages(self.all_pkgs, [pattern], casematch=1, pkgdict=self.pkg_refs.copy()) matches = filter(self._filtersrcdebug, exactmatched + matched) matches = [ i for i in matches if not i.name.endswith("-devel") and not i.name.endswith("-static") and i.name != "man-pages-overrides" ] matches = [ i for i in matches if fnmatch(i.name, pattern) ] @@ -481,23 +502,30 @@ class Pungi(pypungi.PungiBase): match = self.ayum._bestPackageFromList(pkg_sack) msg = 'Added langpack %s.%s for package %s (pattern: %s)' % (match.name, match.arch, po.name, pattern) self.add_package(match, msg) - added.append(match) + self.completed_langpacks.add(match) # assuming langpack doesn't have langpacks + added.add(match) return added - def getMultilib(self, po_list): - added = [] + def add_multilib(self, po_list=None): + po_list = po_list or self.po_list + added = set() if not self.multilib_methods: return added - for po in po_list: + for po in sorted(po_list): + if po in self.completed_multilib: + continue + if po.arch in ("noarch", "src", "nosrc"): continue if po.arch in self.valid_multilib_arches: continue + self.completed_multilib.add(po) + matches = self.ayum.pkgSack.searchNevra(name=po.name, ver=po.version, rel=po.release) matches = [i for i in matches if i.arch in self.valid_multilib_arches] if not matches: @@ -511,7 +539,8 @@ class Pungi(pypungi.PungiBase): continue msg = "Added multilib package %s.%s for package %s.%s (method: %s)" % (match.name, match.arch, po.name, po.arch, method) self.add_package(match, msg) - added.append(match) + self.completed_multilib.add(match) + added.add(match) return added def getPackagesFromGroup(self, group): @@ -546,19 +575,18 @@ class Pungi(pypungi.PungiBase): # of the package objects it would bring in. To be used later if # we match the conditional. for condreq, cond in groupobj.conditional_packages.iteritems(): - pkgs = self.ayum.pkgSack.searchNevra(name=condreq) - if pkgs: - pkgs = self.ayum.bestPackagesFromList(pkgs, arch=self.ayum.compatarch) - if self.ayum.tsInfo.conditionals.has_key(cond): - self.ayum.tsInfo.conditionals[cond].extend(pkgs) - else: - self.ayum.tsInfo.conditionals[cond] = pkgs + matches = self.ayum.pkgSack.searchNevra(name=condreq) + if matches: + if not self.is_greedy: + matches = [self.ayum._bestPackageFromList(matches)] + self.ayum.tsInfo.conditionals.setdefault(cond, []).extend(matches) return packages - def _addDefaultGroups(self, excludeGroups=[]): + def _addDefaultGroups(self, excludeGroups=None): """Cycle through the groups and return at list of the ones that ara default.""" + excludeGroups = excludeGroups or [] # This is mostly stolen from anaconda. groups = map(lambda x: x.groupid, @@ -569,21 +597,7 @@ class Pungi(pypungi.PungiBase): self.logger.debug('Add default groups %s' % groups) return groups - def getPackageObjects(self): - """Cycle through the list of packages, get package object - matches, and resolve deps. - - Returns a list of package objects""" - - final_pkgobjs = {} # The final list of package objects - searchlist = [] # The list of package names/globs to search for - matchdict = {} # A dict of objects to names - excludeGroups = [] #A list of groups for removal defined in the ks file - - # precompute pkgs and pkg_refs to speed things up - self.pkgs = self.ayum.pkgSack.returnPackages() - self.pkg_refs = yum.packages.buildPkgRefDict(self.pkgs, casematch=True) - + def get_langpacks(self): try: self.langpacks = list(self.ayum.comps.langpacks) except AttributeError: @@ -595,6 +609,19 @@ class Pungi(pypungi.PungiBase): self.logger.warning("Could not get langpacks due to missing comps in repodata or --ignoregroups=true option.") self.langpacks = [] + def getPackageObjects(self): + """Cycle through the list of packages and get package object matches.""" + + searchlist = [] # The list of package names/globs to search for + matchdict = {} # A dict of objects to names + excludeGroups = [] # A list of groups for removal defined in the ks file + + # precompute pkgs and pkg_refs to speed things up + self.all_pkgs = self.ayum.pkgSack.returnPackages() + self.pkg_refs = yum.packages.buildPkgRefDict(self.all_pkgs, casematch=True) + + self.get_langpacks() + # First remove the excludes self.ayum.excludePackages() @@ -634,16 +661,16 @@ class Pungi(pypungi.PungiBase): name = name[:-2] multilib = True - if self.greedy and name == "system-release": + if self.is_greedy and name == "system-release": # HACK: handles a special case, when system-release virtual provide is specified in the greedy mode matches = self.ayum.whatProvides(name, None, None).returnPackages() else: - exactmatched, matched, unmatched = yum.packages.parsePackages(self.pkgs, [name], casematch=1, pkgdict=self.pkg_refs.copy()) + exactmatched, matched, unmatched = yum.packages.parsePackages(self.all_pkgs, [name], casematch=1, pkgdict=self.pkg_refs.copy()) matches = exactmatched + matched matches = filter(self._filtersrcdebug, matches) - if multilib and not self.greedy: + if multilib and not self.is_greedy: matches = [ po for po in matches if po.arch in self.valid_multilib_arches ] if not matches: @@ -656,7 +683,7 @@ class Pungi(pypungi.PungiBase): for name, packages in packages_by_name.iteritems(): packages = self.excludePackages(packages) - if self.greedy: + if self.is_greedy: packages = yum.packageSack.ListPackageSack(packages).returnNewestByNameArch() else: packages = [self.ayum._bestPackageFromList(packages)] @@ -665,33 +692,75 @@ class Pungi(pypungi.PungiBase): msg = 'Found %s.%s' % (po.name, po.arch) self.add_package(po, msg) - if len(self.ayum.tsInfo) == 0: - raise yum.Errors.MiscError, 'No packages found to download.' + if not self.po_list: + raise RuntimeError("No packages found") - moretoprocess = True - while moretoprocess: # Our fun loop - moretoprocess = False - for txmbr in self.ayum.tsInfo: - if not final_pkgobjs.has_key(txmbr.po): - final_pkgobjs[txmbr.po] = None # Add the pkg to our final list - self.getPackageDeps(txmbr.po) # Get the deps of our package - moretoprocess = True - - self.polist = final_pkgobjs.keys() self.logger.info('Finished gathering package objects.') - def getSRPMPo(self, po): - """Given a package object, get a package object for the - corresponding source rpm. Requires yum still configured - and a valid package object.""" + def gather(self): + + # get package objects according to the input list + self.getPackageObjects() + self.createSourceHashes() + + pass_num = 0 + added = set() + while 1: + if pass_num > 0 and not added: + break + added = set() + pass_num += 1 + self.logger.info("Pass #%s" % pass_num) + + for txmbr in self.ayum.tsInfo: + if not txmbr.po in self.po_list: + self.add_package(txmbr.po) + + # resolve deps + if self.is_resolve_deps: + for po in sorted(self.po_list): + added.update(self.get_package_deps(po)) + + added_srpms = self.add_srpms() + added.update(added_srpms) + if self.is_selfhosting: + for srpm_po in sorted(added_srpms): + added.update(self.get_package_deps(srpm_po)) + + if self.is_fulltree: + added.update(self.add_fulltree()) + if added: + continue + + # add langpacks + added.update(self.add_langpacks(self.po_list)) + if added: + continue + + # add multilib packages + added.update(self.add_multilib(self.po_list)) + if added: + continue + + def get_srpm_po(self, po): + """Given a package object, get a package object for the corresponding source rpm.""" + + # return srpm_po from cache if available + srpm_po = self.sourcerpm_srpmpo_map.get(po.sourcerpm, None) + if srpm_po is not None: + return srpm_po + + # arch can be "src" or "nosrc" nvr, arch, _ = po.sourcerpm.rsplit(".", 2) name, ver, rel = nvr.rsplit('-', 2) - try: - srpmpo = self.ayum.pkgSack.searchNevra(name=name, ver=ver, rel=rel, arch='src')[0] - return srpmpo - except IndexError: - print >> sys.stderr, "Error: Cannot find a source rpm for %s" % srpm - sys.exit(1) + + # ... but even "nosrc" packages are stored as "src" in repodata + srpm_po_list = self.ayum.pkgSack.searchNevra(name=name, ver=ver, rel=rel, arch="src") + if not srpm_po_list: + raise RuntimeError("Cannot find a source rpm for %s" % po.sourcerpm) + srpm_po = srpm_po_list[0] + self.sourcerpm_srpmpo_map[po.sourcerpm] = srpm_po + return srpm_po def createSourceHashes(self): """Create two dicts - one that maps binary POs to source POs, and @@ -700,127 +769,109 @@ class Pungi(pypungi.PungiBase): self.src_by_bin = {} self.bin_by_src = {} self.logger.info("Generating source <-> binary package mappings") - (dummy1, everything, dummy2) = yum.packages.parsePackages(self.pkgs, ['*'], pkgdict=self.pkg_refs.copy()) - for po in everything: - if po.arch == 'src': + #(dummy1, everything, dummy2) = yum.packages.parsePackages(self.all_pkgs, ['*'], pkgdict=self.pkg_refs.copy()) + for po in self.all_pkgs: + if is_source(po): continue - srpmpo = self.getSRPMPo(po) + srpmpo = self.get_srpm_po(po) self.src_by_bin[po] = srpmpo if self.bin_by_src.has_key(srpmpo): self.bin_by_src[srpmpo].append(po) else: self.bin_by_src[srpmpo] = [po] - def getSRPMList(self): + def add_srpms(self, po_list=None): """Cycle through the list of package objects and find the sourcerpm for them. Requires yum still configured and a list of package objects""" - for po in self.polist[self.last_po:]: - srpm_po = self.src_by_bin[po] - if not srpm_po in self.srpmpolist: - msg = "Adding source package %s.%s" % (srpm_po.name, srpm_po.arch) - self.add_source(srpm_po) - self.last_po = len(self.polist) - def resolvePackageBuildDeps(self): - """Make the package lists self hosting. Requires yum - still configured, a list of package objects, and a - a list of source rpms.""" - deppass = 1 - while 1: - self.logger.info("Resolving build dependencies, pass %d" % (deppass)) - prev = list(self.ayum.tsInfo.getMembers()) - for srpm in self.srpmpolist[len(self.srpms_build):]: - self.getPackageDeps(srpm) - for txmbr in self.ayum.tsInfo: - if txmbr.po.arch != 'src' and txmbr.po not in self.polist: - self.polist.append(txmbr.po) - self.getPackageDeps(txmbr.po) - self.srpms_build = list(self.srpmpolist) - # Now that we've resolved deps, refresh the source rpm list - self.getSRPMList() - deppass = deppass + 1 - if len(prev) == len(self.ayum.tsInfo.getMembers()): - break + srpms = set() + po_list = po_list or self.po_list + for po in sorted(po_list): + srpm_po = self.sourcerpm_srpmpo_map[po.sourcerpm] + if srpm_po in self.completed_add_srpms: + continue + msg = "Adding source package %s.%s" % (srpm_po.name, srpm_po.arch) + self.add_source(srpm_po, msg) + self.completed_add_srpms.add(srpm_po) + srpms.add(srpm_po) + return srpms - def completePackageSet(self): + def add_fulltree(self, srpm_po_list=None): """Cycle through all package objects, and add any that correspond to a source rpm that we are including. Requires yum still configured and a list of package objects.""" - thepass = 1 - while 1: - prevlen = len(self.srpmpolist) - self.logger.info("Completing package set, pass %d" % (thepass,)) - for srpm in self.srpmpolist[len(self.srpms_fulltree):]: - include_native = False - include_multilib = False - has_native = False - has_multilib = False - for po in self.excludePackages(self.bin_by_src[srpm]): - if not is_package(po): - continue - if po.arch == "noarch": - continue - if po not in self.polist: - # process only already included packages - if po.arch in self.valid_multilib_arches: - has_multilib = True - elif po.arch in self.valid_native_arches: - has_native = True - continue + self.logger.info("Completing package set") + + srpm_po_list = srpm_po_list or self.srpm_po_list + srpms = [] + for srpm_po in srpm_po_list: + if srpm_po in self.completed_fulltree: + continue + srpms.append(srpm_po) + self.completed_fulltree.add(srpm_po) + + added = set() + for srpm_po in srpms: + include_native = False + include_multilib = False + has_native = False + has_multilib = False + + for po in self.excludePackages(self.bin_by_src[srpm_po]): + if not is_package(po): + continue + if po.arch == "noarch": + continue + if po not in self.po_list: + # process only already included packages if po.arch in self.valid_multilib_arches: - include_multilib = True + has_multilib = True elif po.arch in self.valid_native_arches: - include_native = True + has_native = True + continue + if po.arch in self.valid_multilib_arches: + include_multilib = True + elif po.arch in self.valid_native_arches: + include_native = True - # XXX: this is very fragile! - # Do not make any changes unless you really know what you're doing! - if not include_native: - # if there's no native package already pulled in... - if has_native and not include_multilib: - # include all native packages, but only if we're not pulling multilib already - # SCENARIO: a noarch package was already pulled in and there are x86_64 and i686 packages -> we want x86_64 in to complete the package set - include_native = True - elif has_multilib: - # SCENARIO: a noarch package was already pulled in and there are no x86_64 packages; we want i686 in to complete the package set - include_multilib = True - - for po in self.excludePackages(self.bin_by_src[srpm]): - if not is_package(po): - continue - if po in self.polist: - continue - if po.arch != "noarch": - if po.arch in self.valid_multilib_arches: - if not include_multilib: - continue - if po.arch in self.valid_native_arches: - if not include_native: - continue - msg = "Adding %s.%s to complete package set" % (po.name, po.arch) - self.add_package(po, msg) - self.getPackageDeps(po) - for txmbr in self.ayum.tsInfo: - if txmbr.po.arch != 'src' and txmbr.po not in self.polist: - self.polist.append(txmbr.po) - self.getPackageDeps(po) - self.srpms_fulltree = list(self.srpmpolist) - # Now that we've resolved deps, refresh the source rpm list - self.getSRPMList() - if len(self.srpmpolist) == prevlen: - self.logger.info("Completion finished in %d passes" % (thepass,)) - break - thepass = thepass + 1 + # XXX: this is very fragile! + # Do not make any changes unless you really know what you're doing! + if not include_native: + # if there's no native package already pulled in... + if has_native and not include_multilib: + # include all native packages, but only if we're not pulling multilib already + # SCENARIO: a noarch package was already pulled in and there are x86_64 and i686 packages -> we want x86_64 in to complete the package set + include_native = True + elif has_multilib: + # SCENARIO: a noarch package was already pulled in and there are no x86_64 packages; we want i686 in to complete the package set + include_multilib = True + for po in self.excludePackages(self.bin_by_src[srpm_po]): + if not is_package(po): + continue + if po in self.po_list: + continue + if po.arch != "noarch": + if po.arch in self.valid_multilib_arches: + if not include_multilib: + continue + if po.arch in self.valid_native_arches: + if not include_native: + continue + msg = "Adding %s.%s to complete package set" % (po.name, po.arch) + self.add_package(po, msg) + return added def getDebuginfoList(self): """Cycle through the list of package objects and find debuginfo rpms for them. Requires yum still configured and a list of package objects""" - for po in self.pkgs: + added = set() + for po in self.all_pkgs: if not is_debug(po): continue @@ -833,6 +884,8 @@ class Pungi(pypungi.PungiBase): continue msg = 'Added debuginfo %s.%s' % (po.name, po.arch) self.add_debuginfo(po, msg) + added.add(po) + return added def _downloadPackageList(self, polist, relpkgdir): """Cycle through the list of package objects and @@ -891,7 +944,7 @@ class Pungi(pypungi.PungiBase): def downloadPackages(self): """Download the package objects obtained in getPackageObjects().""" - self._downloadPackageList(self.polist, + self._downloadPackageList(self.po_list, os.path.join(self.tree_arch, self.config.get('pungi', 'osdir'), self.config.get('pungi', 'product_path'))) @@ -941,30 +994,44 @@ class Pungi(pypungi.PungiBase): find the package objects for them, Then download them.""" # do the downloads - self._downloadPackageList(self.srpmpolist, os.path.join('source', 'SRPMS')) + self._downloadPackageList(self.srpm_po_list, os.path.join('source', 'SRPMS')) def downloadDebuginfo(self): """Cycle through the list of debuginfo rpms and download them.""" # do the downloads - self._downloadPackageList(self.debuginfolist, os.path.join(self.tree_arch, 'debug')) + self._downloadPackageList(self.debuginfo_po_list, os.path.join(self.tree_arch, 'debug')) - def _listPackages(self, polist): + def _list_packages(self, po_list): """Cycle through the list of packages and return their paths.""" - return [ os.path.join(pkg.basepath or "", pkg.relativepath) for pkg in polist if pkg.repoid not in self.lookaside_repos ] + result = [ os.path.join(po.basepath or "", po.relativepath) for po in po_list if po.repoid not in self.lookaside_repos ] + result.sort() + return result - def listPackages(self): + def list_packages(self): """Cycle through the list of RPMs and return their paths.""" - return self._listPackages(self.polist) + return self._list_packages(self.po_list) - def listSRPMs(self): + def list_srpms(self): """Cycle through the list of SRPMs and return their paths.""" - return self._listPackages(self.srpmpolist) + return self._list_packages(self.srpm_po_list) - def listDebuginfo(self): + def list_debuginfo(self): """Cycle through the list of DEBUGINFO RPMs and return their paths.""" - return self._listPackages(self.debuginfolist) + return self._list_packages(self.debuginfo_po_list) + + def _size_packages(self, po_list): + return sum([ po.size for po in po_list if po.repoid not in self.lookaside_repos ]) + + def size_packages(self): + return self._size_packages(self.po_list) + + def size_srpms(self): + return self._size_packages(self.srpm_po_list) + + def size_debuginfo(self): + return self._size_packages(self.debuginfo_po_list) def writeinfo(self, line): """Append a line to the infofile in self.infofile""" @@ -996,7 +1063,7 @@ class Pungi(pypungi.PungiBase): conf.directory = path conf.database = True if comps: - conf.groupfile = comps + conf.groupfile = comps if basedir: conf.basedir = basedir if baseurl: @@ -1241,7 +1308,7 @@ class Pungi(pypungi.PungiBase): self.logger.info("ARCH: arm, not doing doCreateIsos().") return - isolist=[] + isolist = [] ppcbootinfo = '/usr/share/lorax/config_files/ppc' pypungi.util._ensuredir(self.isodir, self.logger,