diff --git a/pungi/checks.py b/pungi/checks.py index 291cf234..08426bdd 100644 --- a/pungi/checks.py +++ b/pungi/checks.py @@ -703,7 +703,6 @@ def make_schema(): ), "repoclosure_backend": { "type": "string", - # Gather and repoclosure both have the same backends: yum + dnf "default": _get_default_gather_backend(), "enum": _get_gather_backends(), }, @@ -1620,10 +1619,8 @@ def update_schema(schema, update_dict): def _get_gather_backends(): - if six.PY2: - return ["yum", "dnf"] return ["dnf"] def _get_default_gather_backend(): - return "yum" if six.PY2 else "dnf" + return "dnf" diff --git a/pungi/compose.py b/pungi/compose.py index 97d7705b..29b7881b 100644 --- a/pungi/compose.py +++ b/pungi/compose.py @@ -466,13 +466,10 @@ class Compose(kobo.log.LoggingBase): @property def should_create_yum_database(self): - """Explicit configuration trumps all. Otherwise check gather backend - and only create it for Yum. + """Explicit configuration trumps all. Yum is no longer supported, so + default to False. """ - config = self.conf.get("createrepo_database") - if config is not None: - return config - return self.conf["gather_backend"] == "yum" + return self.conf.get("createrepo_database", False) def read_variants(self): # TODO: move to phases/init ? diff --git a/pungi/config.py b/pungi/config.py deleted file mode 100644 index daaf2a6c..00000000 --- a/pungi/config.py +++ /dev/null @@ -1,78 +0,0 @@ -# -*- coding: utf-8 -*- - - -# This program is free software; you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation; version 2 of the License. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU Library General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program; if not, see . - - -import os -import sys -import time - -from ConfigParser import SafeConfigParser - -from .arch_utils import getBaseArch - -# In development, `here` will point to the bin/ directory with scripts. -here = sys.path[0] -MULTILIBCONF = ( - os.path.join(os.path.dirname(__file__), "..", "share", "multilib") - if here != "/usr/bin" - else "/usr/share/pungi/multilib" -) - - -class Config(SafeConfigParser): - def __init__(self, pungirc=None): - SafeConfigParser.__init__(self) - - self.add_section("pungi") - self.add_section("lorax") - - self.set("pungi", "osdir", "os") - self.set("pungi", "sourcedir", "source") - self.set("pungi", "debugdir", "debug") - self.set("pungi", "isodir", "iso") - self.set("pungi", "multilibconf", MULTILIBCONF) - self.set( - "pungi", "relnotefilere", "LICENSE README-BURNING-ISOS-en_US.txt ^RPM-GPG" - ) - self.set("pungi", "relnotedirre", "") - self.set( - "pungi", "relnotepkgs", "fedora-repos fedora-release fedora-release-notes" - ) - self.set("pungi", "product_path", "Packages") - self.set("pungi", "cachedir", "/var/cache/pungi") - self.set("pungi", "compress_type", "xz") - self.set("pungi", "arch", getBaseArch()) - self.set("pungi", "family", "Fedora") - self.set("pungi", "iso_basename", "Fedora") - self.set("pungi", "version", time.strftime("%Y%m%d", time.localtime())) - self.set("pungi", "variant", "") - self.set("pungi", "destdir", os.getcwd()) - self.set("pungi", "workdirbase", "/work") - self.set("pungi", "bugurl", "https://bugzilla.redhat.com") - self.set("pungi", "debuginfo", "True") - self.set("pungi", "alldeps", "True") - self.set("pungi", "isfinal", "False") - self.set("pungi", "nohash", "False") - self.set("pungi", "full_archlist", "False") - self.set("pungi", "multilib", "") - self.set("pungi", "lookaside_repos", "") - self.set("pungi", "resolve_deps", "True") - self.set("pungi", "no_dvd", "False") - self.set("pungi", "nomacboot", "False") - self.set("pungi", "rootfs_size", "False") - - # if missing, self.read() is a noop, else change 'defaults' - if pungirc: - self.read(os.path.expanduser(pungirc)) diff --git a/pungi/gather.py b/pungi/gather.py deleted file mode 100644 index 1493323e..00000000 --- a/pungi/gather.py +++ /dev/null @@ -1,1585 +0,0 @@ -# -*- coding: utf-8 -*- - - -# This program is free software; you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation; version 2 of the License. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU Library General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program; if not, see . - - -import logging -import os -import sys -from fnmatch import fnmatch - -import lockfile -import urlgrabber.progress -import yum - -import arch as arch_module -import multilib_yum as multilib -import pungi.util - - -class ReentrantYumLock(object): - """A lock that can be acquired multiple times by the same process.""" - - def __init__(self, lock, log): - self.lock = lock - self.log = log - self.count = 0 - - def __enter__(self): - if not self.count: - self.log.info("Waiting on %r" % self.lock.lock_file) - self.lock.acquire() - self.log.info("Got %r" % self.lock.lock_file) - self.count = self.count + 1 - self.log.info("Lock count upped to %i" % self.count) - - def __exit__(self, type, value, tb): - self.count = self.count - 1 - self.log.info("Lock count downed to %i" % self.count) - self.log.info("%r %r %r" % (type, value, tb)) - if not self.count: - self.lock.release() - self.log.info("Released %r" % self.lock.lock_file) - - -def yumlocked(method): - """A locking decorator.""" - - def wrapper(self, *args, **kwargs): - with self.yumlock: - return method(self, *args, **kwargs) - - # TODO - replace argspec, signature, etc.. - return wrapper - - -def is_source(po): - if po.arch in ("src", "nosrc"): - return True - return False - - -def is_noarch(po): - if po.arch == "noarch": - return True - return False - - -def is_package(po): - if pungi.util.pkg_is_debug(po): - return False - if is_source(po): - return False - return True - - -FLAGS = { - "EQ": "=", - "GE": ">=", - "LE": "<=", - "GT": ">", - "LT": "<", -} - - -class Req(object): - """A wrapper for a tuple representing a Requires tag. - - Only useful for formatting the value into a human readable string. - """ - - def __init__(self, req): - self.r, self.f, self.v = req - - def __str__(self): - if self.f and self.v: - flag = FLAGS.get(self.f, "??") - version = "%s:%s-%s" % self.v - return "%s %s %s" % (self.r, flag, version) - return self.r - - -class PungiBase(object): - """The base Pungi class. Set up config items and logging here""" - - def __init__(self, config): - self.config = config - multilib.init(self.config.get("pungi", "multilibconf")) - - # ARCH setup - self.tree_arch = self.config.get("pungi", "arch") - self.yum_arch = arch_module.tree_arch_to_yum_arch(self.tree_arch) - full_archlist = self.config.getboolean("pungi", "full_archlist") - self.valid_arches = arch_module.get_valid_arches( - self.tree_arch, multilib=full_archlist - ) - self.valid_arches.append("src") # throw source in there, filter it later - self.valid_native_arches = arch_module.get_valid_arches( - self.tree_arch, multilib=False - ) - self.valid_multilib_arches = arch_module.get_valid_multilib_arches( - self.tree_arch - ) - - # arch: compatible arches - self.compatible_arches = {} - for i in self.valid_arches: - self.compatible_arches[i] = arch_module.get_compatible_arches(i) - - self.doLoggerSetup() - self.workdir = os.path.join( - self.config.get("pungi", "workdirbase"), - self.config.get("pungi", "variant"), - self.tree_arch, - ) - - def doLoggerSetup(self): - """Setup our logger""" - - logdir = os.path.join(self.config.get("pungi", "destdir"), "logs") - - pungi.util._ensuredir( - logdir, None, force=True - ) # Always allow logs to be written out - - if self.config.get("pungi", "variant"): - logfile = os.path.join( - logdir, - "%s.%s.log" % (self.config.get("pungi", "variant"), self.tree_arch), - ) - else: - logfile = os.path.join(logdir, "%s.log" % (self.tree_arch)) - - # Create the root logger, that will log to our file - logging.basicConfig( - level=logging.DEBUG, - format="%(name)s.%(levelname)s: %(message)s", - filename=logfile, - ) - - -class CallBack(urlgrabber.progress.TextMeter): - """A call back function used with yum.""" - - def __init__(self, logger): - self.logger = logger - - def start( - self, filename=None, url=None, basename=None, size=None, now=None, text=None - ): - self.logger.info( - "Downloading %s (%sB)" % (text, urlgrabber.progress.format_number(size)) - ) - - def update(self, amount_read, name=None): - return - - def end(self, amount_read, now=None): - return - - -class PungiYum(yum.YumBase): - """Subclass of Yum""" - - def __init__(self, config): - self.pungiconfig = config - yum.YumBase.__init__(self) - - def doLoggingSetup( - self, debuglevel, errorlevel, syslog_ident=None, syslog_facility=None - ): - """Setup the logging facility.""" - - logdir = os.path.join(self.pungiconfig.get("pungi", "destdir"), "logs") - if not os.path.exists(logdir): - os.makedirs(logdir) - if self.pungiconfig.get("pungi", "variant"): - logfile = os.path.join( - logdir, - "%s.%s.log" - % ( - self.pungiconfig.get("pungi", "variant"), - self.pungiconfig.get("pungi", "arch"), - ), - ) - else: - logfile = os.path.join( - logdir, "%s.log" % (self.pungiconfig.get("pungi", "arch")) - ) - - yum.logging.basicConfig(level=yum.logging.DEBUG, filename=logfile) - - def doFileLogSetup(self, uid, logfile): - # This function overrides a yum function, allowing pungi to control - # the logging. - pass - - def _compare_providers(self, *args, **kwargs): - # HACK: always prefer 64bit over 32bit packages - result = yum.YumBase._compare_providers(self, *args, **kwargs) - if len(result) >= 2: - pkg1 = result[0][0] - pkg2 = result[1][0] - if pkg1.name == pkg2.name: - best_arch = self.arch.get_best_arch_from_list( - [pkg1.arch, pkg2.arch], self.arch.canonarch - ) - if best_arch != "noarch" and best_arch != pkg1.arch: - result[0:1] = result[0:1:-1] - return result - - -class Pungi(PungiBase): - def __init__(self, config, ksparser): - PungiBase.__init__(self, config) - - # Set our own logging name space - self.logger = logging.getLogger("Pungi") - - # Create a lock object for later use. - filename = self.config.get("pungi", "cachedir") + "/yumlock" - lock = lockfile.LockFile(filename) - self.yumlock = ReentrantYumLock(lock, self.logger) - - if not self.logger.handlers: - # Create the stdout/err streams and only send INFO+ stuff there - formatter = logging.Formatter("%(name)s:%(levelname)s: %(message)s") - console = logging.StreamHandler() - console.setFormatter(formatter) - console.setLevel(logging.INFO) - self.logger.addHandler(console) - - self.destdir = self.config.get("pungi", "destdir") - self.archdir = os.path.join( - self.destdir, - self.config.get("pungi", "version"), - self.config.get("pungi", "variant"), - self.tree_arch, - ) - - self.topdir = os.path.join(self.archdir, "os") - self.isodir = os.path.join(self.archdir, self.config.get("pungi", "isodir")) - - pungi.util._ensuredir(self.workdir, self.logger, force=True) - - self.common_files = [] - self.infofile = os.path.join( - self.config.get("pungi", "destdir"), - self.config.get("pungi", "version"), - ".composeinfo", - ) - - self.ksparser = ksparser - - self.resolved_deps = {} # list the deps we've already resolved, short circuit - self.excluded_packages = set() # set of packages we've already excluded - self.multilib_blacklist = ( - set() - ) # set of packages we've already excluded through a multilib blacklist - self.seen_pkgs = ( - {} - ) # list the packages we've already seen so we can check all deps only once - self.multilib_methods = self.config.get("pungi", "multilib").split(" ") - - # greedy methods: - # * none: only best match package - # * all: all packages matching a provide - # * build: best match package + all other packages from - # the same SRPM having the same provide - self.greedy_method = self.config.get("pungi", "greedy") - - self.lookaside_repos = self.config.get("pungi", "lookaside_repos").split(" ") - self.sourcerpm_arch_map = ( - {} - ) # {sourcerpm: set[arches]} - used for gathering debuginfo - - # package object lists - self.po_list = set() - self.srpm_po_list = set() - self.debuginfo_po_list = set() - - # get_srpm_po() cache - self.sourcerpm_srpmpo_map = {} - - # flags - # packages specified in %packages kickstart section including - # those defined via comps groups - self.input_packages = set() - # packages specified in %packages kickstart section - # *indirectly* via comps groups - self.comps_packages = set() - self.prepopulate_packages = ( - set() - ) # packages specified in %prepopulate kickstart section - self.fulltree_packages = set() - self.langpack_packages = set() - self.multilib_packages = set() - - # already processed packages - self.completed_add_srpms = set() # srpms - self.completed_debuginfo = set() # rpms - self.completed_depsolve = set() # rpms - self.completed_langpacks = set() # rpms - self.completed_multilib = set() # rpms - self.completed_fulltree = set() # srpms - self.completed_selfhosting = set() # srpms - self.completed_greedy_build = set() # po.sourcerpm - - self.is_fulltree = self.config.getboolean("pungi", "fulltree") - self.is_selfhosting = self.config.getboolean("pungi", "selfhosting") - self.is_sources = not self.config.getboolean("pungi", "nosource") - self.is_debuginfo = not self.config.getboolean("pungi", "nodebuginfo") - self.is_resolve_deps = self.config.getboolean("pungi", "resolve_deps") - - self.fulltree_excludes = set(self.ksparser.handler.fulltree_excludes) - - # rootfs image size - self.rootfs_size = self.config.get("pungi", "rootfs_size") - - def _add_yum_repo( - self, - name, - url, - mirrorlist=False, - groups=True, - cost=1000, - includepkgs=None, - excludepkgs=None, - proxy=None, - ): - """This function adds a repo to the yum object. - name: Name of the repo - url: Full url to the repo - mirrorlist: Bool for whether or not url is a mirrorlist - groups: Bool for whether or not to use groupdata from this repo - cost: an optional int representing the cost of a repo - includepkgs: An optional list of includes to use - excludepkgs: An optional list of excludes to use - proxy: An optional proxy to use - """ - includepkgs = includepkgs or [] - excludepkgs = excludepkgs or [] - - self.logger.info("Adding repo %s" % name) - thisrepo = yum.yumRepo.YumRepository(name) - thisrepo.name = name - # add excludes and such here when pykickstart gets them - if mirrorlist: - thisrepo.mirrorlist = yum.parser.varReplace(url, self.ayum.conf.yumvar) - self.mirrorlists.append(thisrepo.mirrorlist) - self.logger.info( - "Mirrorlist for repo %s is %s" % (thisrepo.name, thisrepo.mirrorlist) - ) - else: - thisrepo.baseurl = yum.parser.varReplace(url, self.ayum.conf.yumvar) - self.repos.extend(thisrepo.baseurl) - self.logger.info( - "URL for repo %s is %s" % (thisrepo.name, thisrepo.baseurl) - ) - thisrepo.basecachedir = self.ayum.conf.cachedir - thisrepo.enablegroups = groups - # This is until yum uses this failover by default - thisrepo.failovermethod = "priority" - thisrepo.exclude = excludepkgs - thisrepo.includepkgs = includepkgs - thisrepo.cost = cost - # Yum doesn't like proxy being None - if proxy: - thisrepo.proxy = proxy - self.ayum.repos.add(thisrepo) - self.ayum.repos.enableRepo(thisrepo.id) - self.ayum._getRepos(thisrepo=thisrepo.id, doSetup=True) - # Set the repo callback. - self.ayum.repos.setProgressBar(CallBack(logger=self.logger)) - self.ayum.repos.callback = CallBack(logger=self.logger) - thisrepo.metadata_expire = 0 - thisrepo.mirrorlist_expire = 0 - if os.path.exists(os.path.join(thisrepo.cachedir, "repomd.xml")): - os.remove(os.path.join(thisrepo.cachedir, "repomd.xml")) - - @yumlocked - def _inityum(self): - """Initialize the yum object. Only needed for certain actions.""" - - # Create a yum object to use - self.repos = [] - self.mirrorlists = [] - self.ayum = PungiYum(self.config) - self.ayum.doLoggingSetup(6, 6) - yumconf = yum.config.YumConf() - yumconf.debuglevel = 6 - yumconf.errorlevel = 6 - yumconf.cachedir = self.config.get("pungi", "cachedir") - yumconf.persistdir = ( - "/var/lib/yum" # keep at default, gets appended to installroot - ) - yumconf.installroot = os.path.join(self.workdir, "yumroot") - yumconf.uid = os.geteuid() - yumconf.cache = 0 - yumconf.failovermethod = "priority" - yumconf.deltarpm = 0 - yumvars = yum.config._getEnvVar() - yumvars["releasever"] = self.config.get("pungi", "version") - yumvars["basearch"] = yum.rpmUtils.arch.getBaseArch(myarch=self.tree_arch) - yumconf.yumvar = yumvars - self.ayum._conf = yumconf - # I have no idea why this fixes a traceback, but James says it does. - del self.ayum.prerepoconf - self.ayum.repos.setCacheDir(self.ayum.conf.cachedir) - - self.ayum.arch.setup_arch(self.yum_arch) - - # deal with our repos - try: - self.ksparser.handler.repo.methodToRepo() - except Exception: - pass - - for repo in self.ksparser.handler.repo.repoList: - if repo.mirrorlist: - # The not bool() thing is because pykickstart is yes/no on - # whether to ignore groups, but yum is a yes/no on whether to - # include groups. Awkward. - self._add_yum_repo( - repo.name, - repo.mirrorlist, - mirrorlist=True, - groups=not bool(repo.ignoregroups), - cost=repo.cost, - includepkgs=repo.includepkgs, - excludepkgs=repo.excludepkgs, - proxy=repo.proxy, - ) - else: - self._add_yum_repo( - repo.name, - repo.baseurl, - mirrorlist=False, - groups=not bool(repo.ignoregroups), - cost=repo.cost, - includepkgs=repo.includepkgs, - excludepkgs=repo.excludepkgs, - proxy=repo.proxy, - ) - - self.logger.info("Getting sacks for arches %s" % self.valid_arches) - self.ayum._getSacks(archlist=self.valid_arches) - - def _filtersrcdebug(self, po): - """Filter out package objects that are of 'src' arch.""" - - if po.arch == "src" or pungi.util.pkg_is_debug(po): - return False - - return True - - def add_package(self, po, msg=None): - if not is_package(po): - raise ValueError("Not a binary package: %s" % po) - if msg: - self.logger.info(msg) - if po not in self.po_list: - self.po_list.add(po) - self.ayum.install(po) - self.sourcerpm_arch_map.setdefault(po.sourcerpm, set()).add(po.arch) - - def add_debuginfo(self, po, msg=None): - if not pungi.util.pkg_is_debug(po): - raise ValueError("Not a debuginfog package: %s" % po) - if msg: - self.logger.info(msg) - if po not in self.debuginfo_po_list: - self.debuginfo_po_list.add(po) - - def add_source(self, po, msg=None): - if not is_source(po): - raise ValueError("Not a source package: %s" % po) - if msg: - self.logger.info(msg) - if po not in self.srpm_po_list: - self.srpm_po_list.add(po) - - def verifyCachePkg(self, po, path): # Stolen from yum - """check the package checksum vs the cache - return True if pkg is good, False if not""" - - (csum_type, csum) = po.returnIdSum() - - try: - filesum = yum.misc.checksum(csum_type, path) - except yum.Errors.MiscError: - return False - - if filesum != csum: - return False - - return True - - def expand_multilib_blacklist(self): - multilib_blacklist = self.ksparser.handler.multilib_blacklist - exactmatched, matched, unmatched = yum.packages.parsePackages( - self.all_pkgs, multilib_blacklist, casematch=1, pkgdict=self.pkg_refs.copy() - ) - - for i in sorted(unmatched): - self.logger.warning("Unmatched multilib blacklist pattern: %s" % i) - - for pkg in exactmatched + matched: - if pkg.arch == "src": - continue - if pkg.arch not in self.valid_multilib_arches: - continue - - found = None - for pattern in multilib_blacklist: - if fnmatch(pkg.name, pattern): - found = pattern - break - - if found: - if pkg not in self.multilib_blacklist: - self.logger.info( - "Excluding %s.%s (multilib-blacklist pattern: %s)" - % (pkg.name, pkg.arch, found) - ) - self.multilib_blacklist.add(pkg) - - def expand_excluded_list(self): - excluded_list = [] - multilib_excluded_list = [] - source_excluded_list = [] - - for pattern in self.ksparser.handler.packages.excludedList: - if pattern.endswith(".+"): - multilib_excluded_list.append(pattern[:-2]) - elif pattern.endswith(".src"): - source_excluded_list.append(pattern[:-4]) - else: - excluded_list.append(pattern) - - # native packages - exactmatched, matched, unmatched = yum.packages.parsePackages( - self.all_pkgs, excluded_list, casematch=1, pkgdict=self.pkg_refs.copy() - ) - - for i in sorted(unmatched): - self.logger.warning("Unmatched exclude: %s" % i) - - for pkg in exactmatched + matched: - if pkg.arch == "src": - continue - if pkg.repoid in self.lookaside_repos: - # Don't exclude packages from lookaside - continue - - found = None - for pattern in excluded_list: - if fnmatch(pkg.name, pattern): - found = pattern - break - - if found: - if pkg not in self.excluded_packages: - self.logger.info( - "Excluding %s.%s (pattern: %s)" % (pkg.name, pkg.arch, found) - ) - self.excluded_packages.add(pkg) - - # multilib packages - exactmatched, matched, unmatched = yum.packages.parsePackages( - self.all_pkgs, - multilib_excluded_list, - casematch=1, - pkgdict=self.pkg_refs.copy(), - ) - - for i in sorted(unmatched): - self.logger.warning("Unmatched multilib exclude: %s.+" % i) - - for pkg in exactmatched + matched: - if pkg.arch == "src": - continue - if pkg.arch not in self.valid_multilib_arches: - continue - if pkg.repoid in self.lookaside_repos: - # Don't exclude packages from lookaside - continue - - found = None - for pattern in multilib_excluded_list: - if fnmatch(pkg.name, pattern): - found = pattern - break - - if found: - if pkg not in self.excluded_packages: - self.logger.info( - "Excluding %s.%s (pattern: %s.+)" % (pkg.name, pkg.arch, found) - ) - self.excluded_packages.add(pkg) - - # source packages - exactmatched, matched, unmatched = yum.packages.parsePackages( - self.all_pkgs, - source_excluded_list, - casematch=1, - pkgdict=self.pkg_refs.copy(), - ) - - for i in sorted(unmatched): - self.logger.warning("Unmatched source exclude: %s.src" % i) - - for pkg in exactmatched + matched: - if pkg.arch != "src": - continue - - found = None - for pattern in source_excluded_list: - if fnmatch(pkg.name, pattern): - found = pattern - break - - if found: - if pkg not in self.excluded_packages: - self.logger.info( - "Excluding %s.%s (pattern: %s.src)" - % (pkg.name, pkg.arch, found) - ) - self.excluded_packages.add(pkg) - - def excludePackages(self, pkg_sack): - """exclude packages according to config file""" - if not pkg_sack: - return pkg_sack - - result = [] - for pkg in pkg_sack: - if pkg in self.multilib_blacklist: - continue - if pkg in self.excluded_packages: - continue - result.append(pkg) - - return result - - def get_package_deps(self, po): - """Add the dependencies for a given package to the - transaction info""" - added = set() - if po.repoid in self.lookaside_repos: - # Don't resolve deps for stuff in lookaside. - return added - if po in self.completed_depsolve: - return added - self.completed_depsolve.add(po) - - self.logger.info("Checking deps of %s.%s" % (po.name, po.arch)) - - reqs = po.requires - provs = po.provides - - for req in reqs: - if req in self.resolved_deps: - continue - r, f, v = req - if r.startswith("rpmlib(") or r.startswith("config("): - continue - if req in provs: - continue - - try: - deps = self.ayum.whatProvides(r, f, v).returnPackages() - deps = self.excludePackages(deps) - if not deps: - self.logger.warning( - "Unresolvable dependency %s in %s.%s", - Req(req), - po.name, - po.arch, - ) - continue - - if self.greedy_method == "all": - deps = yum.packageSack.ListPackageSack( - deps - ).returnNewestByNameArch() - else: - found = False - for dep in deps: - if dep in self.po_list: - # HACK: there can be builds in the input list on - # which we want to apply the "build" greedy rules - if ( - self.greedy_method == "build" - and dep.sourcerpm not in self.completed_greedy_build - ): - break - found = True - break - if found: - deps = [] - else: - all_deps = deps - deps = [self.ayum._bestPackageFromList(all_deps)] - if self.greedy_method == "build": - # handle "build" greedy method - if deps: - build_po = deps[0] - if is_package(build_po): - if ( - build_po.arch != "noarch" - and build_po.arch - not in self.valid_multilib_arches - ): - all_deps = [ - i - for i in all_deps - if i.arch not in self.valid_multilib_arches - ] - for dep in all_deps: - if ( - dep != build_po - and dep.sourcerpm == build_po.sourcerpm - ): - deps.append(dep) - self.completed_greedy_build.add( - dep.sourcerpm - ) - - for dep in deps: - if dep not in added: - msg = "Added %s.%s (repo: %s) for %s.%s (Requires: %s)" % ( - dep.name, - dep.arch, - dep.repoid, - po.name, - po.arch, - Req(req), - ) - self.add_package(dep, msg) - added.add(dep) - - except (yum.Errors.InstallError, yum.Errors.YumBaseError): - self.logger.warning( - "Unresolvable dependency %s in %s.%s (repo: %s)", - r, - po.name, - po.arch, - po.repoid, - ) - continue - self.resolved_deps[req] = None - - for add in sorted(added): - self.get_package_deps(add) - return added - - def add_langpacks(self, po_list=None): - po_list = po_list or self.po_list - added = set() - - for po in sorted(po_list): - if po in self.completed_langpacks: - continue - - # get all langpacks matching the package name - langpacks = [i for i in self.langpacks if i["name"] == po.name] - if not langpacks: - continue - - self.completed_langpacks.add(po) - - for langpack in langpacks: - pattern = langpack["install"] % "*" # replace '%s' with '*' - exactmatched, matched, unmatched = yum.packages.parsePackages( - self.all_pkgs, [pattern], casematch=1, pkgdict=self.pkg_refs.copy() - ) - matches = filter(self._filtersrcdebug, exactmatched + matched) - matches = [ - i - for i in matches - if not i.name.endswith("-devel") - and not i.name.endswith("-static") - and i.name != "man-pages-overrides" - ] - matches = [i for i in matches if fnmatch(i.name, pattern)] - - packages_by_name = {} - for i in matches: - packages_by_name.setdefault(i.name, []).append(i) - - for i, pkg_sack in packages_by_name.iteritems(): - pkg_sack = self.excludePackages(pkg_sack) - if not pkg_sack: - continue - match = self.ayum._bestPackageFromList(pkg_sack) - msg = ( - "Added langpack %s.%s (repo: %s) for package %s (pattern: %s)" - % (match.name, match.arch, match.repoid, po.name, pattern) - ) - self.add_package(match, msg) - self.completed_langpacks.add( - match - ) # assuming langpack doesn't have langpacks - added.add(match) - - return added - - def add_multilib(self, po_list=None): - po_list = po_list or self.po_list - added = set() - - if not self.multilib_methods: - return added - - for po in sorted(po_list): - if po in self.completed_multilib: - continue - - if po.arch in ("noarch", "src", "nosrc"): - continue - - if po.arch in self.valid_multilib_arches: - continue - - self.completed_multilib.add(po) - - matches = self.ayum.pkgSack.searchNevra( - name=po.name, ver=po.version, rel=po.release - ) - matches = [i for i in matches if i.arch in self.valid_multilib_arches] - if not matches: - continue - matches = self.excludePackages(matches) - match = self.ayum._bestPackageFromList(matches) - if not match: - continue - - found = False - for pattern in self.ksparser.handler.multilib_whitelist: - if fnmatch(po.name, pattern): - found = True - break - if found: - msg = ( - "Added multilib package %s.%s (repo: %s) for " - "package %s.%s (method: %s)" - % ( - match.name, - match.arch, - match.repoid, - po.name, - po.arch, - "multilib-whitelist", - ) - ) - self.add_package(match, msg) - self.completed_multilib.add(match) - added.add(match) - continue - - method = multilib.po_is_multilib(po, self.multilib_methods) - if not method: - continue - msg = ( - "Added multilib package %s.%s (repo: %s) for package %s.%s (method: %s)" - % (match.name, match.arch, match.repoid, po.name, po.arch, method) - ) - self.add_package(match, msg) - self.completed_multilib.add(match) - added.add(match) - return added - - def getPackagesFromGroup(self, group): - """Get a list of package names from a ksparser group object - - Returns a list of package names""" - - packages = [] - - # Check if we have the group - if not self.ayum.comps.has_group(group.name): - self.logger.error("Group %s not found in comps!" % group) - return packages - - # Get the group object to work with - groupobj = self.ayum.comps.return_group(group.name) - - # Add the mandatory packages - packages.extend(groupobj.mandatory_packages.keys()) - - # Add the default packages unless we don't want them - if group.include == 1: - packages.extend(groupobj.default_packages.keys()) - - # Add the optional packages if we want them - if group.include == 2: - packages.extend(groupobj.default_packages.keys()) - packages.extend(groupobj.optional_packages.keys()) - - # Deal with conditional packages - # Populate a dict with the name of the required package and value - # of the package objects it would bring in. To be used later if - # we match the conditional. - for condreq, cond in groupobj.conditional_packages.iteritems(): - matches = self.ayum.pkgSack.searchNevra(name=condreq) - if matches: - if self.greedy_method != "all": - # works for both "none" and "build" greedy methods - matches = [self.ayum._bestPackageFromList(matches)] - self.ayum.tsInfo.conditionals.setdefault(cond, []).extend(matches) - - return packages - - def _addDefaultGroups(self, excludeGroups=None): - """Cycle through the groups and return at list of the ones that ara - default.""" - excludeGroups = excludeGroups or [] - - # This is mostly stolen from anaconda. - groups = map( - lambda x: x.groupid, filter(lambda x: x.default, self.ayum.comps.groups) - ) - - groups = [x for x in groups if x not in excludeGroups] - - self.logger.debug("Add default groups %s" % groups) - return groups - - def get_langpacks(self): - try: - self.langpacks = list(self.ayum.comps.langpacks) - except AttributeError: - # old yum - self.logger.warning( - "Could not get langpacks via yum.comps. You may need to update yum." - ) - self.langpacks = [] - except yum.Errors.GroupsError: - # no groups or no comps at all - self.logger.warning( - "Could not get langpacks due to missing comps in repodata " - "or --ignoregroups=true option." - ) - self.langpacks = [] - - def getPackageObjects(self): - """Cycle through the list of packages and get package object matches.""" - - searchlist = [] # The list of package names/globs to search for - excludeGroups = [] # A list of groups for removal defined in the ks file - - # precompute pkgs and pkg_refs to speed things up - self.all_pkgs = list(set(self.ayum.pkgSack.returnPackages())) - self.pkg_refs = yum.packages.buildPkgRefDict(self.all_pkgs, casematch=True) - self.expand_excluded_list() - self.expand_multilib_blacklist() - self.all_pkgs = self.excludePackages(self.all_pkgs) - - lookaside_nvrs = set() - for po in self.all_pkgs: - if po.repoid in self.lookaside_repos: - lookaside_nvrs.add(po.nvra) - all_pkgs = [] # building a new list is cheaper than deleting from existing - for po in sorted(self.all_pkgs): - if po.repoid not in self.lookaside_repos and po.nvra in lookaside_nvrs: - self.logger.info( - "Removed %s (repo: %s), because it's also in a lookaside repo" - % (po, po.repoid) - ) - self.excluded_packages.add(po) - else: - all_pkgs.append(po) - self.all_pkgs = all_pkgs - - self.get_langpacks() - - # First remove the excludes - self.ayum.excludePackages() - - # Get the groups set for removal - for group in self.ksparser.handler.packages.excludedGroupList: - excludeGroups.append(str(group)[1:]) - - if "core" in [i.groupid for i in self.ayum.comps.groups]: - if "core" not in [i.name for i in self.ksparser.handler.packages.groupList]: - self.logger.warning( - "The @core group is no longer added by default; Please add " - "@core to the kickstart if you want it in." - ) - - if "base" in [i.groupid for i in self.ayum.comps.groups]: - if "base" not in [i.name for i in self.ksparser.handler.packages.groupList]: - if self.ksparser.handler.packages.addBase: - self.logger.warning( - "The --nobase kickstart option is no longer supported; " - "Please add @base to the kickstart if you want it in." - ) - - # Check to see if we want all the defaults - if self.ksparser.handler.packages.default: - for group in self._addDefaultGroups(excludeGroups): - self.ksparser.handler.packages.add(["@%s" % group]) - - # Get a list of packages from groups - comps_package_names = set() - for group in self.ksparser.handler.packages.groupList: - comps_package_names.update(self.getPackagesFromGroup(group)) - searchlist.extend(sorted(comps_package_names)) - - # Add packages - searchlist.extend(self.ksparser.handler.packages.packageList) - input_packages = searchlist[:] - - # Add prepopulate packages - prepopulate_packages = self.ksparser.handler.prepopulate - searchlist.extend(prepopulate_packages) - - # Make the search list unique - searchlist = yum.misc.unique(searchlist) - - for name in searchlist: - pattern = name - multilib = False - orig_name = name - if name.endswith(".+"): - name = name[:-2] - multilib = True - - if self.greedy_method == "all" and name == "system-release": - # HACK: handles a special case, when system-release virtual - # provide is specified in the greedy mode - matches = self.ayum.whatProvides(name, None, None).returnPackages() - else: - exactmatched, matched, unmatched = yum.packages.parsePackages( - self.all_pkgs, [name], casematch=1, pkgdict=self.pkg_refs.copy() - ) - matches = exactmatched + matched - - matches = filter(self._filtersrcdebug, matches) - - if multilib and self.greedy_method != "all": - matches = [ - po for po in matches if po.arch in self.valid_multilib_arches - ] - - if not matches: - self.logger.warning( - "Could not find a match for %s in any configured repo", pattern - ) - continue - - packages_by_name = {} - for po in matches: - packages_by_name.setdefault(po.name, []).append(po) - - for name, packages in packages_by_name.iteritems(): - packages = self.excludePackages(packages or []) - if not packages: - continue - if self.greedy_method == "all": - packages = yum.packageSack.ListPackageSack( - packages - ).returnNewestByNameArch() - else: - # works for both "none" and "build" greedy methods - packages = [self.ayum._bestPackageFromList(packages)] - - if orig_name in input_packages: - self.input_packages.update(packages) - if name in comps_package_names: - self.comps_packages.update(packages) - - for po in packages: - msg = "Found %s.%s" % (po.name, po.arch) - self.add_package(po, msg) - name_arch = "%s.%s" % (po.name, po.arch) - if name_arch in prepopulate_packages: - self.prepopulate_packages.add(po) - - self.logger.info("Finished gathering package objects.") - - def gather(self): - # get package objects according to the input list - self.getPackageObjects() - if self.is_sources: - self.createSourceHashes() - - pass_num = 0 - added = set() - while 1: - if pass_num > 0 and not added: - break - added = set() - pass_num += 1 - self.logger.info("Pass #%s" % pass_num) - - if self.is_resolve_deps: - # get conditional deps (defined in comps) - for txmbr in self.ayum.tsInfo: - if txmbr.po not in self.po_list: - if not is_package(txmbr.po): - # we don't want sources which can be pulled in, - # because 'src' arch is part of self.valid_arches - continue - if not txmbr.isDep: - continue - self.add_package(txmbr.po) - - # resolve deps - if self.is_resolve_deps: - for po in sorted(self.po_list): - added.update(self.get_package_deps(po)) - - if self.is_sources: - added_srpms = self.add_srpms() - added.update(added_srpms) - - if self.is_selfhosting: - for srpm_po in sorted(added_srpms): - added.update(self.get_package_deps(srpm_po)) - - if self.is_fulltree: - new = self.add_fulltree() - self.fulltree_packages.update(new) - self.fulltree_packages.update( - [self.sourcerpm_srpmpo_map[i.sourcerpm] for i in new] - ) - added.update(new) - if added: - continue - - # add langpacks - new = self.add_langpacks(self.po_list) - self.langpack_packages.update(new) - if self.is_sources: - self.langpack_packages.update( - [self.sourcerpm_srpmpo_map[i.sourcerpm] for i in new] - ) - added.update(new) - if added: - continue - - # add multilib packages - new = self.add_multilib(self.po_list) - self.multilib_packages.update(new) - self.multilib_packages.update( - [self.sourcerpm_srpmpo_map[i.sourcerpm] for i in new] - ) - added.update(new) - if added: - continue - - def get_srpm_po(self, po): - """Given a package object, get a package object for the - corresponding source rpm.""" - - # return srpm_po from cache if available - srpm_po = self.sourcerpm_srpmpo_map.get(po.sourcerpm, None) - if srpm_po is not None: - return srpm_po - - # arch can be "src" or "nosrc" - nvr, arch, _ = po.sourcerpm.rsplit(".", 2) - name, ver, rel = nvr.rsplit("-", 2) - - # ... but even "nosrc" packages are stored as "src" in repodata - srpm_po_list = self.ayum.pkgSack.searchNevra( - name=name, ver=ver, rel=rel, arch="src" - ) - srpm_po_list = self.excludePackages(srpm_po_list) - try: - srpm_po = srpm_po_list[0] - except IndexError: - self.logger.warning("Cannot find a source rpm for %s" % po.sourcerpm) - srpm_po = None - self.sourcerpm_srpmpo_map[po.sourcerpm] = srpm_po - return srpm_po - - def createSourceHashes(self): - """Create two dicts - one that maps binary POs to source POs, and - one that maps a single source PO to all binary POs it produces. - Requires yum still configured.""" - self.src_by_bin = {} - self.bin_by_src = {} - self.logger.info("Generating source <-> binary package mappings") - for po in self.all_pkgs: - if is_source(po): - continue - srpmpo = self.get_srpm_po(po) - - self.src_by_bin[po] = srpmpo - self.bin_by_src.setdefault(srpmpo, []).append(po) - - def add_srpms(self, po_list=None): - """Cycle through the list of package objects and - find the sourcerpm for them. Requires yum still - configured and a list of package objects""" - - srpms = set() - po_list = po_list or self.po_list - for po in sorted(po_list): - try: - srpm_po = self.sourcerpm_srpmpo_map[po.sourcerpm] - except KeyError: - self.logger.error( - "Cannot get source RPM '%s' for %s" % (po.sourcerpm, po.nvra) - ) - srpm_po = None - - if srpm_po is None: - continue - - # flags - if po in self.input_packages: - self.input_packages.add(srpm_po) - if po in self.fulltree_packages: - self.fulltree_packages.add(srpm_po) - if po in self.langpack_packages: - self.langpack_packages.add(srpm_po) - if po in self.multilib_packages: - self.multilib_packages.add(srpm_po) - - if srpm_po in self.completed_add_srpms: - continue - - msg = "Added source package %s.%s (repo: %s)" % ( - srpm_po.name, - srpm_po.arch, - srpm_po.repoid, - ) - self.add_source(srpm_po, msg) - - self.completed_add_srpms.add(srpm_po) - srpms.add(srpm_po) - return srpms - - def add_fulltree(self, srpm_po_list=None): - """Cycle through all package objects, and add any - that correspond to a source rpm that we are including. - Requires yum still configured and a list of package - objects.""" - - self.logger.info("Completing package set") - - srpm_po_list = srpm_po_list or self.srpm_po_list - srpms = [] - for srpm_po in srpm_po_list: - if srpm_po in self.completed_fulltree: - continue - if srpm_po.name not in self.fulltree_excludes: - srpms.append(srpm_po) - self.completed_fulltree.add(srpm_po) - - added = set() - for srpm_po in srpms: - if srpm_po.repoid in self.lookaside_repos: - # Don't run fulltree on packages in lookaside - continue - include_native = False - include_multilib = False - has_native = False - has_multilib = False - - for po in self.excludePackages(self.bin_by_src[srpm_po]): - if not is_package(po): - continue - if po.arch == "noarch": - continue - if po not in self.po_list: - # process only already included packages - if po.arch in self.valid_multilib_arches: - has_multilib = True - elif po.arch in self.valid_native_arches: - has_native = True - continue - if po.arch in self.valid_multilib_arches and ( - po in self.input_packages or self.greedy_method == "all" - ): - include_multilib = True - elif po.arch in self.valid_native_arches: - include_native = True - - # XXX: this is very fragile! - # Do not make any changes unless you really know what you're doing! - if not include_native: - # if there's no native package already pulled in... - if has_native and not include_multilib: - # include all native packages, but only if we're not pulling - # multilib already - # SCENARIO: a noarch package was already pulled in and there - # are x86_64 and i686 packages -> we want x86_64 in to complete - # the package set - include_native = True - elif has_multilib: - # SCENARIO: a noarch package was already pulled in and there are - # no x86_64 packages; we want i686 in to complete the package set - include_multilib = True - - for po in self.excludePackages(self.bin_by_src[srpm_po]): - if not is_package(po): - continue - if po in self.po_list: - continue - if po.arch != "noarch": - if po.arch in self.valid_multilib_arches: - if not include_multilib: - continue - if po.arch in self.valid_native_arches: - if not include_native: - continue - msg = "Added %s.%s (repo: %s) to complete package set" % ( - po.name, - po.arch, - po.repoid, - ) - self.add_package(po, msg) - return added - - def getDebuginfoList(self): - """Cycle through the list of package objects and find - debuginfo rpms for them. Requires yum still - configured and a list of package objects""" - - added = set() - for po in self.all_pkgs: - if not pungi.util.pkg_is_debug(po): - continue - - if po.sourcerpm not in self.sourcerpm_arch_map: - # TODO: print a warning / throw an error - continue - if po.arch != "noarch" and not ( - set(self.compatible_arches[po.arch]) - & set(self.sourcerpm_arch_map[po.sourcerpm]) - set(["noarch"]) - ): - # skip all incompatible arches unless it's a noarch debuginfo - # this pulls i386 debuginfo for a i686 package for example - continue - msg = "Added debuginfo %s.%s (repo: %s)" % (po.name, po.arch, po.repoid) - self.add_debuginfo(po, msg) - - # flags - try: - srpm_po = self.sourcerpm_srpmpo_map[po.sourcerpm] - except Exception: - self.logger.warning("Failed to find source for %s", po.sourcerpm) - srpm_po = None - if srpm_po in self.input_packages: - self.input_packages.add(po) - if srpm_po in self.fulltree_packages: - self.fulltree_packages.add(po) - if srpm_po in self.langpack_packages: - self.langpack_packages.add(po) - if srpm_po in self.multilib_packages: - self.multilib_packages.add(po) - - added.add(po) - return added - - def _downloadPackageList(self, polist, relpkgdir): - """Cycle through the list of package objects and - download them from their respective repos.""" - - for pkg in sorted(polist): - repo = self.ayum.repos.getRepo(pkg.repoid) - self.logger.info( - "Downloading %s.%s from %s", - pkg.name, - pkg.arch, - repo.baseurl or repo.mirrorlist, - ) - - pkgdir = os.path.join( - self.config.get("pungi", "destdir"), - self.config.get("pungi", "version"), - self.config.get("pungi", "variant"), - relpkgdir, - ) - - # Ensure the pkgdir exists, force if requested, and make sure we clean it out - if relpkgdir.endswith("SRPMS"): - # Since we share source dirs with other arches don't clean, but - # do allow us to use it - pungi.util._ensuredir(pkgdir, self.logger, force=True, clean=False) - else: - pungi.util._ensuredir( - pkgdir, - self.logger, - force=self.config.getboolean("pungi", "force"), - clean=True, - ) - - probs = self.ayum.downloadPkgs(polist) - - if len(probs.keys()) > 0: - self.logger.error("Errors were encountered while downloading packages.") - for key in probs.keys(): - errors = yum.misc.unique(probs[key]) - for error in errors: - self.logger.error("%s: %s" % (key, error)) - sys.exit(1) - - for po in polist: - basename = os.path.basename(po.relativepath) - - local = po.localPkg() - if self.config.getboolean("pungi", "nohash"): - target = os.path.join(pkgdir, basename) - else: - target = os.path.join(pkgdir, po.name[0].lower(), basename) - # Make sure we have the hashed dir available to link into we - # only want dirs there to corrospond to packages - # that we are including so we can not just do A-Z 0-9 - pungi.util._ensuredir( - os.path.join(pkgdir, po.name[0].lower()), - self.logger, - force=True, - clean=False, - ) - - # Link downloaded package in (or link package from file repo) - try: - pungi.util._link(local, target, self.logger, force=True) - continue - except Exception: - self.logger.error("Unable to link %s from the yum cache." % po.name) - sys.exit(1) - - self.logger.info("Finished downloading packages.") - - @yumlocked - def downloadPackages(self): - """Download the package objects obtained in getPackageObjects().""" - - self._downloadPackageList( - self.po_list, - os.path.join( - self.tree_arch, - self.config.get("pungi", "osdir"), - self.config.get("pungi", "product_path"), - ), - ) - - def makeCompsFile(self): - """Gather any comps files we can from repos and merge them into one.""" - - ourcompspath = os.path.join( - self.workdir, - "%s-%s-comps.xml" - % (self.config.get("pungi", "family"), self.config.get("pungi", "version")), - ) - - # Filter out things we don't include - ourgroups = [] - for item in self.ksparser.handler.packages.groupList: - grp = self.ayum.comps.return_group(item.name) - if grp: - ourgroups.append(grp.groupid) - allgroups = [g.groupid for g in self.ayum.comps.get_groups()] - for group in allgroups: - if ( - group not in ourgroups - and not self.ayum.comps.return_group(group).langonly - ): - self.logger.info("Removing extra group %s from comps file" % (group,)) - del self.ayum.comps._groups[group] - - groups = [g.groupid for g in self.ayum.comps.get_groups()] - envs = self.ayum.comps.get_environments() - for env in envs: - for group in env.groups: - if group not in groups: - self.logger.info( - "Removing incomplete environment %s from comps file" % (env,) - ) - del self.ayum.comps._environments[env.environmentid] - break - - ourcomps = open(ourcompspath, "w") - ourcomps.write(self.ayum.comps.xml()) - ourcomps.close() - - # Disable this until https://bugzilla.redhat.com/show_bug.cgi?id=442097 - # is fixed. - # Run the xslt filter over our comps file - # compsfilter = ['/usr/bin/xsltproc', '--novalid'] - # compsfilter.append('-o') - # compsfilter.append(ourcompspath) - # compsfilter.append('/usr/share/pungi/comps-cleanup.xsl') - # compsfilter.append(ourcompspath) - - # pungi.util._doRunCommand(compsfilter, self.logger) - - def _list_packages(self, po_list): - """Cycle through the list of packages and return their paths.""" - result = [] - for po in po_list: - if po.repoid in self.lookaside_repos: - continue - - flags = [] - - # input - if po in self.input_packages: - flags.append("input") - - # comps - if po in self.comps_packages: - flags.append("comps") - - # prepopulate - if po in self.prepopulate_packages: - flags.append("prepopulate") - - # langpack - if po in self.langpack_packages: - flags.append("langpack") - - # multilib - if po in self.multilib_packages: - flags.append("multilib") - - # fulltree - if po in self.fulltree_packages: - flags.append("fulltree") - - # fulltree-exclude - if is_source(po): - srpm_name = po.name - else: - srpm_name = po.sourcerpm.rsplit("-", 2)[0] - if srpm_name in self.fulltree_excludes: - flags.append("fulltree-exclude") - - result.append( - { - "path": os.path.join(po.basepath or "", po.relativepath), - "flags": sorted(flags), - } - ) - result.sort(lambda x, y: cmp(x["path"], y["path"])) # noqa: F821 (py2 only) - return result - - def list_packages(self): - """Cycle through the list of RPMs and return their paths.""" - return self._list_packages(self.po_list) - - def list_srpms(self): - """Cycle through the list of SRPMs and return their paths.""" - return self._list_packages(self.srpm_po_list) - - def list_debuginfo(self): - """Cycle through the list of DEBUGINFO RPMs and return their paths.""" - return self._list_packages(self.debuginfo_po_list) diff --git a/pungi/gather_dnf.py b/pungi/gather_dnf.py index beeec294..095ef6bd 100644 --- a/pungi/gather_dnf.py +++ b/pungi/gather_dnf.py @@ -1080,7 +1080,7 @@ class Gather(GatherBase): if ex.errno == errno.EEXIST: self.logger.warning("Downloaded package exists in %s", target) else: - self.logger.error("Unable to link %s from the yum cache.", pkg.name) + self.logger.error("Unable to link %s from the dnf cache.", pkg.name) raise def log_count(self, msg, method, *args): diff --git a/pungi/phases/gather/methods/method_deps.py b/pungi/phases/gather/methods/method_deps.py index 4b785629..451d188a 100644 --- a/pungi/phases/gather/methods/method_deps.py +++ b/pungi/phases/gather/methods/method_deps.py @@ -15,7 +15,6 @@ import os -import shutil from kobo.shortcuts import run from kobo.pkgset import SimpleRpmWrapper, RpmWrapper @@ -220,9 +219,7 @@ def resolve_deps(compose, arch, variant, source_name=None): yum_arch = tree_arch_to_yum_arch(arch) tmp_dir = compose.paths.work.tmp_dir(arch, variant) cache_dir = compose.paths.work.pungi_cache_dir(arch, variant) - # TODO: remove YUM code, fully migrate to DNF backends = { - "yum": pungi_wrapper.get_pungi_cmd, "dnf": pungi_wrapper.get_pungi_cmd_dnf, } get_cmd = backends[compose.conf["gather_backend"]] @@ -245,17 +242,6 @@ def resolve_deps(compose, arch, variant, source_name=None): with temp_dir(prefix="pungi_") as work_dir: run(cmd, logfile=pungi_log, show_cmd=True, workdir=work_dir, env=os.environ) - # Clean up tmp dir - # Workaround for rpm not honoring sgid bit which only appears when yum is used. - yumroot_dir = os.path.join(tmp_dir, "work", arch, "yumroot") - if os.path.isdir(yumroot_dir): - try: - shutil.rmtree(yumroot_dir) - except Exception as e: - compose.log_warning( - "Failed to clean up tmp dir: %s %s" % (yumroot_dir, str(e)) - ) - with open(pungi_log, "r") as f: packages, broken_deps, missing_comps_pkgs = pungi_wrapper.parse_log(f) diff --git a/pungi/phases/pkgset/sources/source_repos.py b/pungi/phases/pkgset/sources/source_repos.py index 716f6336..e710c9aa 100644 --- a/pungi/phases/pkgset/sources/source_repos.py +++ b/pungi/phases/pkgset/sources/source_repos.py @@ -15,7 +15,6 @@ import os -import shutil from kobo.shortcuts import run @@ -76,7 +75,6 @@ def get_pkgset_from_repos(compose): pungi_dir = compose.paths.work.pungi_download_dir(arch) backends = { - "yum": pungi.get_pungi_cmd, "dnf": pungi.get_pungi_cmd_dnf, } get_cmd = backends[compose.conf["gather_backend"]] @@ -93,8 +91,6 @@ def get_pkgset_from_repos(compose): cache_dir=compose.paths.work.pungi_cache_dir(arch=arch), profiler=profiler, ) - if compose.conf["gather_backend"] == "yum": - cmd.append("--force") # TODO: runroot run(cmd, logfile=pungi_log, show_cmd=True, stdout=False) @@ -111,17 +107,6 @@ def get_pkgset_from_repos(compose): flist.append(dst) pool.queue_put((src, dst)) - # Clean up tmp dir - # Workaround for rpm not honoring sgid bit which only appears when yum is used. - yumroot_dir = os.path.join(pungi_dir, "work", arch, "yumroot") - if os.path.isdir(yumroot_dir): - try: - shutil.rmtree(yumroot_dir) - except Exception as e: - compose.log_warning( - "Failed to clean up tmp dir: %s %s" % (yumroot_dir, str(e)) - ) - msg = "Linking downloaded pkgset packages" compose.log_info("[BEGIN] %s" % msg) pool.start() diff --git a/pungi/phases/repoclosure.py b/pungi/phases/repoclosure.py index 9cca398f..86d2d190 100644 --- a/pungi/phases/repoclosure.py +++ b/pungi/phases/repoclosure.py @@ -101,18 +101,13 @@ def run_repoclosure(compose): def _delete_repoclosure_cache_dirs(compose): - if "dnf" == compose.conf["repoclosure_backend"]: - from dnf.const import SYSTEM_CACHEDIR - from dnf.util import am_i_root - from dnf.yum.misc import getCacheDir + from dnf.const import SYSTEM_CACHEDIR + from dnf.util import am_i_root + from dnf.yum.misc import getCacheDir - if am_i_root(): - top_cache_dir = SYSTEM_CACHEDIR - else: - top_cache_dir = getCacheDir() + if am_i_root(): + top_cache_dir = SYSTEM_CACHEDIR else: - from yum.misc import getCacheDir - top_cache_dir = getCacheDir() for name in os.listdir(top_cache_dir): diff --git a/pungi/scripts/pungi.py b/pungi/scripts/pungi.py deleted file mode 100644 index b6dc91ef..00000000 --- a/pungi/scripts/pungi.py +++ /dev/null @@ -1,304 +0,0 @@ -# This program is free software; you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation; version 2 of the License. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU Library General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program; if not, see . - -from __future__ import absolute_import -from __future__ import print_function - -import os -import sys - -from argparse import ArgumentParser, Action - -from pungi import get_full_version -import pungi.gather -import pungi.config -import pungi.ks - - -def get_arguments(config): - parser = ArgumentParser() - - class SetConfig(Action): - def __call__(self, parser, namespace, value, option_string=None): - config.set("pungi", self.dest, value) - - parser.add_argument("--version", action="version", version=get_full_version()) - - # Pulled in from config file to be cli options as part of pykickstart conversion - parser.add_argument( - "--destdir", - dest="destdir", - action=SetConfig, - help="destination directory (defaults to current directory)", - ) - parser.add_argument( - "--cachedir", - dest="cachedir", - action=SetConfig, - help="package cache directory (defaults to /var/cache/pungi)", - ) - parser.add_argument( - "--selfhosting", - action="store_true", - dest="selfhosting", - help="build a self-hosting tree by following build dependencies (optional)", - ) - parser.add_argument( - "--fulltree", - action="store_true", - dest="fulltree", - help="build a tree that includes all packages built from corresponding source rpms (optional)", # noqa: E501 - ) - parser.add_argument( - "--nosource", - action="store_true", - dest="nosource", - help="disable gathering of source packages (optional)", - ) - parser.add_argument( - "--nodebuginfo", - action="store_true", - dest="nodebuginfo", - help="disable gathering of debuginfo packages (optional)", - ) - parser.add_argument( - "--nodownload", - action="store_true", - dest="nodownload", - help="disable downloading of packages. instead, print the package URLs (optional)", # noqa: E501 - ) - parser.add_argument( - "--nogreedy", - action="store_true", - dest="nogreedy", - help="disable pulling of all providers of package dependencies (optional)", - ) - parser.add_argument( - "--nodeps", - action="store_false", - dest="resolve_deps", - default=True, - help="disable resolving dependencies", - ) - parser.add_argument( - "--force", - default=False, - action="store_true", - help="Force reuse of an existing destination directory (will overwrite files)", - ) - parser.add_argument( - "--nohash", - default=False, - action="store_true", - help="disable hashing the Packages trees", - ) - parser.add_argument( - "--full-archlist", - action="store_true", - help="Use the full arch list for x86_64 (include i686, i386, etc.)", - ) - parser.add_argument("--arch", help="Override default (uname based) arch") - parser.add_argument( - "--greedy", metavar="METHOD", help="Greedy method; none, all, build" - ) - parser.add_argument( - "--multilib", - action="append", - metavar="METHOD", - help="Multilib method; can be specified multiple times; recommended: devel, runtime", # noqa: E501 - ) - parser.add_argument( - "--lookaside-repo", - action="append", - dest="lookaside_repos", - metavar="NAME", - help="Specify lookaside repo name(s) (packages will used for depsolving but not be included in the output)", # noqa: E501 - ) - parser.add_argument( - "--workdirbase", - dest="workdirbase", - action=SetConfig, - help="base working directory (defaults to destdir + /work)", - ) - parser.add_argument( - "--no-dvd", - default=False, - action="store_true", - dest="no_dvd", - help="Do not make a install DVD/CD only the netinstall image and the tree", - ) - parser.add_argument("--lorax-conf", help="Path to lorax.conf file (optional)") - parser.add_argument( - "-i", - "--installpkgs", - default=[], - action="append", - metavar="STRING", - help="Package glob for lorax to install before runtime-install.tmpl runs. (may be listed multiple times)", # noqa: E501 - ) - parser.add_argument( - "--multilibconf", - default=None, - action=SetConfig, - help="Path to multilib conf files. Default is /usr/share/pungi/multilib/", - ) - - parser.add_argument( - "-c", - "--config", - dest="config", - required=True, - help="Path to kickstart config file", - ) - parser.add_argument( - "-G", - action="store_true", - default=False, - dest="do_gather", - help="Flag to enable processing the Gather stage", - ) - - parser.add_argument( - "--pungirc", - dest="pungirc", - default="~/.pungirc", - action=SetConfig, - help="Read pungi options from config file ", - ) - - return parser.parse_args() - - -def main(): - config = pungi.config.Config() - opts = get_arguments(config) - - # Read the config to create "new" defaults - # reparse command line options so they take precedence - config = pungi.config.Config(pungirc=opts.pungirc) - opts = get_arguments(config) - - # Set up the kickstart parser and pass in the kickstart file we were handed - ksparser = pungi.ks.get_ksparser(ks_path=opts.config) - - config.set("pungi", "force", str(opts.force)) - - if config.get("pungi", "workdirbase") == "/work": - config.set("pungi", "workdirbase", "%s/work" % config.get("pungi", "destdir")) - # Set up our directories - if not os.path.exists(config.get("pungi", "destdir")): - try: - os.makedirs(config.get("pungi", "destdir")) - except OSError: - print( - "Error: Cannot create destination dir %s" - % config.get("pungi", "destdir"), - file=sys.stderr, - ) - sys.exit(1) - else: - print("Warning: Reusing existing destination directory.") - - if not os.path.exists(config.get("pungi", "workdirbase")): - try: - os.makedirs(config.get("pungi", "workdirbase")) - except OSError: - print( - "Error: Cannot create working base dir %s" - % config.get("pungi", "workdirbase"), - file=sys.stderr, - ) - sys.exit(1) - else: - print("Warning: Reusing existing working base directory.") - - cachedir = config.get("pungi", "cachedir") - - if not os.path.exists(cachedir): - try: - os.makedirs(cachedir) - except OSError: - print("Error: Cannot create cache dir %s" % cachedir, file=sys.stderr) - sys.exit(1) - - # Set debuginfo flag - if opts.nodebuginfo: - config.set("pungi", "debuginfo", "False") - if opts.greedy: - config.set("pungi", "greedy", opts.greedy) - else: - # XXX: compatibility - if opts.nogreedy: - config.set("pungi", "greedy", "none") - else: - config.set("pungi", "greedy", "all") - config.set("pungi", "resolve_deps", str(bool(opts.resolve_deps))) - if opts.nohash: - config.set("pungi", "nohash", "True") - if opts.full_archlist: - config.set("pungi", "full_archlist", "True") - if opts.arch: - config.set("pungi", "arch", opts.arch) - if opts.multilib: - config.set("pungi", "multilib", " ".join(opts.multilib)) - if opts.lookaside_repos: - config.set("pungi", "lookaside_repos", " ".join(opts.lookaside_repos)) - config.set("pungi", "fulltree", str(bool(opts.fulltree))) - config.set("pungi", "selfhosting", str(bool(opts.selfhosting))) - config.set("pungi", "nosource", str(bool(opts.nosource))) - config.set("pungi", "nodebuginfo", str(bool(opts.nodebuginfo))) - - # Actually do work. - mypungi = pungi.gather.Pungi(config, ksparser) - - with mypungi.yumlock: - mypungi._inityum() # initialize the yum object for things that need it - mypungi.gather() - if opts.nodownload: - for line in mypungi.list_packages(): - flags_str = ",".join(line["flags"]) - if flags_str: - flags_str = "(%s)" % flags_str - sys.stdout.write("RPM%s: %s\n" % (flags_str, line["path"])) - sys.stdout.flush() - else: - mypungi.downloadPackages() - mypungi.makeCompsFile() - if not opts.nodebuginfo: - mypungi.getDebuginfoList() - if opts.nodownload: - for line in mypungi.list_debuginfo(): - flags_str = ",".join(line["flags"]) - if flags_str: - flags_str = "(%s)" % flags_str - sys.stdout.write("DEBUGINFO%s: %s\n" % (flags_str, line["path"])) - sys.stdout.flush() - else: - mypungi.downloadDebuginfo() - if not opts.nosource: - if opts.nodownload: - for line in mypungi.list_srpms(): - flags_str = ",".join(line["flags"]) - if flags_str: - flags_str = "(%s)" % flags_str - sys.stdout.write("SRPM%s: %s\n" % (flags_str, line["path"])) - sys.stdout.flush() - else: - mypungi.downloadSRPMs() - - print("RPM size: %s MiB" % (mypungi.size_packages() / 1024**2)) - if not opts.nodebuginfo: - print("DEBUGINFO size: %s MiB" % (mypungi.size_debuginfo() / 1024**2)) - if not opts.nosource: - print("SRPM size: %s MiB" % (mypungi.size_srpms() / 1024**2)) - - print("All done!") diff --git a/pungi/wrappers/pungi.py b/pungi/wrappers/pungi.py index 8d11322e..0be13d2d 100644 --- a/pungi/wrappers/pungi.py +++ b/pungi/wrappers/pungi.py @@ -105,74 +105,6 @@ class PungiWrapper(object): kickstart.close() - def get_pungi_cmd( - self, - config, - destdir, - name, - version=None, - flavor=None, - selfhosting=False, - fulltree=False, - greedy=None, - nodeps=False, - nodownload=True, - full_archlist=False, - arch=None, - cache_dir=None, - lookaside_repos=None, - multilib_methods=None, - profiler=False, - ): - cmd = ["pungi"] - - # Gather stage - cmd.append("-G") - - # path to a kickstart file - cmd.append("--config=%s" % config) - - # destdir is optional in Pungi (defaults to current dir), but - # want it mandatory here - cmd.append("--destdir=%s" % destdir) - - # turn selfhosting on - if selfhosting: - cmd.append("--selfhosting") - - # NPLB - if fulltree: - cmd.append("--fulltree") - - greedy = greedy or "none" - cmd.append("--greedy=%s" % greedy) - - if nodeps: - cmd.append("--nodeps") - - # don't download packages, just print paths - if nodownload: - cmd.append("--nodownload") - - if full_archlist: - cmd.append("--full-archlist") - - if arch: - cmd.append("--arch=%s" % arch) - - if multilib_methods: - for i in multilib_methods: - cmd.append("--multilib=%s" % i) - - if cache_dir: - cmd.append("--cachedir=%s" % cache_dir) - - if lookaside_repos: - for i in lookaside_repos: - cmd.append("--lookaside-repo=%s" % i) - - return cmd - def get_pungi_cmd_dnf( self, config, @@ -258,68 +190,3 @@ class PungiWrapper(object): broken_deps.setdefault(match.group(2), set()).add(match.group(1)) return packages, broken_deps, missing_comps - - def run_pungi( - self, - ks_file, - destdir, - name, - selfhosting=False, - fulltree=False, - greedy="", - cache_dir=None, - arch="", - multilib_methods=[], - nodeps=False, - lookaside_repos=[], - ): - """ - This is a replacement for get_pungi_cmd that runs it in-process. Not - all arguments are supported. - """ - from .. import ks, gather, config - - ksparser = ks.get_ksparser(ks_path=ks_file) - cfg = config.Config() - cfg.set("pungi", "destdir", destdir) - cfg.set("pungi", "fulltree", str(fulltree)) - cfg.set("pungi", "selfhosting", str(selfhosting)) - cfg.set("pungi", "cachedir", cache_dir) - cfg.set("pungi", "full_archlist", "True") - cfg.set("pungi", "workdirbase", "%s/work" % destdir) - cfg.set("pungi", "greedy", greedy) - cfg.set("pungi", "nosource", "False") - cfg.set("pungi", "nodebuginfo", "False") - cfg.set("pungi", "force", "False") - cfg.set("pungi", "resolve_deps", str(not nodeps)) - if arch: - cfg.set("pungi", "arch", arch) - if multilib_methods: - cfg.set("pungi", "multilib", " ".join(multilib_methods)) - if lookaside_repos: - cfg.set("pungi", "lookaside_repos", " ".join(lookaside_repos)) - - mypungi = gather.Pungi(cfg, ksparser) - - with open(os.path.join(destdir, "out"), "w") as f: - with mypungi.yumlock: - mypungi._inityum() - mypungi.gather() - - for line in mypungi.list_packages(): - flags_str = ",".join(line["flags"]) - if flags_str: - flags_str = "(%s)" % flags_str - f.write("RPM%s: %s\n" % (flags_str, line["path"])) - mypungi.makeCompsFile() - mypungi.getDebuginfoList() - for line in mypungi.list_debuginfo(): - flags_str = ",".join(line["flags"]) - if flags_str: - flags_str = "(%s)" % flags_str - f.write("DEBUGINFO%s: %s\n" % (flags_str, line["path"])) - for line in mypungi.list_srpms(): - flags_str = ",".join(line["flags"]) - if flags_str: - flags_str = "(%s)" % flags_str - f.write("SRPM%s: %s\n" % (flags_str, line["path"])) diff --git a/pungi/wrappers/repoclosure.py b/pungi/wrappers/repoclosure.py index 268df094..dff24c71 100644 --- a/pungi/wrappers/repoclosure.py +++ b/pungi/wrappers/repoclosure.py @@ -19,13 +19,8 @@ import os from kobo.shortcuts import force_list -def get_repoclosure_cmd(backend="yum", arch=None, repos=None, lookaside=None): +def get_repoclosure_cmd(backend="dnf", arch=None, repos=None, lookaside=None): cmds = { - "yum": { - "cmd": ["/usr/bin/repoclosure", "--tempcache"], - "repoarg": "--repoid=%s", - "lookaside": "--lookaside=%s", - }, "dnf": { "cmd": ["dnf", "repoclosure"], "repoarg": "--repo=%s", @@ -44,18 +39,17 @@ def get_repoclosure_cmd(backend="yum", arch=None, repos=None, lookaside=None): for i in arches: cmd.append("--arch=%s" % i) - if backend == "dnf" and arches: + if arches: cmd.append("--forcearch=%s" % arches[0]) repos = repos or {} for repo_id, repo_path in repos.items(): cmd.append("--repofrompath=%s,%s" % (repo_id, _to_url(repo_path))) cmd.append(cmds[backend]["repoarg"] % repo_id) - if backend == "dnf": - # For dnf we want to add all repos with the --repo option (which - # enables only those and not any system repo), and the repos to - # check are also listed with the --check option. - cmd.append("--check=%s" % repo_id) + # For dnf we want to add all repos with the --repo option (which + # enables only those and not any system repo), and the repos to + # check are also listed with the --check option. + cmd.append("--check=%s" % repo_id) lookaside = lookaside or {} for repo_id, repo_path in lookaside.items(): diff --git a/setup.py b/setup.py index 1054d1e0..724620f5 100755 --- a/setup.py +++ b/setup.py @@ -30,7 +30,6 @@ setup( entry_points={ "console_scripts": [ "comps_filter = pungi.scripts.comps_filter:main", - "pungi = pungi.scripts.pungi:main", "pungi-create-unified-isos = pungi.scripts.create_unified_isos:main", "pungi-fedmsg-notification = pungi.scripts.fedmsg_notification:main", "pungi-patch-iso = pungi.scripts.patch_iso:cli_main", diff --git a/tests/Dockerfile-test-py2 b/tests/Dockerfile-test-py2 deleted file mode 100644 index 84ce1f99..00000000 --- a/tests/Dockerfile-test-py2 +++ /dev/null @@ -1,27 +0,0 @@ -FROM centos:7 -LABEL \ - name="Pungi test" \ - description="Run tests using tox with Python 2" \ - vendor="Pungi developers" \ - license="MIT" - -RUN yum -y update && yum -y install epel-release && yum -y install \ - git \ - libmodulemd2 \ - make \ - python3 \ - python-createrepo_c \ - python-gobject-base \ - python-gssapi \ - python-libcomps \ - pykickstart \ - && yum clean all - -# python-tox in yum repo is too old, let's install latest version -RUN pip3 install tox - -WORKDIR /src - -COPY . . - -CMD ["tox", "-e", "py27"] diff --git a/tests/Jenkinsfile b/tests/Jenkinsfile index 04826e8c..ebd56d43 100644 --- a/tests/Jenkinsfile +++ b/tests/Jenkinsfile @@ -40,7 +40,6 @@ git fetch proposed git checkout origin/master git merge --no-ff "proposed/$params.BRANCH" -m "Merge PR" podman run --rm -v .:/src:Z quay.io/exd-guild-compose/pungi-test tox -r -e flake8,black,py3,bandit -podman run --rm -v .:/src:Z quay.io/exd-guild-compose/pungi-test-py2 tox -r -e py27 """ sh "cat job.sh" sh "ssh -o StrictHostKeyChecking=no root@$hostname mkdir $remote_dir" diff --git a/tests/test_compose.py b/tests/test_compose.py index 41abbaf9..07f32b59 100644 --- a/tests/test_compose.py +++ b/tests/test_compose.py @@ -748,15 +748,6 @@ class StatusTest(unittest.TestCase): self.compose.conf["createrepo_database"] = True self.assertTrue(self.compose.should_create_yum_database) - def test_no_database_with_yum_backend(self): - self.compose.conf["gather_backend"] = "yum" - self.assertTrue(self.compose.should_create_yum_database) - - def test_no_database_with_yum_backend_config_override(self): - self.compose.conf["gather_backend"] = "yum" - self.compose.conf["createrepo_database"] = False - self.assertFalse(self.compose.should_create_yum_database) - class DumpContainerMetadataTest(unittest.TestCase): def setUp(self): diff --git a/tests/test_config.py b/tests/test_config.py index 7d084bd7..26f68d1d 100644 --- a/tests/test_config.py +++ b/tests/test_config.py @@ -271,16 +271,6 @@ class GatherConfigTestCase(ConfigTestCase): self.assertValidation(cfg, []) self.assertEqual(cfg["gather_backend"], "dnf") - def test_yum_backend_is_default_on_py2(self): - cfg = load_config( - pkgset_source="koji", - pkgset_koji_tag="f27", - ) - - with mock.patch("six.PY2", new=True): - self.assertValidation(cfg, []) - self.assertEqual(cfg["gather_backend"], "yum") - def test_yum_backend_is_rejected_on_py3(self): cfg = load_config( pkgset_source="koji", @@ -464,7 +454,7 @@ class RepoclosureTestCase(ConfigTestCase): repoclosure_backend="fnd", # Intentionally with a typo ) - options = ["yum", "dnf"] if six.PY2 else ["dnf"] + options = ["dnf"] self.assertValidation( cfg, [ diff --git a/tests/test_gather.py b/tests/test_gather.py index 1ae95797..51fda80e 100644 --- a/tests/test_gather.py +++ b/tests/test_gather.py @@ -25,11 +25,6 @@ try: except ImportError: HAS_DNF = False -if six.PY2: - HAS_YUM = True -else: - HAS_YUM = False - def convert_pkg_map(data): """ @@ -2137,71 +2132,6 @@ class DepsolvingBase(object): self.assertEqual(pkg_map["debuginfo"], []) -@unittest.skipUnless(HAS_YUM, "YUM only available on Python 2") -class PungiYumDepsolvingTestCase(DepsolvingBase, unittest.TestCase): - def setUp(self): - super(PungiYumDepsolvingTestCase, self).setUp() - self.ks = os.path.join(self.tmp_dir, "ks") - self.out = os.path.join(self.tmp_dir, "out") - self.cwd = os.path.join(self.tmp_dir, "cwd") - os.mkdir(self.cwd) - self.old_cwd = os.getcwd() - os.chdir(self.cwd) - - logger = logging.getLogger("Pungi") - if not logger.handlers: - formatter = logging.Formatter("%(name)s:%(levelname)s: %(message)s") - console = logging.StreamHandler(sys.stdout) - console.setFormatter(formatter) - console.setLevel(logging.INFO) - logger.addHandler(console) - - def tearDown(self): - os.chdir(self.old_cwd) - super(PungiYumDepsolvingTestCase, self).tearDown() - - def go( - self, - packages, - groups, - lookaside=None, - prepopulate=None, - fulltree_excludes=None, - multilib_blacklist=None, - multilib_whitelist=None, - **kwargs - ): - """ - Write a kickstart with given packages and groups, then run the - depsolving and parse the output. - """ - p = PungiWrapper() - repos = {"repo": self.repo} - if lookaside: - repos["lookaside"] = lookaside - kwargs["lookaside_repos"] = ["lookaside"] - p.write_kickstart( - self.ks, - repos, - groups, - packages, - prepopulate=prepopulate, - multilib_whitelist=multilib_whitelist, - multilib_blacklist=multilib_blacklist, - fulltree_excludes=fulltree_excludes, - ) - kwargs.setdefault("cache_dir", self.tmp_dir) - # Unless the test specifies an arch, we need to default to x86_64. - # Otherwise the arch of current machine will be used, which will cause - # failure most of the time. - kwargs.setdefault("arch", "x86_64") - - p.run_pungi(self.ks, self.tmp_dir, "DP", **kwargs) - with open(self.out, "r") as f: - pkg_map, self.broken_deps, _ = p.parse_log(f) - return convert_pkg_map(pkg_map) - - def convert_dnf_packages(pkgs, flags): convert_table = { # Hawkey returns nosrc package as src diff --git a/tests/test_repoclosure_phase.py b/tests/test_repoclosure_phase.py index bea7040e..2cc61626 100644 --- a/tests/test_repoclosure_phase.py +++ b/tests/test_repoclosure_phase.py @@ -1,31 +1,14 @@ # -*- coding: utf-8 -*- - try: - import unittest2 as unittest + from unittest import mock except ImportError: - import unittest - -from unittest import mock + import mock import six import pungi.phases.repoclosure as repoclosure_phase from tests.helpers import DummyCompose, PungiTestCase, mk_boom -try: - import dnf # noqa: F401 - - HAS_DNF = True -except ImportError: - HAS_DNF = False - -try: - import yum # noqa: F401 - - HAS_YUM = True -except ImportError: - HAS_YUM = False - class TestRepoclosure(PungiTestCase): def setUp(self): @@ -49,53 +32,6 @@ class TestRepoclosure(PungiTestCase): self.assertEqual(mock_grc.call_args_list, []) - @unittest.skipUnless(HAS_YUM, "YUM is not available") - @mock.patch("pungi.wrappers.repoclosure.get_repoclosure_cmd") - @mock.patch("pungi.phases.repoclosure.run") - def test_repoclosure_default_backend(self, mock_run, mock_grc): - with mock.patch("six.PY2", new=True): - compose = DummyCompose(self.topdir, {}) - - repoclosure_phase.run_repoclosure(compose) - - six.assertCountEqual( - self, - mock_grc.call_args_list, - [ - mock.call( - backend="yum", - arch=["amd64", "x86_64", "noarch"], - lookaside={}, - repos=self._get_repo(compose.compose_id, "Everything", "amd64"), - ), - mock.call( - backend="yum", - arch=["amd64", "x86_64", "noarch"], - lookaside={}, - repos=self._get_repo(compose.compose_id, "Client", "amd64"), - ), - mock.call( - backend="yum", - arch=["amd64", "x86_64", "noarch"], - lookaside={}, - repos=self._get_repo(compose.compose_id, "Server", "amd64"), - ), - mock.call( - backend="yum", - arch=["x86_64", "noarch"], - lookaside={}, - repos=self._get_repo(compose.compose_id, "Server", "x86_64"), - ), - mock.call( - backend="yum", - arch=["x86_64", "noarch"], - lookaside={}, - repos=self._get_repo(compose.compose_id, "Everything", "x86_64"), - ), - ], - ) - - @unittest.skipUnless(HAS_DNF, "DNF is not available") @mock.patch("pungi.wrappers.repoclosure.get_repoclosure_cmd") @mock.patch("pungi.phases.repoclosure.run") def test_repoclosure_dnf_backend(self, mock_run, mock_grc): @@ -179,7 +115,6 @@ class TestRepoclosure(PungiTestCase): with self.assertRaises(RuntimeError): repoclosure_phase.run_repoclosure(compose) - @unittest.skipUnless(HAS_DNF, "DNF is not available") @mock.patch("pungi.wrappers.repoclosure.get_repoclosure_cmd") @mock.patch("pungi.phases.repoclosure.run") def test_repoclosure_overwrite_options_creates_correct_commands( diff --git a/tests/test_repoclosure_wrapper.py b/tests/test_repoclosure_wrapper.py index 361d5846..6ea38b6a 100755 --- a/tests/test_repoclosure_wrapper.py +++ b/tests/test_repoclosure_wrapper.py @@ -9,11 +9,6 @@ from . import helpers class RepoclosureWrapperTestCase(helpers.BaseTestCase): - def test_minimal_command(self): - self.assertEqual( - rc.get_repoclosure_cmd(), ["/usr/bin/repoclosure", "--tempcache"] - ) - def test_minimal_dnf_command(self): self.assertEqual(rc.get_repoclosure_cmd(backend="dnf"), ["dnf", "repoclosure"]) @@ -23,37 +18,6 @@ class RepoclosureWrapperTestCase(helpers.BaseTestCase): self.assertEqual(str(ctx.exception), "Unknown repoclosure backend: rpm") - def test_multiple_arches(self): - self.assertEqual( - rc.get_repoclosure_cmd(arch=["x86_64", "i686", "noarch"]), - [ - "/usr/bin/repoclosure", - "--tempcache", - "--arch=x86_64", - "--arch=i686", - "--arch=noarch", - ], - ) - - def test_full_command(self): - repos = {"my-repo": "/mnt/koji/repo"} - lookaside = {"fedora": "http://kojipkgs.fp.o/repo"} - - cmd = rc.get_repoclosure_cmd(arch="x86_64", repos=repos, lookaside=lookaside) - self.assertEqual(cmd[0], "/usr/bin/repoclosure") - six.assertCountEqual( - self, - cmd[1:], - [ - "--tempcache", - "--arch=x86_64", - "--repofrompath=my-repo,file:///mnt/koji/repo", - "--repofrompath=fedora,http://kojipkgs.fp.o/repo", - "--repoid=my-repo", - "--lookaside=fedora", - ], - ) - def test_full_dnf_command(self): repos = {"my-repo": "/mnt/koji/repo"} lookaside = {"fedora": "http://kojipkgs.fp.o/repo"} @@ -103,44 +67,6 @@ class RepoclosureWrapperTestCase(helpers.BaseTestCase): ], ) - def test_expand_repo(self): - repos = { - "local": "/mnt/koji/repo", - "remote": "http://kojipkgs.fp.o/repo", - } - cmd = rc.get_repoclosure_cmd(repos=repos) - self.assertEqual(cmd[0], "/usr/bin/repoclosure") - six.assertCountEqual( - self, - cmd[1:], - [ - "--tempcache", - "--repofrompath=local,file:///mnt/koji/repo", - "--repofrompath=remote,http://kojipkgs.fp.o/repo", - "--repoid=local", - "--repoid=remote", - ], - ) - - def test_expand_lookaside(self): - repos = { - "local": "/mnt/koji/repo", - "remote": "http://kojipkgs.fp.o/repo", - } - cmd = rc.get_repoclosure_cmd(lookaside=repos) - self.assertEqual(cmd[0], "/usr/bin/repoclosure") - six.assertCountEqual( - self, - cmd[1:], - [ - "--tempcache", - "--repofrompath=local,file:///mnt/koji/repo", - "--repofrompath=remote,http://kojipkgs.fp.o/repo", - "--lookaside=local", - "--lookaside=remote", - ], - ) - class FusExtractorTestCase(helpers.PungiTestCase): def setUp(self): diff --git a/tests/test_test_phase.py b/tests/test_test_phase.py index 81d9e582..fb48d8be 100644 --- a/tests/test_test_phase.py +++ b/tests/test_test_phase.py @@ -6,20 +6,6 @@ import os import pungi.phases.test as test_phase from tests.helpers import DummyCompose, PungiTestCase, touch, FIXTURE_DIR -try: - import dnf # noqa: F401 - - HAS_DNF = True -except ImportError: - HAS_DNF = False - -try: - import yum # noqa: F401 - - HAS_YUM = True -except ImportError: - HAS_YUM = False - PAD = b"\0" * 100 UNBOOTABLE_ISO = (b"\0" * 0x8001) + b"CD001" + PAD