From e10b893adc9a0cf75a439d975ca6fb6a5685e8ee Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lubom=C3=ADr=20Sedl=C3=A1=C5=99?= Date: Tue, 23 Oct 2018 08:29:45 +0200 Subject: [PATCH] gather: Expand wildcards in Pungi MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit When the configuration lists `*` in `additional_packages`, it has a special meaning. If it's passed to fus directly, it will use it to match all modules and RPMs that are not masked by a package available in some default stream. Neither is good. We don't want it to match modules, and we want even the masked packages. The fix is to expand the wildcard to a list of NVRs and give that to fus. It should include the package even if it is masked. Signed-off-by: Lubomír Sedlář --- pungi/phases/gather/methods/method_hybrid.py | 57 ++++++++++++++++---- tests/test_gather_method_hybrid.py | 5 +- 2 files changed, 51 insertions(+), 11 deletions(-) diff --git a/pungi/phases/gather/methods/method_hybrid.py b/pungi/phases/gather/methods/method_hybrid.py index 7c4ab264..242c62b5 100644 --- a/pungi/phases/gather/methods/method_hybrid.py +++ b/pungi/phases/gather/methods/method_hybrid.py @@ -76,6 +76,8 @@ class GatherMethodHybrid(pungi.phases.gather.method.GatherMethodBase): self.langpacks = {} # Set of packages for which we already added langpacks. self.added_langpacks = set() + # Set of NEVRAs of modular packages + self.modular_packages = set() def _get_pkg_map(self, arch): """Create a mapping from NEVRA to actual package object. This will be @@ -108,19 +110,24 @@ class GatherMethodHybrid(pungi.phases.gather.method.GatherMethodBase): self._prepare_packages() return self.packages[nevra] - def expand_list(self, arch, patterns): + def expand_list(self, patterns): """Given a list of globs, create a list of package names matching any of the pattern. """ expanded = set() - for pkg_arch in self.package_sets[arch].rpms_by_arch: - for pkg in self.package_sets[arch].rpms_by_arch[pkg_arch]: + for pkg_arch in self.package_sets[self.arch].rpms_by_arch: + for pkg in self.package_sets[self.arch].rpms_by_arch[pkg_arch]: for pattern in patterns: if fnmatch(pkg.name, pattern): - expanded.add(pkg.name) + expanded.add(pkg) break return expanded + def prepare_modular_packages(self): + for var in self.compose.all_variants.values(): + for mmd in var.arch_mmds.get(self.arch, {}).values(): + self.modular_packages.update(mmd.get_rpm_artifacts().dup()) + def prepare_langpacks(self, arch, variant): if not self.compose.has_comps: return @@ -159,14 +166,15 @@ class GatherMethodHybrid(pungi.phases.gather.method.GatherMethodBase): self.package_sets = package_sets self.prepare_langpacks(arch, variant) + self.prepare_modular_packages() self.multilib_methods = get_arch_variant_data( self.compose.conf, "multilib", arch, variant ) self.multilib = multilib_dnf.Multilib( self.multilib_methods, - self.expand_list(arch, multilib_blacklist), - self.expand_list(arch, multilib_whitelist), + set(p.name for p in self.expand_list(multilib_blacklist)), + set(p.name for p in self.expand_list(multilib_whitelist)), ) platform = create_module_repo(self.compose, variant, arch) @@ -200,9 +208,9 @@ class GatherMethodHybrid(pungi.phases.gather.method.GatherMethodBase): for mmd in variant.arch_mmds[arch].values(): modules.append("%s:%s" % (mmd.peek_name(), mmd.peek_stream())) - input_packages = [ - _fmt_pkg(pkg_name, pkg_arch) for pkg_name, pkg_arch in packages - ] + input_packages = [] + for pkg_name, pkg_arch in packages: + input_packages.extend(self._expand_wildcard(pkg_name, pkg_arch)) step = 0 @@ -292,6 +300,37 @@ class GatherMethodHybrid(pungi.phases.gather.method.GatherMethodBase): return sorted(added) + def _expand_wildcard(self, pkg_name, pkg_arch): + if "*" not in pkg_name: + return [_fmt_pkg(pkg_name, pkg_arch)] + + packages = [] + + for pkg in self.expand_list([pkg_name]): + if pkg_is_debug(pkg): + # No debuginfo + continue + + if pkg_arch: + if pkg_arch != pkg.arch: + # Arch is specified and does not match, skip the package. + continue + else: + if pkg.arch not in ("noarch", self.arch): + # No arch specified and package does not match + continue + + strict_nevra = "%s-%s:%s-%s.%s" % ( + pkg.name, pkg.epoch or "0", pkg.version, pkg.release, pkg.arch + ) + if strict_nevra in self.modular_packages: + # Wildcards should not match modular packages. + continue + + packages.append(_fmt_nevra(pkg, pkg.arch)) + + return packages + def get_lookaside_modules(lookasides): """Get list of NSVC of all modules in all lookaside repos.""" diff --git a/tests/test_gather_method_hybrid.py b/tests/test_gather_method_hybrid.py index 70463637..66e5492f 100644 --- a/tests/test_gather_method_hybrid.py +++ b/tests/test_gather_method_hybrid.py @@ -140,6 +140,7 @@ class TestMethodHybrid(helpers.PungiTestCase): def test_expand_list(self): compose = helpers.DummyCompose(self.topdir, {}) m = hybrid.GatherMethodHybrid(compose) + m.arch = "x86_64" m.package_sets = { "x86_64": mock.Mock( rpms_by_arch={ @@ -175,9 +176,9 @@ class TestMethodHybrid(helpers.PungiTestCase): } ) } - expanded = m.expand_list("x86_64", ["foo*"]) + expanded = m.expand_list(["foo*"]) - self.assertItemsEqual(expanded, ["foo", "foo-en"]) + self.assertItemsEqual([p.name for p in expanded], ["foo", "foo-en"]) class MockModule(object):