diff --git a/pungi/phases/gather/methods/method_hybrid.py b/pungi/phases/gather/methods/method_hybrid.py index 7161f072..058d43a6 100644 --- a/pungi/phases/gather/methods/method_hybrid.py +++ b/pungi/phases/gather/methods/method_hybrid.py @@ -17,6 +17,7 @@ from collections import defaultdict import os from kobo.shortcuts import run import kobo.rpmlib +from fnmatch import fnmatch import pungi.phases.gather.method from pungi import Modulemd, multilib_dnf @@ -29,6 +30,7 @@ from pungi.util import ( temp_dir, ) from pungi.wrappers import fus +from pungi.wrappers.comps import CompsWrapper from pungi.wrappers.createrepo import CreaterepoWrapper from .method_nodeps import expand_groups @@ -68,6 +70,11 @@ class GatherMethodHybrid(pungi.phases.gather.method.GatherMethodBase): super(GatherMethodHybrid, self).__init__(*args, **kwargs) self.package_maps = {} self.packages = {} + # Mapping from package name to set of langpack packages (stored as + # names). + self.langpacks = {} + # Set of packages for which we already added langpacks. + self.added_langpacks = set() def _get_pkg_map(self, arch): """Create a mapping from NEVRA to actual package object. This will be @@ -100,6 +107,25 @@ class GatherMethodHybrid(pungi.phases.gather.method.GatherMethodBase): self._prepare_packages() return self.packages[nevra] + def prepare_langpacks(self, arch, variant): + if not self.compose.has_comps: + return + comps_file = self.compose.paths.work.comps(arch, variant, create_dir=False) + comps = CompsWrapper(comps_file) + + for name, install in comps.get_langpacks().items(): + # Replace %s with * for fnmatch. + install_match = install % "*" + self.langpacks[name] = set() + for pkg_arch in self.package_sets[arch].rpms_by_arch: + for pkg in self.package_sets[arch].rpms_by_arch[pkg_arch]: + if not fnmatch(pkg.name, install_match): + # Does not match the pattern, ignore... + continue + if pkg.name.endswith("-devel") or pkg.name.endswith("-static"): + continue + self.langpacks[name].add(pkg.name) + def __call__( self, arch, @@ -115,6 +141,8 @@ class GatherMethodHybrid(pungi.phases.gather.method.GatherMethodBase): self.valid_arches = get_valid_arches(arch, multilib=True) self.package_sets = package_sets + self.prepare_langpacks(arch, variant) + self.multilib_methods = get_arch_variant_data( self.compose.conf, "multilib", arch, variant ) @@ -169,13 +197,19 @@ class GatherMethodHybrid(pungi.phases.gather.method.GatherMethodBase): run(cmd, logfile=logfile, show_cmd=True) output, output_modules = fus.parse_output(logfile) new_multilib = self.add_multilib(variant, arch, output, modular_rpms) - if not new_multilib: - # No new multilib packages were added, we're done. - break + if new_multilib: + input_packages.extend( + _fmt_pkg(pkg_name, pkg_arch) for pkg_name, pkg_arch in new_multilib + ) + continue - input_packages.extend( - _fmt_pkg(pkg_name, pkg_arch) for pkg_name, pkg_arch in new_multilib - ) + new_langpacks = self.add_langpacks(output) + if new_langpacks: + input_packages.extend(new_langpacks) + continue + + # Nothing new was added, we can stop now. + break return output, output_modules @@ -212,6 +246,20 @@ class GatherMethodHybrid(pungi.phases.gather.method.GatherMethodBase): return sorted(added) + def add_langpacks(self, nvrs): + if not self.langpacks: + return set() + added = set() + for nvr, pkg_arch in nvrs: + name = nvr.rsplit("-", 2)[0] + if name in self.added_langpacks: + # This package is already processed. + continue + added.update(self.langpacks.get(name, [])) + self.added_langpacks.add(name) + + return sorted(added) + def create_module_repo(compose, variant, arch): """Create repository with module metadata. There are no packages otherwise.""" diff --git a/tests/test_gather_method_hybrid.py b/tests/test_gather_method_hybrid.py index 46f2574a..0b508c8b 100644 --- a/tests/test_gather_method_hybrid.py +++ b/tests/test_gather_method_hybrid.py @@ -24,11 +24,12 @@ class NamedMock(mock.Mock): class TestMethodHybrid(helpers.PungiTestCase): + @mock.patch("pungi.phases.gather.methods.method_hybrid.CompsWrapper") @mock.patch("pungi.phases.gather.get_lookaside_repos") @mock.patch("pungi.phases.gather.methods.method_hybrid.expand_groups") @mock.patch("pungi.phases.gather.methods.method_hybrid.expand_packages") @mock.patch("pungi.phases.gather.methods.method_hybrid.create_module_repo") - def test_call_method(self, cmr, ep, eg, glr): + def test_call_method(self, cmr, ep, eg, glr, CW): compose = helpers.DummyCompose(self.topdir, {}) cmr.return_value = (mock.Mock(), mock.Mock()) m = hybrid.GatherMethodHybrid(compose) @@ -42,6 +43,7 @@ class TestMethodHybrid(helpers.PungiTestCase): sourcerpm=None, file_path=None, ) + CW.return_value.get_langpacks.return_value = {"glibc": "glibc-langpack-%s"} eg.return_value = ["foo", "bar"] package_sets = {"x86_64": mock.Mock(rpms_by_arch={"x86_64": [pkg]})} arch = "x86_64" @@ -71,6 +73,62 @@ class TestMethodHybrid(helpers.PungiTestCase): eg.call_args_list, [mock.call(compose, arch, variant, ["standard"], set_pkg_arch=False)], ) + print(CW.mock_calls) + self.assertEqual( + CW.mock_calls, + [ + mock.call( + os.path.join( + self.topdir, "work/x86_64/comps/comps-Server.x86_64.xml" + ) + ), + mock.call().get_langpacks(), + ], + ) + + @mock.patch("pungi.phases.gather.methods.method_hybrid.CompsWrapper") + def test_prepare_langpacks(self, CW): + compose = helpers.DummyCompose(self.topdir, {}) + CW.return_value.get_langpacks.return_value = {"foo": "foo-%s"} + m = hybrid.GatherMethodHybrid(compose) + m.package_sets = { + "x86_64": mock.Mock( + rpms_by_arch={ + "x86_64": [ + MockPkg( + name="foo", + version="1", + release="2", + arch="x86_64", + epoch=0, + sourcerpm=None, + file_path=None, + ), + MockPkg( + name="foo-en", + version="1", + release="2", + arch="x86_64", + epoch=0, + sourcerpm=None, + file_path=None, + ), + MockPkg( + name="foo-devel", + version="1", + release="2", + arch="x86_64", + epoch=0, + sourcerpm=None, + file_path=None, + ), + ] + } + ) + } + m.prepare_langpacks("x86_64", compose.variants["Server"]) + + self.assertEqual(m.langpacks, {"foo": set(["foo-en"])}) class MockModule(object): @@ -212,7 +270,7 @@ class TestRunSolver(HelperMixin, helpers.PungiTestCase): ) def test_with_modules(self, run, gc, po): - self.compose.has_comps = None + self.compose.has_comps = False self.compose.variants["Server"].arch_mmds["x86_64"] = { "mod:master": mock.Mock( peek_name=mock.Mock(return_value="mod"), @@ -270,6 +328,47 @@ class TestRunSolver(HelperMixin, helpers.PungiTestCase): [mock.call("x86_64", [self._repo("repo")], [], ["pkg"], [], platform=None)], ) + def test_with_langpacks(self, run, gc, po): + self.phase.langpacks = {"pkg": set(["pkg-en"])} + final = ([("pkg-1.0-1", "x86_64"), ("pkg-en-1.0-1", "noarch")], set()) + po.side_effect = [([("pkg-1.0-1", "x86_64")], set()), final] + + res = self.phase.run_solver( + self.compose.variants["Server"], + "x86_64", + [("pkg", None)], + platform=None, + modular_rpms=[], + ) + + self.assertEqual(res, final) + self.assertEqual( + po.call_args_list, [mock.call(self.logfile1), mock.call(self.logfile2)] + ) + self.assertEqual( + run.call_args_list, + [ + mock.call(gc.return_value, logfile=self.logfile1, show_cmd=True), + mock.call(gc.return_value, logfile=self.logfile2, show_cmd=True), + ], + ) + self.assertEqual( + gc.call_args_list, + [ + mock.call( + "x86_64", [self._repo("repo")], [], ["pkg"], [], platform=None + ), + mock.call( + "x86_64", + [self._repo("repo")], + [], + ["pkg", "pkg-en"], + [], + platform=None, + ), + ], + ) + @mock.patch("pungi.phases.gather.methods.method_hybrid.cr") def test_multilib_devel(self, cr, run, gc, po): self.phase.arch = "x86_64"