gather: Add langpacks in hybrid solver

Comps file specifies a pattern for some packages. If that package is
installed, all packages matching the pattern are added as well. This can
be added to fus as another pass.

Signed-off-by: Lubomír Sedlář <lsedlar@redhat.com>

pungi/phases/gather/methods/method_hybrid.py

@@ -17,6 +17,7 @@ from collections import defaultdict
 import os
 from kobo.shortcuts import run
 import kobo.rpmlib
+from fnmatch import fnmatch
 
 import pungi.phases.gather.method
 from pungi import Modulemd, multilib_dnf
@@ -29,6 +30,7 @@ from pungi.util import (
     temp_dir,
 )
 from pungi.wrappers import fus
+from pungi.wrappers.comps import CompsWrapper
 from pungi.wrappers.createrepo import CreaterepoWrapper
 
 from .method_nodeps import expand_groups
@@ -68,6 +70,11 @@ class GatherMethodHybrid(pungi.phases.gather.method.GatherMethodBase):
         super(GatherMethodHybrid, self).__init__(*args, **kwargs)
         self.package_maps = {}
         self.packages = {}
+        # Mapping from package name to set of langpack packages (stored as
+        # names).
+        self.langpacks = {}
+        # Set of packages for which we already added langpacks.
+        self.added_langpacks = set()
 
     def _get_pkg_map(self, arch):
         """Create a mapping from NEVRA to actual package object. This will be
@@ -100,6 +107,25 @@ class GatherMethodHybrid(pungi.phases.gather.method.GatherMethodBase):
             self._prepare_packages()
         return self.packages[nevra]
 
+    def prepare_langpacks(self, arch, variant):
+        if not self.compose.has_comps:
+            return
+        comps_file = self.compose.paths.work.comps(arch, variant, create_dir=False)
+        comps = CompsWrapper(comps_file)
+
+        for name, install in comps.get_langpacks().items():
+            # Replace %s with * for fnmatch.
+            install_match = install % "*"
+            self.langpacks[name] = set()
+            for pkg_arch in self.package_sets[arch].rpms_by_arch:
+                for pkg in self.package_sets[arch].rpms_by_arch[pkg_arch]:
+                    if not fnmatch(pkg.name, install_match):
+                        # Does not match the pattern, ignore...
+                        continue
+                    if pkg.name.endswith("-devel") or pkg.name.endswith("-static"):
+                        continue
+                    self.langpacks[name].add(pkg.name)
+
     def __call__(
         self,
         arch,
@@ -115,6 +141,8 @@ class GatherMethodHybrid(pungi.phases.gather.method.GatherMethodBase):
         self.valid_arches = get_valid_arches(arch, multilib=True)
         self.package_sets = package_sets
 
+        self.prepare_langpacks(arch, variant)
+
         self.multilib_methods = get_arch_variant_data(
             self.compose.conf, "multilib", arch, variant
         )
@@ -169,13 +197,19 @@ class GatherMethodHybrid(pungi.phases.gather.method.GatherMethodBase):
             run(cmd, logfile=logfile, show_cmd=True)
             output, output_modules = fus.parse_output(logfile)
             new_multilib = self.add_multilib(variant, arch, output, modular_rpms)
-            if not new_multilib:
+            if new_multilib:
-                # No new multilib packages were added, we're done.
+                input_packages.extend(
-                break
+                    _fmt_pkg(pkg_name, pkg_arch) for pkg_name, pkg_arch in new_multilib
+                )
+                continue
 
-            input_packages.extend(
+            new_langpacks = self.add_langpacks(output)
-                _fmt_pkg(pkg_name, pkg_arch) for pkg_name, pkg_arch in new_multilib
+            if new_langpacks:
-            )
+                input_packages.extend(new_langpacks)
+                continue
+
+            # Nothing new was added, we can stop now.
+            break
 
         return output, output_modules
 
@@ -212,6 +246,20 @@ class GatherMethodHybrid(pungi.phases.gather.method.GatherMethodBase):
 
         return sorted(added)
 
+    def add_langpacks(self, nvrs):
+        if not self.langpacks:
+            return set()
+        added = set()
+        for nvr, pkg_arch in nvrs:
+            name = nvr.rsplit("-", 2)[0]
+            if name in self.added_langpacks:
+                # This package is already processed.
+                continue
+            added.update(self.langpacks.get(name, []))
+            self.added_langpacks.add(name)
+
+        return sorted(added)
+
 
 def create_module_repo(compose, variant, arch):
     """Create repository with module metadata. There are no packages otherwise."""

tests/test_gather_method_hybrid.py

@@ -24,11 +24,12 @@ class NamedMock(mock.Mock):
 
 
 class TestMethodHybrid(helpers.PungiTestCase):
+    @mock.patch("pungi.phases.gather.methods.method_hybrid.CompsWrapper")
     @mock.patch("pungi.phases.gather.get_lookaside_repos")
     @mock.patch("pungi.phases.gather.methods.method_hybrid.expand_groups")
     @mock.patch("pungi.phases.gather.methods.method_hybrid.expand_packages")
     @mock.patch("pungi.phases.gather.methods.method_hybrid.create_module_repo")
-    def test_call_method(self, cmr, ep, eg, glr):
+    def test_call_method(self, cmr, ep, eg, glr, CW):
         compose = helpers.DummyCompose(self.topdir, {})
         cmr.return_value = (mock.Mock(), mock.Mock())
         m = hybrid.GatherMethodHybrid(compose)
@@ -42,6 +43,7 @@ class TestMethodHybrid(helpers.PungiTestCase):
             sourcerpm=None,
             file_path=None,
         )
+        CW.return_value.get_langpacks.return_value = {"glibc": "glibc-langpack-%s"}
         eg.return_value = ["foo", "bar"]
         package_sets = {"x86_64": mock.Mock(rpms_by_arch={"x86_64": [pkg]})}
         arch = "x86_64"
@@ -71,6 +73,62 @@ class TestMethodHybrid(helpers.PungiTestCase):
             eg.call_args_list,
             [mock.call(compose, arch, variant, ["standard"], set_pkg_arch=False)],
         )
+        print(CW.mock_calls)
+        self.assertEqual(
+            CW.mock_calls,
+            [
+                mock.call(
+                    os.path.join(
+                        self.topdir, "work/x86_64/comps/comps-Server.x86_64.xml"
+                    )
+                ),
+                mock.call().get_langpacks(),
+            ],
+        )
+
+    @mock.patch("pungi.phases.gather.methods.method_hybrid.CompsWrapper")
+    def test_prepare_langpacks(self, CW):
+        compose = helpers.DummyCompose(self.topdir, {})
+        CW.return_value.get_langpacks.return_value = {"foo": "foo-%s"}
+        m = hybrid.GatherMethodHybrid(compose)
+        m.package_sets = {
+            "x86_64": mock.Mock(
+                rpms_by_arch={
+                    "x86_64": [
+                        MockPkg(
+                            name="foo",
+                            version="1",
+                            release="2",
+                            arch="x86_64",
+                            epoch=0,
+                            sourcerpm=None,
+                            file_path=None,
+                        ),
+                        MockPkg(
+                            name="foo-en",
+                            version="1",
+                            release="2",
+                            arch="x86_64",
+                            epoch=0,
+                            sourcerpm=None,
+                            file_path=None,
+                        ),
+                        MockPkg(
+                            name="foo-devel",
+                            version="1",
+                            release="2",
+                            arch="x86_64",
+                            epoch=0,
+                            sourcerpm=None,
+                            file_path=None,
+                        ),
+                    ]
+                }
+            )
+        }
+        m.prepare_langpacks("x86_64", compose.variants["Server"])
+
+        self.assertEqual(m.langpacks, {"foo": set(["foo-en"])})
 
 
 class MockModule(object):
@@ -212,7 +270,7 @@ class TestRunSolver(HelperMixin, helpers.PungiTestCase):
         )
 
     def test_with_modules(self, run, gc, po):
-        self.compose.has_comps = None
+        self.compose.has_comps = False
         self.compose.variants["Server"].arch_mmds["x86_64"] = {
             "mod:master": mock.Mock(
                 peek_name=mock.Mock(return_value="mod"),
@@ -270,6 +328,47 @@ class TestRunSolver(HelperMixin, helpers.PungiTestCase):
             [mock.call("x86_64", [self._repo("repo")], [], ["pkg"], [], platform=None)],
         )
 
+    def test_with_langpacks(self, run, gc, po):
+        self.phase.langpacks = {"pkg": set(["pkg-en"])}
+        final = ([("pkg-1.0-1", "x86_64"), ("pkg-en-1.0-1", "noarch")], set())
+        po.side_effect = [([("pkg-1.0-1", "x86_64")], set()), final]
+
+        res = self.phase.run_solver(
+            self.compose.variants["Server"],
+            "x86_64",
+            [("pkg", None)],
+            platform=None,
+            modular_rpms=[],
+        )
+
+        self.assertEqual(res, final)
+        self.assertEqual(
+            po.call_args_list, [mock.call(self.logfile1), mock.call(self.logfile2)]
+        )
+        self.assertEqual(
+            run.call_args_list,
+            [
+                mock.call(gc.return_value, logfile=self.logfile1, show_cmd=True),
+                mock.call(gc.return_value, logfile=self.logfile2, show_cmd=True),
+            ],
+        )
+        self.assertEqual(
+            gc.call_args_list,
+            [
+                mock.call(
+                    "x86_64", [self._repo("repo")], [], ["pkg"], [], platform=None
+                ),
+                mock.call(
+                    "x86_64",
+                    [self._repo("repo")],
+                    [],
+                    ["pkg", "pkg-en"],
+                    [],
+                    platform=None,
+                ),
+            ],
+        )
+
     @mock.patch("pungi.phases.gather.methods.method_hybrid.cr")
     def test_multilib_devel(self, cr, run, gc, po):
         self.phase.arch = "x86_64"