hybrid: Remove modules not listed by fus

It's possible we ask to include module X, but it's in lookaside and as
such it should not be in the output. Therefore we need to remove it from
the variant.

Signed-off-by: Lubomír Sedlář <lsedlar@redhat.com>

pungi/phases/gather/methods/method_hybrid.py

@@ -171,7 +171,8 @@ class GatherMethodHybrid(pungi.phases.gather.method.GatherMethodBase):
             expand_groups(self.compose, arch, variant, groups, set_pkg_arch=False)
         )
 
-        nvrs = self.run_solver(variant, arch, packages, platform)
+        nvrs, out_modules = self.run_solver(variant, arch, packages, platform)
+        filter_modules(variant, arch, out_modules)
         return expand_packages(
             self._get_pkg_map(arch),
             variant.arch_mmds.get(arch, {}),
@@ -214,7 +215,7 @@ class GatherMethodHybrid(pungi.phases.gather.method.GatherMethodBase):
             env = os.environ.copy()
             env["G_MESSAGES_PREFIXED"] = ""
             run(cmd, logfile=logfile, show_cmd=True, env=env)
-            output = fus.parse_output(logfile)
+            output, out_modules = fus.parse_output(logfile)
             new_multilib = self.add_multilib(variant, arch, output)
             if new_multilib:
                 input_packages.extend(
@@ -230,7 +231,7 @@ class GatherMethodHybrid(pungi.phases.gather.method.GatherMethodBase):
             # Nothing new was added, we can stop now.
             break
 
-        return output
+        return output, out_modules
 
     def add_multilib(self, variant, arch, nvrs):
         added = set()
@@ -452,3 +453,13 @@ def expand_packages(nevra_to_pkg, variant_modules, lookasides, nvrs):
                 debuginfo.add(pkg.file_path)
 
     return _mk_pkg_map(_make_result(rpms), _make_result(srpms), _make_result(debuginfo))
+
+
+def filter_modules(variant, arch, nsvcs_to_keep):
+    """Remove any arch-specific module metadata from the module if it's not
+    listed in the list to keep. This will ultimately cause the module to not be
+    included in the final repodata and module metadata.
+    """
+    for nsvc in list(variant.arch_mmds.get(arch, {}).keys()):
+        if nsvc not in nsvcs_to_keep:
+            del variant.arch_mmds[arch][nsvc]

pungi/wrappers/fus.py

@@ -70,6 +70,7 @@ def parse_output(output):
     (NVR, arch, flags) and a set of module NSVCs.
     """
     packages = set()
+    modules = set()
     with open(output) as f:
         for line in f:
             if " " in line or "@" not in line:
@@ -82,4 +83,7 @@ def parse_output(output):
                     flags.add("modular")
                     name = name[1:]
                 packages.add((name, arch, frozenset(flags)))
-    return packages
+            else:
+                name, arch = nevra.rsplit(".", 1)
+                modules.add(name.split(":", 1)[1])
+    return packages, modules

tests/test_fus_wrapper.py

@@ -78,21 +78,30 @@ class TestParseOutput(unittest.TestCase):
 
     def test_skips_debug_line(self):
         touch(self.file, "debug line\n")
-        packages = fus.parse_output(self.file)
+        packages, modules = fus.parse_output(self.file)
         self.assertItemsEqual(packages, [])
+        self.assertItemsEqual(modules, [])
 
     def test_separates_arch(self):
         touch(self.file, "pkg-1.0-1.x86_64@repo-0\npkg-1.0-1.i686@repo-0\n")
-        packages = fus.parse_output(self.file)
+        packages, modules = fus.parse_output(self.file)
         self.assertItemsEqual(
             packages,
             [("pkg-1.0-1", "x86_64", frozenset()), ("pkg-1.0-1", "i686", frozenset())],
         )
+        self.assertItemsEqual(modules, [])
 
     def test_marks_modular(self):
         touch(self.file, "*pkg-1.0-1.x86_64@repo-0\n")
-        packages = fus.parse_output(self.file)
+        packages, modules = fus.parse_output(self.file)
         self.assertItemsEqual(
             packages,
             [("pkg-1.0-1", "x86_64", frozenset(["modular"]))],
         )
+        self.assertItemsEqual(modules, [])
+
+    def test_extracts_modules(self):
+        touch(self.file, "module:mod:master:20181003:cafebeef.x86_64@repo-0\n")
+        packages, modules = fus.parse_output(self.file)
+        self.assertItemsEqual(packages, [])
+        self.assertItemsEqual(modules, ["mod:master:20181003:cafebeef"])

tests/test_gather_method_hybrid.py

@@ -32,7 +32,7 @@ class TestMethodHybrid(helpers.PungiTestCase):
     def test_call_method(self, cmr, ep, eg, glr, CW):
         compose = helpers.DummyCompose(self.topdir, {})
         m = hybrid.GatherMethodHybrid(compose)
-        m.run_solver = mock.Mock()
+        m.run_solver = mock.Mock(return_value=(mock.Mock(), mock.Mock()))
         pkg = MockPkg(
             name="pkg",
             version="1",
@@ -63,7 +63,7 @@ class TestMethodHybrid(helpers.PungiTestCase):
                     {"pkg-3:1-2.x86_64": pkg},
                     {},
                     glr.return_value,
-                    m.run_solver.return_value,
+                    m.run_solver.return_value[0],
                 )
             ],
         )
@@ -435,7 +435,7 @@ class TestRunSolver(HelperMixin, helpers.PungiTestCase):
     def test_with_langpacks(self, run, gc, po):
         self.phase.langpacks = {"pkg": set(["pkg-en"])}
         final = [("pkg-1.0-1", "x86_64", []), ("pkg-en-1.0-1", "noarch", [])]
-        po.side_effect = [[("pkg-1.0-1", "x86_64", [])], final]
+        po.side_effect = [([("pkg-1.0-1", "x86_64", [])], set()), (final, [])]
 
         res = self.phase.run_solver(
             self.compose.variants["Server"],
@@ -444,7 +444,7 @@ class TestRunSolver(HelperMixin, helpers.PungiTestCase):
             platform=None,
         )
 
-        self.assertEqual(res, final)
+        self.assertEqual(res, (final, []))
         self.assertEqual(
             po.call_args_list, [mock.call(self.logfile1), mock.call(self.logfile2)]
         )
@@ -500,7 +500,8 @@ class TestRunSolver(HelperMixin, helpers.PungiTestCase):
             ("pkg-devel-1.0-1", "i686", []),
         ]
         po.side_effect = [
-            [("pkg-devel-1.0-1", "x86_64", []), ("foo-1.0-1", "x86_64", [])], final
+            ([("pkg-devel-1.0-1", "x86_64", []), ("foo-1.0-1", "x86_64", [])], []),
+            (final, []),
         ]
 
         res = self.phase.run_solver(
@@ -510,7 +511,7 @@ class TestRunSolver(HelperMixin, helpers.PungiTestCase):
             platform=None,
         )
 
-        self.assertEqual(res, final)
+        self.assertEqual(res, (final, []))
         self.assertEqual(
             po.call_args_list, [mock.call(self.logfile1), mock.call(self.logfile2)]
         )
@@ -597,7 +598,8 @@ class TestRunSolver(HelperMixin, helpers.PungiTestCase):
             ("foo-1.0-1", "i686", []),
         ]
         po.side_effect = [
-            [("pkg-devel-1.0-1", "x86_64", []), ("foo-1.0-1", "x86_64", [])], final
+            ([("pkg-devel-1.0-1", "x86_64", []), ("foo-1.0-1", "x86_64", [])], []),
+            (final, []),
         ]
 
         res = self.phase.run_solver(
@@ -607,7 +609,7 @@ class TestRunSolver(HelperMixin, helpers.PungiTestCase):
             platform=None,
         )
 
-        self.assertEqual(res, final)
+        self.assertEqual(res, (final, []))
         self.assertEqual(
             po.call_args_list, [mock.call(self.logfile1), mock.call(self.logfile2)]
         )
@@ -801,3 +803,17 @@ class TestExpandPackages(helpers.PungiTestCase):
         )
 
         self.assertEqual(res, {"rpm": [], "srpm": [], "debuginfo": []})
+
+
+class TestFilterModules(helpers.PungiTestCase):
+    def test_remove_one(self):
+        self.compose = helpers.DummyCompose(self.topdir, {})
+        self.variant = self.compose.variants["Server"]
+        self.variant.arch_mmds["x86_64"] = {
+            "mod:1": MockModule("mod", platform="f29"),
+            "mod:2": MockModule("mod", platform="f30"),
+        }
+
+        hybrid.filter_modules(self.variant, "x86_64", ["mod:1"])
+
+        self.assertItemsEqual(self.variant.arch_mmds["x86_64"].keys(), ["mod:1"])