pungi/phases/createrepo.py

@@ -31,7 +31,7 @@ from kobo.shortcuts import run, relative_path
 from ..wrappers.scm import get_dir_from_scm
 from ..wrappers.createrepo import CreaterepoWrapper
 from .base import PhaseBase
-from ..util import get_arch_variant_data, temp_dir
+from ..util import get_arch_variant_data, temp_dir, read_single_module_stream_from_file
 from ..module_util import Modulemd, collect_module_defaults
 
 import productmd.rpms
@@ -268,7 +268,7 @@ def create_variant_repo(
             compose.log_debug("Adding extra modulemd for %s.%s", variant.uid, arch)
             dirname = compose.paths.work.tmp_dir(variant=variant, create_dir=False)
             for filepath in glob.glob(os.path.join(dirname, arch) + "/*.yaml"):
-                module_stream = Modulemd.ModuleStream.read_file(filepath, strict=True)
+                module_stream = read_single_module_stream_from_file(filepath)
                 if not mod_index.add_module_stream(module_stream):
                     raise RuntimeError(
                         "Failed parsing modulemd data from %s" % filepath

pungi/phases/pkgset/sources/source_koji.py

@@ -29,7 +29,13 @@ from pungi.wrappers.comps import CompsWrapper
 from pungi.wrappers.mbs import MBSWrapper
 import pungi.phases.pkgset.pkgsets
 from pungi.arch import getBaseArch
-from pungi.util import retry, get_arch_variant_data, get_variant_data
+from pungi.util import (
+    retry,
+    get_arch_variant_data,
+    get_variant_data,
+    read_single_module_stream_from_file,
+    read_single_module_stream_from_string,
+)
 from pungi.module_util import Modulemd
 
 from pungi.phases.pkgset.common import MaterializedPackageSet, get_all_arches
@@ -258,17 +264,12 @@ def _add_module_to_variant(
             compose.log_debug("Module %s is filtered from %s.%s", nsvc, variant, arch)
             continue
 
-        try:
+        mod_stream = read_single_module_stream_from_file(
-            mmd = Modulemd.ModuleStream.read_file(
+            mmds["modulemd.%s.txt" % arch], compose, arch, build
-                mmds["modulemd.%s.txt" % arch], strict=True
+        )
-            )
+        if mod_stream:
-            variant.arch_mmds.setdefault(arch, {})[nsvc] = mmd
             added = True
-        except KeyError:
+        variant.arch_mmds.setdefault(arch, {})[nsvc] = mod_stream
-            # There is no modulemd for this arch. This could mean an arch was
-            # added to the compose after the module was built. We don't want to
-            # process this, let's skip this module.
-            pass
 
     if not added:
         # The module is filtered on all arches of this variant.
@@ -348,9 +349,7 @@ def _add_scratch_modules_to_variant(
         tag_to_mmd.setdefault(tag, {})
         for arch in variant.arches:
             try:
-                mmd = Modulemd.ModuleStream.read_string(
+                mmd = read_single_module_stream_from_string(final_modulemd[arch])
-                    final_modulemd[arch], strict=True
-                )
                 variant.arch_mmds.setdefault(arch, {})[nsvc] = mmd
             except KeyError:
                 continue

pungi/util.py

@@ -34,6 +34,7 @@ import kobo.conf
 from kobo.shortcuts import run, force_list
 from kobo.threads import WorkerThread, ThreadPool
 from productmd.common import get_major_version
+from pungi.module_util import Modulemd
 
 # Patterns that match all names of debuginfo packages
 DEBUG_PATTERNS = ["*-debuginfo", "*-debuginfo-*", "*-debugsource"]
@@ -1034,6 +1035,46 @@ def load_config(file_path, defaults={}):
     return conf
 
 
+def _read_single_module_stream(
+    file_or_string, compose=None, arch=None, build=None, is_file=True
+):
+    try:
+        mod_index = Modulemd.ModuleIndex.new()
+        if is_file:
+            mod_index.update_from_file(file_or_string, True)
+        else:
+            mod_index.update_from_string(file_or_string, True)
+        mod_names = mod_index.get_module_names()
+        emit_warning = False
+        if len(mod_names) > 1:
+            emit_warning = True
+        mod_streams = mod_index.get_module(mod_names[0]).get_all_streams()
+        if len(mod_streams) > 1:
+            emit_warning = True
+        if emit_warning and compose:
+            compose.log_warning(
+                "Multiple modules/streams for arch: %s. Build: %s. "
+                "Processing first module/stream only.",
+                arch,
+                build,
+            )
+        return mod_streams[0]
+    except (KeyError, IndexError):
+        # There is no modulemd for this arch. This could mean an arch was
+        # added to the compose after the module was built. We don't want to
+        # process this, let's skip this module.
+        if compose:
+            compose.log_info("Skipping arch: %s. Build: %s", arch, build)
+
+
+def read_single_module_stream_from_file(*args, **kwargs):
+    return _read_single_module_stream(*args, is_file=True, **kwargs)
+
+
+def read_single_module_stream_from_string(*args, **kwargs):
+    return _read_single_module_stream(*args, is_file=False, **kwargs)
+
+
 @contextlib.contextmanager
 def as_local_file(url):
     """If URL points to a file over HTTP, the file will be downloaded locally

tests/fixtures/mmds/m1.x86_64.txt

@@ -0,0 +1,20 @@
+---
+document: modulemd
+version: 2
+data:
+  name: m1
+  stream: latest
+  version: 20190101
+  context: cafe
+  arch: x86_64
+  summary: Dummy module
+  description: Dummy module
+  license:
+    module:
+    - Beerware
+    content:
+    - Beerware
+  artifacts:
+    rpms:
+      - foobar-0:1.0-1.noarch
+...

tests/fixtures/mmds/modulemd.armv7hl.txt

@@ -0,0 +1,20 @@
+---
+document: modulemd
+version: 2
+data:
+  name: module
+  stream: master
+  version: 20190318
+  context: abcdef
+  arch: armhfp
+  summary: Dummy module
+  description: Dummy module
+  license:
+    module:
+    - Beerware
+    content:
+    - Beerware
+  artifacts:
+    rpms:
+      - foobar-0:1.0-1.noarch
+...

tests/fixtures/mmds/modulemd.x86_64.txt

@@ -0,0 +1,20 @@
+---
+document: modulemd
+version: 2
+data:
+  name: module
+  stream: master
+  version: 20190318
+  context: abcdef
+  arch: x86_64
+  summary: Dummy module
+  description: Dummy module
+  license:
+    module:
+    - Beerware
+    content:
+    - Beerware
+  artifacts:
+    rpms:
+      - foobar-0:1.0-1.noarch
+...

tests/fixtures/mmds/scratch-module.x86_64.txt

@@ -0,0 +1,20 @@
+---
+document: modulemd
+version: 2
+data:
+  name: scratch-module
+  stream: master
+  version: 20200710
+  context: abcdef
+  arch: x86_64
+  summary: Dummy module
+  description: Dummy module
+  license:
+    module:
+    - Beerware
+    content:
+    - Beerware
+  artifacts:
+    rpms:
+      - foobar-0:1.0-1.noarch
+...

tests/test_pkgset_source_koji.py

@@ -14,7 +14,9 @@ except ImportError:
 from pungi.phases.pkgset.sources import source_koji
 from tests import helpers
 from pungi.module_util import Modulemd
+from pungi.util import read_single_module_stream_from_file
 
+MMDS_DIR = os.path.join(helpers.FIXTURE_DIR, "mmds")
 EVENT_INFO = {"id": 15681980, "ts": 1460956382.81936}
 TAG_INFO = {
     "maven_support": False,
@@ -672,24 +674,12 @@ class TestFilterByWhitelist(unittest.TestCase):
         self.assertEqual(expected, set())
 
 
-class MockModule(object):
-    def __init__(self, path, strict=True):
-        self.path = path
-
-    def __repr__(self):
-        return "MockModule(%r)" % self.path
-
-    def __eq__(self, other):
-        return self.path == other.path
-
-
-@mock.patch("pungi.module_util.Modulemd.ModuleStream.read_file", new=MockModule)
 @unittest.skipIf(Modulemd is None, "Skipping tests, no module support")
 class TestAddModuleToVariant(helpers.PungiTestCase):
     def setUp(self):
         super(TestAddModuleToVariant, self).setUp()
         self.koji = mock.Mock()
-        self.koji.koji_module.pathinfo.typedir.return_value = "/koji"
+        self.koji.koji_module.pathinfo.typedir.return_value = MMDS_DIR
         files = ["modulemd.x86_64.txt", "modulemd.armv7hl.txt", "modulemd.txt"]
         self.koji.koji_proxy.listArchives.return_value = [
             {"btype": "module", "filename": fname} for fname in files
@@ -713,50 +703,35 @@ class TestAddModuleToVariant(helpers.PungiTestCase):
 
         source_koji._add_module_to_variant(self.koji, variant, self.buildinfo)
 
-        self.assertEqual(
+        mod1 = variant.arch_mmds["armhfp"]["module:master:20190318:abcdef"]
-            variant.arch_mmds,
+        self.assertEqual(mod1.get_NSVCA(), "module:master:20190318:abcdef:armhfp")
-            {
+        mod2 = variant.arch_mmds["x86_64"]["module:master:20190318:abcdef"]
-                "armhfp": {
+        self.assertEqual(mod2.get_NSVCA(), "module:master:20190318:abcdef:x86_64")
-                    "module:master:20190318:abcdef": MockModule(
+        self.assertEqual(len(variant.arch_mmds), 2)
-                        "/koji/modulemd.armv7hl.txt"
-                    ),
-                },
-                "x86_64": {
-                    "module:master:20190318:abcdef": MockModule(
-                        "/koji/modulemd.x86_64.txt"
-                    ),
-                },
-            },
-        )
         self.assertEqual(variant.modules, [])
 
     def test_adding_module_to_existing(self):
         variant = mock.Mock(
             arches=["armhfp", "x86_64"],
             arch_mmds={
-                "x86_64": {"m1:latest:20190101:cafe": MockModule("/koji/m1.x86_64.txt")}
+                "x86_64": {
+                    "m1:latest:20190101:cafe": read_single_module_stream_from_file(
+                        os.path.join(MMDS_DIR, "m1.x86_64.txt")
+                    )
+                }
             },
             modules=[{"name": "m1:latest-20190101:cafe", "glob": False}],
         )
 
         source_koji._add_module_to_variant(self.koji, variant, self.buildinfo)
 
-        self.assertEqual(
+        mod1 = variant.arch_mmds["armhfp"]["module:master:20190318:abcdef"]
-            variant.arch_mmds,
+        self.assertEqual(mod1.get_NSVCA(), "module:master:20190318:abcdef:armhfp")
-            {
+        mod2 = variant.arch_mmds["x86_64"]["module:master:20190318:abcdef"]
-                "armhfp": {
+        self.assertEqual(mod2.get_NSVCA(), "module:master:20190318:abcdef:x86_64")
-                    "module:master:20190318:abcdef": MockModule(
+        mod3 = variant.arch_mmds["x86_64"]["m1:latest:20190101:cafe"]
-                        "/koji/modulemd.armv7hl.txt"
+        self.assertEqual(mod3.get_NSVCA(), "m1:latest:20190101:cafe:x86_64")
-                    ),
+
-                },
-                "x86_64": {
-                    "module:master:20190318:abcdef": MockModule(
-                        "/koji/modulemd.x86_64.txt"
-                    ),
-                    "m1:latest:20190101:cafe": MockModule("/koji/m1.x86_64.txt"),
-                },
-            },
-        )
         self.assertEqual(
             variant.modules, [{"name": "m1:latest-20190101:cafe", "glob": False}]
         )
@@ -768,21 +743,11 @@ class TestAddModuleToVariant(helpers.PungiTestCase):
             self.koji, variant, self.buildinfo, add_to_variant_modules=True
         )
 
-        self.assertEqual(
+        mod1 = variant.arch_mmds["armhfp"]["module:master:20190318:abcdef"]
-            variant.arch_mmds,
+        self.assertEqual(mod1.get_NSVCA(), "module:master:20190318:abcdef:armhfp")
-            {
+        mod2 = variant.arch_mmds["x86_64"]["module:master:20190318:abcdef"]
-                "armhfp": {
+        self.assertEqual(mod2.get_NSVCA(), "module:master:20190318:abcdef:x86_64")
-                    "module:master:20190318:abcdef": MockModule(
+
-                        "/koji/modulemd.armv7hl.txt"
-                    ),
-                },
-                "x86_64": {
-                    "module:master:20190318:abcdef": MockModule(
-                        "/koji/modulemd.x86_64.txt"
-                    ),
-                },
-            },
-        )
         self.assertEqual(
             variant.modules, [{"name": "module:master:20190318:abcdef", "glob": False}]
         )
@@ -791,7 +756,11 @@ class TestAddModuleToVariant(helpers.PungiTestCase):
         variant = mock.Mock(
             arches=["armhfp", "x86_64"],
             arch_mmds={
-                "x86_64": {"m1:latest:20190101:cafe": MockModule("/koji/m1.x86_64.txt")}
+                "x86_64": {
+                    "m1:latest:20190101:cafe": read_single_module_stream_from_file(
+                        os.path.join(MMDS_DIR, "m1.x86_64.txt")
+                    )
+                }
             },
             modules=[{"name": "m1:latest-20190101:cafe", "glob": False}],
         )
@@ -800,22 +769,13 @@ class TestAddModuleToVariant(helpers.PungiTestCase):
             self.koji, variant, self.buildinfo, add_to_variant_modules=True
         )
 
-        self.assertEqual(
+        mod1 = variant.arch_mmds["armhfp"]["module:master:20190318:abcdef"]
-            variant.arch_mmds,
+        self.assertEqual(mod1.get_NSVCA(), "module:master:20190318:abcdef:armhfp")
-            {
+        mod2 = variant.arch_mmds["x86_64"]["module:master:20190318:abcdef"]
-                "armhfp": {
+        self.assertEqual(mod2.get_NSVCA(), "module:master:20190318:abcdef:x86_64")
-                    "module:master:20190318:abcdef": MockModule(
+        mod3 = variant.arch_mmds["x86_64"]["m1:latest:20190101:cafe"]
-                        "/koji/modulemd.armv7hl.txt"
+        self.assertEqual(mod3.get_NSVCA(), "m1:latest:20190101:cafe:x86_64")
-                    ),
+
-                },
-                "x86_64": {
-                    "module:master:20190318:abcdef": MockModule(
-                        "/koji/modulemd.x86_64.txt"
-                    ),
-                    "m1:latest:20190101:cafe": MockModule("/koji/m1.x86_64.txt"),
-                },
-            },
-        )
         self.assertEqual(
             variant.modules,
             [
@@ -891,12 +851,8 @@ class MockMBS(object):
         return {"id": 1, "koji_tag": "scratch-module-tag", "name": "scratch-module"}
 
     def final_modulemd(self, module_build_id):
-        return {"x86_64": ""}
+        with open(os.path.join(MMDS_DIR, "scratch-module.x86_64.txt")) as f:
-
+            return {"x86_64": f.read()}
-
-class MockMmd(object):
-    def __init__(self, mmd, strict=True):
-        pass
 
 
 @mock.patch("pungi.phases.pkgset.sources.source_koji.MBSWrapper", new=MockMBS)
@@ -909,10 +865,7 @@ class TestAddScratchModuleToVariant(helpers.PungiTestCase):
         )
         self.nsvc = "scratch-module:master:20200710:abcdef"
 
-    @mock.patch(
+    def test_adding_scratch_module(self):
-        "pungi.phases.pkgset.sources.source_koji.Modulemd.ModuleStream.read_string"
-    )
-    def test_adding_scratch_module(self, mock_mmd):
         variant = mock.Mock(
             arches=["armhfp", "x86_64"],
             arch_mmds={},
@@ -927,11 +880,16 @@ class TestAddScratchModuleToVariant(helpers.PungiTestCase):
             self.compose, variant, scratch_modules, variant_tags, tag_to_mmd
         )
         self.assertEqual(variant_tags, {variant: ["scratch-module-tag"]})
+
         self.assertEqual(
-            variant.arch_mmds, {"x86_64": {self.nsvc: mock_mmd.return_value}}
+            variant.arch_mmds["x86_64"][self.nsvc].get_NSVCA(),
+            "scratch-module:master:20200710:abcdef:x86_64",
         )
+
+        self.assertTrue(isinstance(tag_to_mmd["scratch-module-tag"]["x86_64"], set))
         self.assertEqual(
-            tag_to_mmd, {"scratch-module-tag": {"x86_64": set([mock_mmd.return_value])}}
+            list(tag_to_mmd["scratch-module-tag"]["x86_64"])[0].get_NSVCA(),
+            "scratch-module:master:20200710:abcdef:x86_64",
         )
 
         self.assertEqual(variant.modules, [])