Allow including scratch module builds

JIRA: RHELCMP-439
Signed-off-by: Haibo Lin <hlin@redhat.com>

doc/configuration.rst

@@ -190,6 +190,11 @@ Options
     (*str*) -- Allows to set default compose type. Type set via a command-line
     option overwrites this.
 
+**mbs_api_url**
+    (*str*) -- URL to Module Build Service (MBS) API.
+    For example ``https://mbs.example.com/module-build-service/2``.
+    This is required by ``pkgset_scratch_modules``.
+
 Example
 -------
 ::
@@ -548,6 +553,10 @@ Options
     (*dict*) -- A mapping of architectures to repositories with RPMs: ``{arch:
     [repo]}``. Only use when ``pkgset_source = "repos"``.
 
+**pkgset_scratch_modules**
+    (*dict*) -- A mapping of variants to scratch module builds: ``{variant:
+    [N:S:V:C]}``. Requires ``mbs_api_url``.
+
 **pkgset_exclusive_arch_considers_noarch** = True
     (*bool*) -- If a package includes ``noarch`` in its ``ExclusiveArch`` tag,
     it will be included in all architectures since ``noarch`` is compatible

pungi/checks.py

@@ -786,6 +786,14 @@ def make_schema():
                 "type": "boolean",
                 "default": True,
             },
+            "pkgset_scratch_modules": {
+                "type": "object",
+                "patternProperties": {
+                    "^.+$": {"$ref": "#/definitions/list_of_strings"}
+                },
+                "additionalProperties": False,
+            },
+            "mbs_api_url": {"type": "string"},
             "disc_types": {"type": "object", "default": {}},
             "paths_module": {"type": "string"},
             "skip_phases": {
@@ -1258,6 +1266,7 @@ CONFIG_DEPS = {
         "conflicts": ((lambda x: not x, ["base_product_name", "base_product_short"]),),
     },
     "product_id": {"conflicts": [(lambda x: not x, ["product_id_allow_missing"])]},
+    "pkgset_scratch_modules": {"requires": ((lambda x: x, ["mbs_api_url"]),)},
     "pkgset_source": {
         "requires": [(lambda x: x == "repos", ["pkgset_repos"])],
         "conflicts": [

pungi/paths.py

@@ -490,6 +490,8 @@ class WorkPaths(object):
 
     def module_defaults_dir(self, create_dir=True):
         """
+        Example:
+            work/global/module_defaults
         """
         path = os.path.join(self.topdir(create_dir=create_dir), "module_defaults")
         if create_dir:

pungi/phases/pkgset/sources/source_koji.py

@@ -26,9 +26,10 @@ from kobo.shortcuts import force_list
 
 import pungi.wrappers.kojiwrapper
 from pungi.wrappers.comps import CompsWrapper
+from pungi.wrappers.mbs import MBSWrapper
 import pungi.phases.pkgset.pkgsets
 from pungi.arch import getBaseArch
-from pungi.util import retry, get_arch_variant_data
+from pungi.util import retry, get_arch_variant_data, get_variant_data
 from pungi.module_util import Modulemd
 
 from pungi.phases.pkgset.common import MaterializedPackageSet, get_all_arches
@@ -272,6 +273,51 @@ def _add_module_to_variant(
     return nsvc
 
 
+def _add_scratch_modules_to_variant(
+    compose, variant, scratch_modules, variant_tags, tag_to_mmd
+):
+    if compose.compose_type != "test" and scratch_modules:
+        compose.log_warning("Only test composes could include scratch module builds")
+        return
+
+    mbs = MBSWrapper(compose.conf["mbs_api_url"])
+    for nsvc in scratch_modules:
+        module_build = mbs.get_module_build_by_nsvc(nsvc)
+        if not module_build:
+            continue
+        try:
+            final_modulemd = mbs.final_modulemd(module_build["id"])
+        except Exception:
+            compose.log_error("Unable to get modulemd for build %s" % module_build)
+            raise
+        tag = module_build["koji_tag"]
+        variant_tags[variant].append(tag)
+        tag_to_mmd.setdefault(tag, {})
+        for arch in variant.arches:
+            try:
+                mmd = Modulemd.ModuleStream.read_string(
+                    final_modulemd[arch], strict=True
+                )
+                variant.arch_mmds.setdefault(arch, {})[nsvc] = mmd
+            except KeyError:
+                continue
+            tag_to_mmd[tag].setdefault(arch, set()).add(mmd)
+
+            if tag_to_mmd[tag]:
+                compose.log_info(
+                    "Module '%s' in variant '%s' will use Koji tag '%s' "
+                    "(as a result of querying module '%s')",
+                    nsvc,
+                    variant,
+                    tag,
+                    module_build["name"],
+                )
+
+                # Store mapping NSVC --> koji_tag into variant. This is needed
+                # in createrepo phase where metadata is exposed by productmd
+                variant.module_uid_to_koji_tag[nsvc] = tag
+
+
 def _is_filtered_out(compose, variant, arch, module_name, module_stream):
     """Check if module with given name and stream is filter out from this stream.
     """
@@ -618,6 +664,14 @@ def populate_global_pkgset(compose, koji_wrapper, path_prefix, event):
                 compose, koji_wrapper, event, variant, variant_tags, tag_to_mmd
             )
 
+        variant_scratch_modules = get_variant_data(
+            compose.conf, "pkgset_scratch_modules", variant
+        )
+        if variant_scratch_modules:
+            _add_scratch_modules_to_variant(
+                compose, variant, variant_scratch_modules, variant_tags, tag_to_mmd
+            )
+
         # Ensure that every tag added to `variant_tags` is added also to
         # `compose_tags`.
         for variant_tag in variant_tags[variant]:

pungi/wrappers/mbs.py

@@ -0,0 +1,43 @@
+import os
+import requests
+
+
+class MBSWrapper(object):
+    def __init__(self, api_url):
+        """
+        :param string api_url: e.g. https://example.com/module-build-service/2
+        """
+        self.api_url = api_url
+
+    def _get(self, resource, params=None):
+        """Get specified resource.
+
+        :param string resource: e.g. module-builds, final-modulemd
+        :param dict data:
+        """
+        url = os.path.join(self.api_url, resource)
+        try:
+            resp = requests.get(url, params=params)
+        except Exception as e:
+            raise Exception(
+                "Failed to query URL %s with params %s - %s" % (url, params, str(e))
+            )
+        resp.raise_for_status()
+        return resp
+
+    def module_builds(self, filters=None):
+        return self._get("module-builds", filters).json()
+
+    def get_module_build_by_nsvc(self, nsvc):
+        nsvc_list = nsvc.split(":")
+        if len(nsvc_list) != 4:
+            raise ValueError("Invalid N:S:V:C - %s" % nsvc)
+        filters = dict(zip(["name", "stream", "version", "context"], nsvc_list))
+        resp = self.module_builds(filters)
+        if resp["items"]:
+            return resp["items"][0]
+        else:
+            return None
+
+    def final_modulemd(self, module_build_id):
+        return self._get("final-modulemd/%s" % module_build_id).json()

tests/test_pkgset_source_koji.py

@@ -881,3 +881,68 @@ class TestIsModuleFiltered(helpers.PungiTestCase):
         self.assertIsFiltered("foo", "master")
         self.assertIsNotFiltered("bar", "master")
         self.assertIsNotFiltered("foo", "stable")
+
+
+class MockMBS(object):
+    def __init__(self, api_url):
+        self.api_url = api_url
+
+    def get_module_build_by_nsvc(self, nsvc):
+        return {"id": 1, "koji_tag": "scratch-module-tag", "name": "scratch-module"}
+
+    def final_modulemd(self, module_build_id):
+        return {"x86_64": ""}
+
+
+class MockMmd(object):
+    def __init__(self, mmd, strict=True):
+        pass
+
+
+@mock.patch("pungi.phases.pkgset.sources.source_koji.MBSWrapper", new=MockMBS)
+@unittest.skipIf(Modulemd is None, "Skipping tests, no module support")
+class TestAddScratchModuleToVariant(helpers.PungiTestCase):
+    def setUp(self):
+        super(TestAddScratchModuleToVariant, self).setUp()
+        self.compose = helpers.DummyCompose(
+            self.topdir, {"mbs_api_url": "http://mbs.local/module-build-service/2"}
+        )
+        self.nsvc = "scratch-module:master:20200710:abcdef"
+
+    @mock.patch(
+        "pungi.phases.pkgset.sources.source_koji.Modulemd.ModuleStream.read_string"
+    )
+    def test_adding_scratch_module(self, mock_mmd):
+        variant = mock.Mock(
+            arches=["armhfp", "x86_64"],
+            arch_mmds={},
+            modules=[],
+            module_uid_to_koji_tag={},
+        )
+        variant_tags = {variant: []}
+        tag_to_mmd = {}
+        scratch_modules = [self.nsvc]
+
+        source_koji._add_scratch_modules_to_variant(
+            self.compose, variant, scratch_modules, variant_tags, tag_to_mmd
+        )
+        self.assertEqual(variant_tags, {variant: ["scratch-module-tag"]})
+        self.assertEqual(
+            variant.arch_mmds, {"x86_64": {self.nsvc: mock_mmd.return_value}}
+        )
+        self.assertEqual(
+            tag_to_mmd, {"scratch-module-tag": {"x86_64": {mock_mmd.return_value}}}
+        )
+
+        self.assertEqual(variant.modules, [])
+
+    def test_adding_scratch_module_nontest_compose(self):
+        self.compose.compose_type = "production"
+        scratch_modules = [self.nsvc]
+
+        source_koji._add_scratch_modules_to_variant(
+            self.compose, mock.Mock(), scratch_modules, {}, {}
+        )
+        self.compose.log_warning.assert_called_once_with(
+            "Only test composes could include scratch module builds"
+        )