From 27a825de48e4faf720d4b510ccf94dc9eeca7954 Mon Sep 17 00:00:00 2001
From: Haibo Lin <hlin@redhat.com>
Date: Mon, 21 Sep 2020 11:14:30 +0800
Subject: [PATCH] pkgset: Allow to include extra module builds

Signed-off-by: Haibo Lin <hlin@redhat.com>
---
 doc/configuration.rst                      |  4 ++
 pungi/checks.py                            |  7 +++
 pungi/phases/pkgset/sources/source_koji.py | 57 +++++++++++++++++++++-
 3 files changed, 66 insertions(+), 2 deletions(-)

diff --git a/doc/configuration.rst b/doc/configuration.rst
index ad2d3659..73af4fb4 100644
--- a/doc/configuration.rst
+++ b/doc/configuration.rst
@@ -550,6 +550,10 @@ Options
    list will be used (and taken from the tag). Inheritance is used
    automatically.
 
+**pkgset_koji_module_builds**
+    (*dict*) -- A mapping of variants to extra module builds to include in a
+    package set: ``{variant: [N:S:V:C]}``.
+
 **pkgset_koji_inherit** = True
     (*bool*) -- inherit builds from parent tags; we can turn it off only if we
     have all builds tagged in a single tag
diff --git a/pungi/checks.py b/pungi/checks.py
index 8f23a672..e9ef3349 100644
--- a/pungi/checks.py
+++ b/pungi/checks.py
@@ -780,6 +780,13 @@ def make_schema():
             "pkgset_koji_builds": {"$ref": "#/definitions/strings"},
             "pkgset_koji_scratch_tasks": {"$ref": "#/definitions/strings"},
             "pkgset_koji_module_tag": {"$ref": "#/definitions/strings", "default": []},
+            "pkgset_koji_module_builds": {
+                "type": "object",
+                "patternProperties": {
+                    "^.+$": {"$ref": "#/definitions/list_of_strings"}
+                },
+                "additionalProperties": False,
+            },
             "pkgset_koji_inherit": {"type": "boolean", "default": True},
             "pkgset_koji_inherit_modules": {"type": "boolean", "default": False},
             "pkgset_exclusive_arch_considers_noarch": {
diff --git a/pungi/phases/pkgset/sources/source_koji.py b/pungi/phases/pkgset/sources/source_koji.py
index 309ed654..a557f604 100644
--- a/pungi/phases/pkgset/sources/source_koji.py
+++ b/pungi/phases/pkgset/sources/source_koji.py
@@ -273,6 +273,52 @@ def _add_module_to_variant(
     return nsvc
 
 
+def _add_extra_modules_to_variant(
+    compose, koji_wrapper, variant, extra_modules, variant_tags, tag_to_mmd
+):
+    for nsvc in extra_modules:
+        msg = "Adding extra module build '%s' to variant '%s'" % (nsvc, variant)
+        compose.log_info(msg)
+
+        nsvc_info = nsvc.split(":")
+        if len(nsvc_info) != 4:
+            raise ValueError("Module %s does not in N:S:V:C format" % nsvc)
+
+        koji_build = koji_wrapper.koji_proxy.getBuild(
+            "%s-%s-%s.%s" % tuple(nsvc_info), True
+        )
+
+        added = _add_module_to_variant(
+            koji_wrapper, variant, koji_build, compose=compose
+        )
+
+        if not added:
+            compose.log_warning("%s - Failed" % msg)
+            continue
+
+        tag = koji_build["extra"]["typeinfo"]["module"]["content_koji_tag"]
+        variant_tags[variant].append(tag)
+
+        tag_to_mmd.setdefault(tag, {})
+        for arch in variant.arch_mmds:
+            try:
+                mmd = variant.arch_mmds[arch][nsvc]
+            except KeyError:
+                # Module was filtered from here
+                continue
+            tag_to_mmd[tag].setdefault(arch, set()).add(mmd)
+
+        if tag_to_mmd[tag]:
+            compose.log_info(
+                "Extra module '%s' in variant '%s' will use Koji tag '%s'"
+                % (nsvc, variant, tag)
+            )
+
+            # Store mapping NSVC --> koji_tag into variant. This is needed
+            # in createrepo phase where metadata is exposed by producmd
+            variant.module_uid_to_koji_tag[nsvc] = tag
+
+
 def _add_scratch_modules_to_variant(
     compose, variant, scratch_modules, variant_tags, tag_to_mmd
 ):
@@ -319,8 +365,7 @@ def _add_scratch_modules_to_variant(
 
 
 def _is_filtered_out(compose, variant, arch, module_name, module_stream):
-    """Check if module with given name and stream is filter out from this stream.
-    """
+    """Check if module with given name and stream is filter out from this stream."""
     if not compose:
         return False
 
@@ -664,6 +709,14 @@ def populate_global_pkgset(compose, koji_wrapper, path_prefix, event):
                 compose, koji_wrapper, event, variant, variant_tags, tag_to_mmd
             )
 
+        extra_modules = get_variant_data(
+            compose.conf, "pkgset_koji_module_builds", variant
+        )
+        if extra_modules:
+            _add_extra_modules_to_variant(
+                compose, koji_wrapper, variant, extra_modules, variant_tags, tag_to_mmd
+            )
+
         variant_scratch_modules = get_variant_data(
             compose.conf, "pkgset_scratch_modules", variant
         )