Add support for modular composes

Signed-off-by: Jan Kaluza <jkaluza@redhat.com>

pungi/checks.py

@@ -493,7 +493,7 @@ def _make_schema():
             },
             "gather_source": {
                 "type": "string",
-                "enum": ["json", "comps", "none"],
+                "enum": ["module", "json", "comps", "none"],
             },
             "gather_fulltree": {
                 "type": "boolean",
@@ -614,6 +614,10 @@ def _make_schema():
             "global_target": {"type": "string"},
             "global_release": {"$ref": "#/definitions/optional_string"},
 
+            "pdc_url": {"type": "string"},
+            "pdc_develop": {"type": "boolean", "default": False},
+            "pdc_insecure": {"type": "boolean", "default": False},
+
             "koji_profile": {"type": "string"},
 
             "pkgset_koji_tag": {"type": "string"},

pungi/phases/createrepo.py

@@ -181,6 +181,24 @@ def create_variant_repo(compose, arch, variant, pkg_type):
             # this is a HACK to make CDN happy (dmach: at least I think, need to confirm with dgregor)
             shutil.copy2(product_id_path, os.path.join(repo_dir, "repodata", "productid"))
 
+    # call modifyrepo to inject modulemd if needed
+    if variant.mmds:
+        import yaml
+        modules = {"modules": []}
+        for mmd in variant.mmds:
+            modules["modules"].append(mmd.dumps())
+        tmp_dir = compose.mkdtemp(prefix="pungi_")
+        modules_path = os.path.join(tmp_dir, "modules.yaml")
+        with open(modules_path, "w") as outfile:
+            outfile.write(yaml.safe_dump(modules))
+        cmd = repo.get_modifyrepo_cmd(os.path.join(repo_dir, "repodata"),
+                                      modules_path, mdtype="modules",
+                                      compress_type="gz")
+        log_file = compose.paths.log.log_file(
+            arch, "modifyrepo-modules-%s" % variant)
+        run(cmd, logfile=log_file, show_cmd=True)
+        shutil.rmtree(tmp_dir)
+
     compose.log_info("[DONE ] %s" % msg)
 
 

pungi/phases/gather/sources/source_module.py

@@ -0,0 +1,44 @@
+# -*- coding: utf-8 -*-
+
+
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; version 2 of the License.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Library General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, see <https://gnu.org/licenses/>.
+
+
+"""
+Get a package list based on modulemd metadata loaded in pkgset phase.
+"""
+
+
+from pungi.wrappers.comps import CompsWrapper
+import pungi.phases.gather.source
+import kobo.rpmlib
+
+
+class GatherSourceModule(pungi.phases.gather.source.GatherSourceBase):
+    enabled = True
+
+    def __call__(self, arch, variant):
+        groups = set()
+        packages = set()
+
+        if variant is not None and variant.modules:
+            rpms = variant.pkgset.rpms_by_arch[arch] + \
+                variant.pkgset.rpms_by_arch["noarch"]
+            for rpm_obj in rpms:
+                for mmd in variant.mmds:
+                    srpm = kobo.rpmlib.parse_nvr(rpm_obj.sourcerpm)["name"]
+                    if (srpm in mmd.components.rpms.keys() and
+                            rpm_obj.name not in mmd.filter.rpms):
+                        packages.add((rpm_obj.name, None))
+
+        return packages, groups

pungi/phases/pkgset/pkgsets.py

@@ -148,15 +148,18 @@ class PackageSetBase(kobo.log.LoggingBase):
         seen_sourcerpms = set()
         # {Exclude,Exclusive}Arch must match *tree* arch + compatible native
         # arches (excluding multilib arches)
-        exclusivearch_list = get_valid_arches(primary_arch, multilib=False,
+        if primary_arch:
-                                              add_noarch=False, add_src=False)
+            exclusivearch_list = get_valid_arches(
+                primary_arch, multilib=False, add_noarch=False, add_src=False)
+        else:
+            exclusivearch_list = None
         for arch in arch_list:
             self.rpms_by_arch.setdefault(arch, [])
             for i in other.rpms_by_arch.get(arch, []):
                 if i.file_path in self.file_cache:
                     # TODO: test if it really works
                     continue
-                if arch == "noarch":
+                if exclusivearch_list and arch == "noarch":
                     if i.excludearch and set(i.excludearch) & set(exclusivearch_list):
                         self.log_debug("Excluding (EXCLUDEARCH: %s): %s"
                                        % (sorted(set(i.excludearch)), i.file_name))

pungi/phases/pkgset/sources/source_koji.py

@@ -28,6 +28,84 @@ from pungi.phases.pkgset.common import create_arch_repos, create_global_repo, po
 
 import pungi.phases.pkgset.source
 
+try:
+    from pdc_client import PDCClient
+    import modulemd
+    WITH_MODULES = True
+except:
+    WITH_MODULES = False
+
+
+def get_pdc_client_session(compose):
+    if not WITH_MODULES:
+        compose.log_warning("pdc_client module is not installed, "
+                            "support for modules is disabled")
+        return None
+    try:
+        return PDCClient(
+            server=compose.conf['pdc_url'],
+            develop=compose.conf['pdc_develop'],
+            ssl_verify=not compose.conf['pdc_insecure'],
+        )
+    except KeyError:
+        return None
+
+
+def variant_dict_from_str(module_str):
+    module_info = {}
+
+    release_start = module_str.rfind('-')
+    module_info['variant_version'] = module_str[release_start+1:]
+    module_info['variant_id'] = module_str[:release_start]
+    module_info['variant_type'] = 'module'
+
+    return module_info
+
+
+def get_module(session, module_info, strict=False):
+    """
+    :param session : PDCClient instance
+    :param module_info: pdc variant_dict, str, mmd or module dict
+    :param strict: Normally this function returns None if no module can be
+           found.  If strict=True, then a ValueError is raised.
+
+    :return final list of module_info which pass repoclosure
+    """
+
+    module_info = variant_dict_from_str(module_info)
+
+    query = dict(
+        variant_id=module_info['variant_id'],
+        variant_version=module_info['variant_version'],
+    )
+    if module_info.get('variant_release'):
+        query['variant_release'] = module_info['variant_release']
+
+    retval = session['unreleasedvariants'](page_size=-1, **query)
+
+    # Error handling
+    if not retval:
+        if strict:
+            raise ValueError("Failed to find module in PDC %r" % query)
+        else:
+            return None
+
+    module = None
+    # If we specify 'variant_release', we expect only single module to be
+    # returned, but otherwise we have to pick the one with the highest
+    # release ourselves.
+    if 'variant_release' in query:
+        assert len(retval) <= 1, compose.log_error(
+            "More than one module returned from PDC: %s" % str(retval))
+        module = retval[0]
+    else:
+        module = retval[0]
+        for m in retval:
+            if int(m['variant_release']) > int(module['variant_release']):
+                module = m
+
+    return module
+
 
 class PkgsetSourceKoji(pungi.phases.pkgset.source.PkgsetSourceBase):
     enabled = True
@@ -44,9 +122,7 @@ class PkgsetSourceKoji(pungi.phases.pkgset.source.PkgsetSourceBase):
 
 def get_pkgset_from_koji(compose, koji_wrapper, path_prefix):
     event_info = get_koji_event_info(compose, koji_wrapper)
-    tag_info = get_koji_tag_info(compose, koji_wrapper)
+    pkgset_global = populate_global_pkgset(compose, koji_wrapper, path_prefix, event_info)
-
-    pkgset_global = populate_global_pkgset(compose, koji_wrapper, path_prefix, tag_info, event_info)
     package_sets = populate_arch_pkgsets(compose, path_prefix, pkgset_global)
     package_sets["global"] = pkgset_global
 
@@ -58,32 +134,90 @@ def get_pkgset_from_koji(compose, koji_wrapper, path_prefix):
     return package_sets
 
 
-def populate_global_pkgset(compose, koji_wrapper, path_prefix, compose_tag, event_id):
+def populate_global_pkgset(compose, koji_wrapper, path_prefix, event_id):
     all_arches = set(["src"])
     for arch in compose.get_arches():
         is_multilib = is_arch_multilib(compose.conf, arch)
         arches = get_valid_arches(arch, is_multilib)
         all_arches.update(arches)
 
-    compose_tag = compose.conf["pkgset_koji_tag"]
+    # List of compose tags from which we create this compose
+    compose_tags = []
+
+    # List of compose_tags per variant
+    variant_tags = {}
+
+    session = get_pdc_client_session(compose)
+    for variant in compose.all_variants.values():
+        variant.pkgset = pungi.phases.pkgset.pkgsets.KojiPackageSet(
+            koji_wrapper, compose.conf["sigkeys"], logger=compose._logger,
+            arches=all_arches)
+        variant_tags[variant] = []
+
+        # Find out all modules in every variant and add their compose tags
+        # to compose_tags list.
+        if session:
+            for module in variant.get_modules():
+                pdc_module = get_module(session, module["name"])
+                mmd = modulemd.ModuleMetadata()
+                mmd.loads(pdc_module["modulemd"])
+                tag = pdc_module["koji_tag"]
+                variant.mmds.append(mmd)
+                variant_tags[variant].append(tag)
+                if tag not in compose_tags:
+                    compose_tags.append(tag)
+
+        if not variant_tags[variant]:
+            variant_tags[variant].append(compose.conf["pkgset_koji_tag"])
+
+    # In case we have no compose tag from module, use the default
+    # one from config.
+    if not compose_tags:
+        compose_tags.append(compose.conf["pkgset_koji_tag"])
+
     inherit = compose.conf["pkgset_koji_inherit"]
-    msg = "Populating the global package set from tag '%s'" % compose_tag
+    global_pkgset_path = os.path.join(
-    global_pkgset_path = os.path.join(compose.paths.work.topdir(arch="global"), "pkgset_global.pickle")
+        compose.paths.work.topdir(arch="global"), "pkgset_global.pickle")
     if compose.DEBUG and os.path.isfile(global_pkgset_path):
+        msg = "Populating the global package set from tag '%s'" % compose_tags
         compose.log_warning("[SKIP ] %s" % msg)
-        pkgset = pickle.load(open(global_pkgset_path, "r"))
+        global_pkgset = pickle.load(open(global_pkgset_path, "r"))
     else:
-        compose.log_info(msg)
+        global_pkgset = pungi.phases.pkgset.pkgsets.KojiPackageSet(
-        pkgset = pungi.phases.pkgset.pkgsets.KojiPackageSet(koji_wrapper, compose.conf["sigkeys"],
+            koji_wrapper, compose.conf["sigkeys"], logger=compose._logger,
-                                                            logger=compose._logger, arches=all_arches)
+            arches=all_arches)
+        # Get package set for each compose tag and merge it to global package
+        # list. Also prepare per-variant pkgset, because we do not have list
+        # of binary RPMs in module definition - there is just list of SRPMs.
+        for compose_tag in compose_tags:
+            compose.log_info("Populating the global package set from tag "
+                             "'%s'" % compose_tag)
+            pkgset = pungi.phases.pkgset.pkgsets.KojiPackageSet(
+                koji_wrapper, compose.conf["sigkeys"], logger=compose._logger,
+                arches=all_arches)
             pkgset.populate(compose_tag, event_id, inherit=inherit)
+            for variant in compose.all_variants.values():
+                if compose_tag in variant_tags[variant]:
+                    # Optimization for case where we have just single compose
+                    # tag - we do not have to merge in this case...
+                    if len(compose_tags) == 1:
+                        variant.pkgset = pkgset
+                    else:
+                        variant.pkgset.merge(pkgset, None, list(all_arches))
+            # Optimization for case where we have just single compose
+            # tag - we do not have to merge in this case...
+            if len(compose_tags) == 1:
+                global_pkgset = pkgset
+            else:
+                global_pkgset.merge(pkgset, None, list(all_arches))
         with open(global_pkgset_path, 'w') as f:
-            f.write(pickle.dumps(pkgset))
+            f.write(pickle.dumps(global_pkgset))
 
     # write global package list
-    pkgset.save_file_list(compose.paths.work.package_list(arch="global"),
+    global_pkgset.save_file_list(
+        compose.paths.work.package_list(arch="global"),
         remove_path_prefix=path_prefix)
-    return pkgset
+    return global_pkgset
 
 
 def get_koji_event_info(compose, koji_wrapper):
@@ -106,22 +240,3 @@ def get_koji_event_info(compose, koji_wrapper):
         json.dump(result, open(event_file, "w"))
     compose.log_info("Koji event: %s" % result["id"])
     return result
-
-
-def get_koji_tag_info(compose, koji_wrapper):
-    koji_proxy = koji_wrapper.koji_proxy
-    tag_file = os.path.join(compose.paths.work.topdir(arch="global"), "koji-tag")
-    msg = "Getting a koji tag info"
-    if compose.DEBUG and os.path.exists(tag_file):
-        compose.log_warning("[SKIP ] %s" % msg)
-        result = json.load(open(tag_file, "r"))
-    else:
-        compose.log_info(msg)
-        tag_name = compose.conf["pkgset_koji_tag"]
-        result = koji_proxy.getTag(tag_name)
-        if result is None:
-            raise ValueError("Unknown koji tag: %s" % tag_name)
-        result["name"] = tag_name
-        json.dump(result, open(tag_file, "w"))
-    compose.log_info("Koji compose tag: %(name)s (ID: %(id)s)" % result)
-    return result

pungi/wrappers/variants.py

@@ -77,6 +77,7 @@ class VariantsXmlParser(object):
             "type": str(variant_node.attrib["type"]),
             "arches": [str(i) for i in variant_node.xpath("arches/arch/text()")],
             "groups": [],
+            "modules": [],
             "environments": [],
             "buildinstallpackages": [],
             "is_empty": bool(variant_node.attrib.get("is_empty", False)),
@@ -108,6 +109,15 @@ class VariantsXmlParser(object):
 
                 variant_dict["groups"].append(group)
 
+        for modulelist_node in variant_node.xpath("modules"):
+            for module_node in modulelist_node.xpath("module"):
+                module = {
+                    "name": str(module_node.text),
+                    "glob": self._is_true(module_node.attrib.get("glob", "false"))
+                }
+
+                variant_dict["modules"].append(module)
+
         for environments_node in variant_node.xpath("environments"):
             for environment_node in environments_node.xpath("environment"):
                 environment = {
@@ -198,9 +208,11 @@ class VariantsXmlParser(object):
 
 class Variant(object):
     def __init__(self, id, name, type, arches, groups, environments=None,
-                 buildinstallpackages=None, is_empty=False, parent=None):
+                 buildinstallpackages=None, is_empty=False, parent=None,
+                 modules=None):
 
         environments = environments or []
+        modules = modules or []
         buildinstallpackages = buildinstallpackages or []
 
         self.id = id
@@ -209,11 +221,16 @@ class Variant(object):
         self.arches = sorted(copy.deepcopy(arches))
         self.groups = sorted(copy.deepcopy(groups), lambda x, y: cmp(x["name"], y["name"]))
         self.environments = sorted(copy.deepcopy(environments), lambda x, y: cmp(x["name"], y["name"]))
+        self.modules = sorted(copy.deepcopy(modules),
+                              lambda x, y: cmp(x["name"], y["name"]))
         self.buildinstallpackages = sorted(buildinstallpackages)
         self.variants = {}
         self.parent = parent
         self.is_empty = is_empty
 
+        self.pkgset = None
+        self.mmds = []
+
     def __getitem__(self, name):
         return self.variants[name]
 
@@ -274,6 +291,22 @@ class Variant(object):
                     result.append(group)
         return result
 
+    def get_modules(self, arch=None, types=None, recursive=False):
+        """Return list of groups, default types is ["self"]"""
+
+        types = types or ["self"]
+        result = copy.deepcopy(self.modules)
+        for variant in self.get_variants(arch=arch, types=types,
+                                         recursive=recursive):
+            if variant == self:
+                # XXX
+                continue
+            for module in variant.get_modules(arch=arch, types=types,
+                                              recursive=recursive):
+                if module not in result:
+                    result.append(module)
+        return result
+
     def get_variants(self, arch=None, types=None, recursive=False):
         """
         Return all variants of given arch and types.
@@ -339,6 +372,8 @@ def main(argv):
             print("    ARCHES: %s" % ", ".join(sorted(i.arches)))
             for group in i.groups:
                 print("    GROUP:  %(name)-40s GLOB: %(glob)-5s DEFAULT: %(default)-5s USERVISIBLE: %(uservisible)-5s" % group)
+            for module in i.modules:
+                print("    MODULE:  %(name)-40s GLOB: %(glob)-5s DEFAULT: %(default)-5s USERVISIBLE: %(uservisible)-5s" % module)
             for env in i.environments:
                 print("    ENV:    %(name)-40s DISPLAY_ORDER: %(display_order)s" % env)
             print()

share/variants.dtd

@@ -1,6 +1,6 @@
 <!ELEMENT variants (ref*,variant*)>
 
-<!ELEMENT variant (release?,arches,groups?,environments*,variants*,buildinstallpackages?)?>
+<!ELEMENT variant (release?,arches,groups?,environments*,variants*,buildinstallpackages?,modules?)?>
 <!ATTLIST variant
     id CDATA #REQUIRED
     name CDATA #REQUIRED
@@ -27,6 +27,13 @@
     uservisible (true|false) #IMPLIED
 >
 
+<!ELEMENT modules (module)+>
+
+<!ELEMENT module (#PCDATA)>
+<!ATTLIST module
+    glob (true|false) #IMPLIED
+>
+
 <!ELEMENT environments (environment)+>
 
 <!ELEMENT environment (#PCDATA)>

tests/helpers.py

@@ -33,6 +33,7 @@ class MockVariant(mock.Mock):
     def __init__(self, *args, **kwargs):
         super(MockVariant, self).__init__(*args, **kwargs)
         self.parent = kwargs.get('parent', None)
+        self.mmds = []
 
     def __str__(self):
         return self.uid

tests/test_pkgset_source_koji.py

@@ -81,50 +81,6 @@ class TestGetKojiEvent(helpers.PungiTestCase):
         with open(self.event_file) as f:
             self.assertEqual(json.load(f), EVENT_INFO)
 
-
-class TestGetKojiTagInfo(helpers.PungiTestCase):
-    def setUp(self):
-        super(TestGetKojiTagInfo, self).setUp()
-        self.compose = helpers.DummyCompose(self.topdir, {
-            'pkgset_koji_tag': 'f25'
-        })
-        self.compose.DEBUG = False
-        self.tag_file = os.path.join(self.topdir, 'work', 'global', 'koji-tag')
-        self.koji_wrapper = mock.Mock()
-
-    def test_get_tag_info(self):
-        self.koji_wrapper.koji_proxy.getTag.return_value = TAG_INFO
-
-        tag_info = source_koji.get_koji_tag_info(self.compose, self.koji_wrapper)
-
-        self.assertEqual(tag_info, TAG_INFO)
-        self.assertItemsEqual(
-            self.koji_wrapper.mock_calls,
-            [mock.call.koji_proxy.getTag('f25')]
-        )
-        with open(self.tag_file) as f:
-            self.assertEqual(json.load(f), TAG_INFO)
-
-    def test_bad_tag_name(self):
-        self.koji_wrapper.koji_proxy.getTag.return_value = None
-
-        with self.assertRaises(ValueError) as ctx:
-            source_koji.get_koji_tag_info(self.compose, self.koji_wrapper)
-
-        self.assertIn('Unknown', str(ctx.exception))
-
-    def test_get_tag_info_in_debug_mode(self):
-        self.compose.DEBUG = True
-        helpers.touch(self.tag_file, json.dumps(TAG_INFO))
-
-        tag_info = source_koji.get_koji_tag_info(self.compose, self.koji_wrapper)
-
-        self.assertEqual(tag_info, TAG_INFO)
-        self.assertItemsEqual(self.koji_wrapper.mock_calls, [])
-        with open(self.tag_file) as f:
-            self.assertEqual(json.load(f), TAG_INFO)
-
-
 class TestPopulateGlobalPkgset(helpers.PungiTestCase):
     def setUp(self):
         super(TestPopulateGlobalPkgset, self).setUp()
@@ -145,7 +101,7 @@ class TestPopulateGlobalPkgset(helpers.PungiTestCase):
         orig_pkgset = KojiPackageSet.return_value
 
         pkgset = source_koji.populate_global_pkgset(
-            self.compose, self.koji_wrapper, '/prefix', 'f25', 123456)
+            self.compose, self.koji_wrapper, '/prefix', 123456)
 
         self.assertIs(pkgset, orig_pkgset)
         self.assertEqual(
@@ -168,7 +124,7 @@ class TestPopulateGlobalPkgset(helpers.PungiTestCase):
         with mock.patch('pungi.phases.pkgset.sources.source_koji.open',
                         mock.mock_open(), create=True) as m:
             pkgset = source_koji.populate_global_pkgset(
-                self.compose, self.koji_wrapper, '/prefix', 'f25', 123456)
+                self.compose, self.koji_wrapper, '/prefix', 123456)
 
         self.assertEqual(pickle_load.call_args_list,
                          [mock.call(m.return_value)])
@@ -204,12 +160,12 @@ class TestGetPackageSetFromKoji(helpers.PungiTestCase):
 
         self.assertItemsEqual(
             self.koji_wrapper.koji_proxy.mock_calls,
-            [mock.call.getLastEvent(),
+            [mock.call.getLastEvent()]
-             mock.call.getTag('f25')]
         )
+
         self.assertEqual(pgp.call_args_list,
                          [mock.call(self.compose, self.koji_wrapper, '/prefix',
-                                    TAG_INFO, EVENT_INFO)])
+                                    EVENT_INFO)])
         self.assertEqual(pap.call_args_list,
                          [mock.call(self.compose, '/prefix', pgp.return_value)])
         self.assertEqual(cgr.call_args_list,