Handling multiple modules with the same NSV - PDC

Signed-off-by: Ondrej Nosek <onosek@redhat.com>

JIRA: COMPOSE-2510
Signed-off-by: Ondrej Nosek <onosek@redhat.com>

pungi/phases/pkgset/sources/source_koji.py

@@ -124,7 +124,7 @@ def variant_dict_from_str(compose, module_str):
 
 
 @retry(wait_on=IOError)
-def get_module(compose, session, module_info_str):
+def get_pdc_modules(compose, session, module_info_str):
     """
     :param session : PDCClient instance
     :param module_info_str: pdc variant_dict, str, mmd or module dict
@@ -150,22 +150,19 @@ def get_module(compose, session, module_info_str):
     if not retval:
         raise ValueError("Failed to find module in PDC %r" % query)
 
-    module = None
+    modules = []
-    # If we specify 'version', we expect only single module to be
-    # returned, but otherwise we have to pick the one with the highest
-    # release ourselves.
-    if 'version' in query:
-        if len(retval) > 1:
-            raise RuntimeError("More than one module returned from PDC for %s: %s"
-                               % (module_info_str, retval))
-        module = retval[0]
-    else:
-        module = retval[0]
-        for m in retval:
-            if int(m['version']) > int(module['version']):
-                module = m
 
-    return module
+    # If there is version provided, then all modules with that version will go in.
+    # In case version is missing, we will find the latest version and include all modules with that version.
+    if 'version' in query:
+        modules = retval  # all found modules
+    else:
+        # select all found modules with latest version
+        sorted_retval = sorted(retval, key=lambda item: int(item['version']), reverse=True)
+        latest_version = int(sorted_retval[0]['version'])
+        modules = [module for module in sorted_retval if latest_version == int(module['version'])]
+
+    return modules
 
 
 class PkgsetSourceKoji(pungi.phases.pkgset.source.PkgsetSourceBase):
@@ -267,7 +264,8 @@ def _get_modules_from_pdc(compose, session, variant, variant_tags):
     # Find out all modules in every variant and add their Koji tags
     # to variant and variant_tags list.
     for module in variant.get_modules():
-        pdc_module = get_module(compose, session, module["name"])
+        pdc_modules = get_pdc_modules(compose, session, module["name"])
+        for pdc_module in pdc_modules:
 
             mmd = Modulemd.Module.new_from_string(pdc_module["modulemd"])
             mmd.upgrade()
@@ -283,8 +281,8 @@ def _get_modules_from_pdc(compose, session, variant, variant_tags):
             # This is needed in createrepo phase where metadata is exposed by producmd
             variant.module_uid_to_koji_tag[uid] = tag
 
-        module_msg = "Module {module} in variant {variant} will use Koji tag {tag}.".format(
+            module_msg = "Module '{uid}' in variant '{variant}' will use Koji tag '{tag}' (as a result of querying module '{module}')".format(
-            variant=variant, tag=tag, module=module["name"])
+                uid=uid, variant=variant, tag=tag, module=module["name"])
             compose.log_info("%s" % module_msg)
 
 

tests/test_pkgset_source_koji.py

@@ -126,13 +126,13 @@ class TestPopulateGlobalPkgset(helpers.PungiTestCase):
     @unittest.skipUnless(Modulemd is not None, 'Modulemd not available')   # noqa
     @mock.patch('six.moves.cPickle.dumps')
     @mock.patch('pungi.phases.pkgset.pkgsets.KojiPackageSet')
-    @mock.patch('pungi.phases.pkgset.sources.source_koji.get_module')
+    @mock.patch('pungi.phases.pkgset.sources.source_koji.get_pdc_modules')
     @mock.patch('pungi.phases.pkgset.sources.source_koji.get_pdc_client_session')
-    def test_pdc_log(self, get_pdc_client_session, get_module, KojiPackageSet, pickle_dumps):
+    def test_pdc_log(self, get_pdc_client_session, get_pdc_modules, KojiPackageSet, pickle_dumps):
 
         pickle_dumps.return_value = b'DATA'
 
-        modulemd = """
+        modulemd1 = """
 document: modulemd
 version: 2
 data:
@@ -146,9 +146,24 @@ data:
             - MIT
 """
 
-        get_module.return_value = {
+        modulemd2 = """
+document: modulemd
+version: 2
+data:
+    name: foo
+    stream: bar
+    version: 4
+    summary: foo
+    description: foo
+    license:
+        module:
+            - MIT
+"""
+
+        get_pdc_modules.return_value = [
+            {
                 'abc': 'def',
-            'modulemd': modulemd,
+                'modulemd': modulemd1,
                 'rpms': [],
                 'koji_tag': 'taggg',
                 'uid': 'modulenamefoo:rhel:1:00000000',
@@ -156,11 +171,23 @@ data:
                 'stream': 'rhel',
                 'version': '1',
                 'context': '00000000'
-        }
+            },
+            {
+                'abc': 'def',
+                'modulemd': modulemd2,
+                'rpms': [],
+                'koji_tag': 'taggg',
+                'uid': 'modulenamefoo:rhel:4:00000000',
+                'name': 'modulenamefoo',
+                'stream': 'rhel',
+                'version': '4',
+                'context': '00000000'
+            },
+        ]
         for name, variant in self.compose.variants.items():
             variant.get_modules = mock.MagicMock()
             if name == 'Server':
-                variant.modules = [{'name': 'a'}]
+                variant.modules = [{'name': 'modulenamefoo'}]
                 variant.get_modules.return_value = variant.modules
 
         source_koji.populate_global_pkgset(