From c85d80f3c2a8913fec83c3ccc198545480fa36c8 Mon Sep 17 00:00:00 2001 From: Ondrej Nosek Date: Thu, 17 May 2018 17:01:20 +0200 Subject: [PATCH] Handling multiple modules with the same NSV - PDC Signed-off-by: Ondrej Nosek JIRA: COMPOSE-2510 Signed-off-by: Ondrej Nosek --- pungi/phases/pkgset/sources/source_koji.py | 60 +++++++++++----------- tests/test_pkgset_source_koji.py | 57 ++++++++++++++------ 2 files changed, 71 insertions(+), 46 deletions(-) diff --git a/pungi/phases/pkgset/sources/source_koji.py b/pungi/phases/pkgset/sources/source_koji.py index 67d4805f..c96b4d44 100644 --- a/pungi/phases/pkgset/sources/source_koji.py +++ b/pungi/phases/pkgset/sources/source_koji.py @@ -124,7 +124,7 @@ def variant_dict_from_str(compose, module_str): @retry(wait_on=IOError) -def get_module(compose, session, module_info_str): +def get_pdc_modules(compose, session, module_info_str): """ :param session : PDCClient instance :param module_info_str: pdc variant_dict, str, mmd or module dict @@ -150,22 +150,19 @@ def get_module(compose, session, module_info_str): if not retval: raise ValueError("Failed to find module in PDC %r" % query) - module = None - # If we specify 'version', we expect only single module to be - # returned, but otherwise we have to pick the one with the highest - # release ourselves. - if 'version' in query: - if len(retval) > 1: - raise RuntimeError("More than one module returned from PDC for %s: %s" - % (module_info_str, retval)) - module = retval[0] - else: - module = retval[0] - for m in retval: - if int(m['version']) > int(module['version']): - module = m + modules = [] - return module + # If there is version provided, then all modules with that version will go in. + # In case version is missing, we will find the latest version and include all modules with that version. + if 'version' in query: + modules = retval # all found modules + else: + # select all found modules with latest version + sorted_retval = sorted(retval, key=lambda item: int(item['version']), reverse=True) + latest_version = int(sorted_retval[0]['version']) + modules = [module for module in sorted_retval if latest_version == int(module['version'])] + + return modules class PkgsetSourceKoji(pungi.phases.pkgset.source.PkgsetSourceBase): @@ -267,25 +264,26 @@ def _get_modules_from_pdc(compose, session, variant, variant_tags): # Find out all modules in every variant and add their Koji tags # to variant and variant_tags list. for module in variant.get_modules(): - pdc_module = get_module(compose, session, module["name"]) + pdc_modules = get_pdc_modules(compose, session, module["name"]) + for pdc_module in pdc_modules: - mmd = Modulemd.Module.new_from_string(pdc_module["modulemd"]) - mmd.upgrade() - _add_module_to_variant(variant, mmd, pdc_module["rpms"]) - _log_modulemd(compose, variant, mmd) + mmd = Modulemd.Module.new_from_string(pdc_module["modulemd"]) + mmd.upgrade() + _add_module_to_variant(variant, mmd, pdc_module["rpms"]) + _log_modulemd(compose, variant, mmd) - tag = pdc_module["koji_tag"] - uid = ':'.join([pdc_module['name'], pdc_module['stream'], - pdc_module['version'], pdc_module['context']]) - variant_tags[variant].append(tag) + tag = pdc_module["koji_tag"] + uid = ':'.join([pdc_module['name'], pdc_module['stream'], + pdc_module['version'], pdc_module['context']]) + variant_tags[variant].append(tag) - # Store mapping module-uid --> koji_tag into variant. - # This is needed in createrepo phase where metadata is exposed by producmd - variant.module_uid_to_koji_tag[uid] = tag + # Store mapping module-uid --> koji_tag into variant. + # This is needed in createrepo phase where metadata is exposed by producmd + variant.module_uid_to_koji_tag[uid] = tag - module_msg = "Module {module} in variant {variant} will use Koji tag {tag}.".format( - variant=variant, tag=tag, module=module["name"]) - compose.log_info("%s" % module_msg) + module_msg = "Module '{uid}' in variant '{variant}' will use Koji tag '{tag}' (as a result of querying module '{module}')".format( + uid=uid, variant=variant, tag=tag, module=module["name"]) + compose.log_info("%s" % module_msg) def _get_modules_from_koji_tags( diff --git a/tests/test_pkgset_source_koji.py b/tests/test_pkgset_source_koji.py index c71f415f..63df2156 100644 --- a/tests/test_pkgset_source_koji.py +++ b/tests/test_pkgset_source_koji.py @@ -126,13 +126,13 @@ class TestPopulateGlobalPkgset(helpers.PungiTestCase): @unittest.skipUnless(Modulemd is not None, 'Modulemd not available') # noqa @mock.patch('six.moves.cPickle.dumps') @mock.patch('pungi.phases.pkgset.pkgsets.KojiPackageSet') - @mock.patch('pungi.phases.pkgset.sources.source_koji.get_module') + @mock.patch('pungi.phases.pkgset.sources.source_koji.get_pdc_modules') @mock.patch('pungi.phases.pkgset.sources.source_koji.get_pdc_client_session') - def test_pdc_log(self, get_pdc_client_session, get_module, KojiPackageSet, pickle_dumps): + def test_pdc_log(self, get_pdc_client_session, get_pdc_modules, KojiPackageSet, pickle_dumps): pickle_dumps.return_value = b'DATA' - modulemd = """ + modulemd1 = """ document: modulemd version: 2 data: @@ -146,21 +146,48 @@ data: - MIT """ - get_module.return_value = { - 'abc': 'def', - 'modulemd': modulemd, - 'rpms': [], - 'koji_tag': 'taggg', - 'uid': 'modulenamefoo:rhel:1:00000000', - 'name': 'modulenamefoo', - 'stream': 'rhel', - 'version': '1', - 'context': '00000000' - } + modulemd2 = """ +document: modulemd +version: 2 +data: + name: foo + stream: bar + version: 4 + summary: foo + description: foo + license: + module: + - MIT +""" + + get_pdc_modules.return_value = [ + { + 'abc': 'def', + 'modulemd': modulemd1, + 'rpms': [], + 'koji_tag': 'taggg', + 'uid': 'modulenamefoo:rhel:1:00000000', + 'name': 'modulenamefoo', + 'stream': 'rhel', + 'version': '1', + 'context': '00000000' + }, + { + 'abc': 'def', + 'modulemd': modulemd2, + 'rpms': [], + 'koji_tag': 'taggg', + 'uid': 'modulenamefoo:rhel:4:00000000', + 'name': 'modulenamefoo', + 'stream': 'rhel', + 'version': '4', + 'context': '00000000' + }, + ] for name, variant in self.compose.variants.items(): variant.get_modules = mock.MagicMock() if name == 'Server': - variant.modules = [{'name': 'a'}] + variant.modules = [{'name': 'modulenamefoo'}] variant.get_modules.return_value = variant.modules source_koji.populate_global_pkgset(