diff --git a/pungi/phases/pkgset/sources/source_koji.py b/pungi/phases/pkgset/sources/source_koji.py index ad32806e..4cbcf4c1 100644 --- a/pungi/phases/pkgset/sources/source_koji.py +++ b/pungi/phases/pkgset/sources/source_koji.py @@ -26,7 +26,7 @@ from kobo.shortcuts import force_list, relative_path import pungi.wrappers.kojiwrapper from pungi.wrappers.comps import CompsWrapper import pungi.phases.pkgset.pkgsets -from pungi.arch import get_valid_arches +from pungi.arch import get_valid_arches, getBaseArch from pungi.util import is_arch_multilib, retry, find_old_compose from pungi import Modulemd @@ -207,36 +207,46 @@ def get_pkgset_from_koji(compose, koji_wrapper, path_prefix): return package_sets -def _add_module_to_variant(variant, mmd, add_to_variant_modules=False): +def _add_module_to_variant(koji_wrapper, variant, build, add_to_variant_modules=False): """ - Adds module defined by Modulemd.Module `mmd` to variant. + Adds module defined by Koji build info to variant. :param Variant variant: Variant to add the module to. - :param Modulemd.Module: Modulemd instance defining the module. + :param int: build id :param bool add_to_variant_modules: Adds the modules also to variant.modules. """ - # Get the NSVC of module and handle the case where for some reason the - # name/strea/version is not set. - if not mmd.get_name() or not mmd.get_stream() or not mmd.get_version(): - raise ValueError( - "Input module %s does not name or stream or version set." - % mmd.dumps()) - nsvc_list = [mmd.get_name(), mmd.get_stream(), str(mmd.get_version())] - if mmd.get_context(): - nsvc_list.append(mmd.get_context()) - nsvc = ":".join(nsvc_list) + mmds = {} + archives = koji_wrapper.koji_proxy.listArchives(build["id"]) + for archive in archives: + if archive["btype"] != "module": + # Skip non module archives + continue + typedir = koji_wrapper.koji_module.pathinfo.typedir(build, archive["btype"]) + filename = archive["filename"] + file_path = os.path.join(typedir, filename) + try: + # If there are two dots, the arch is in the middle. MBS uploads + # files with actual architecture in the filename, but Pungi deals + # in basearch. This assumes that each arch in the build maps to a + # unique basearch. + _, arch, _ = filename.split(".") + filename = "modulemd.%s.txt" % getBaseArch(arch) + except ValueError: + pass + mmds[filename] = Modulemd.Module.new_from_file(file_path) - variant.mmds.append(mmd) + source_mmd = mmds["modulemd.txt"] + nsvc = source_mmd.dup_nsvc() + + variant.mmds.append(source_mmd) + for arch in variant.arches: + variant.arch_mmds.setdefault(arch, {})[nsvc] = mmds["modulemd.%s.txt" % arch] if add_to_variant_modules: variant.modules.append(nsvc) - -def _log_modulemd(compose, variant, mmd): - """Dump module metadata to a log file for easy inspection.""" - mmd.dump(compose.paths.log.log_file('global', 'modulemd-%s-%s' - % (variant.uid, mmd.dup_nsvc()))) + return source_mmd def _get_modules_from_koji( @@ -258,10 +268,7 @@ def _get_modules_from_koji( for module in variant.get_modules(): koji_modules = get_koji_modules(compose, koji_wrapper, event, module["name"]) for koji_module in koji_modules: - mmd = Modulemd.Module.new_from_string(koji_module["modulemd"]) - mmd.upgrade() - _add_module_to_variant(variant, mmd) - _log_modulemd(compose, variant, mmd) + mmd = _add_module_to_variant(koji_wrapper, variant, koji_module) tag = koji_module["tag"] uid = ':'.join([koji_module['name'], koji_module['stream'], @@ -422,17 +429,10 @@ def _get_modules_from_koji_tags( build = koji_proxy.getBuild(build["build_id"]) module_tag = build.get("extra", {}).get("typeinfo", {}).get( "module", {}).get("content_koji_tag", "") - modulemd = build.get("extra", {}).get("typeinfo", {}).get( - "module", {}).get("modulemd_str", "") - if not module_tag or not modulemd: - continue variant_tags[variant].append(module_tag) - mmd = Modulemd.Module.new_from_string(modulemd) - mmd.upgrade() - _add_module_to_variant(variant, mmd, True) - _log_modulemd(compose, variant, mmd) + mmd = _add_module_to_variant(koji_wrapper, variant, build, True) # Store mapping module-uid --> koji_tag into variant. # This is needed in createrepo phase where metadata is exposed by producmd diff --git a/tests/test_pkgset_source_koji.py b/tests/test_pkgset_source_koji.py index 9d350f65..6a849966 100644 --- a/tests/test_pkgset_source_koji.py +++ b/tests/test_pkgset_source_koji.py @@ -1,4 +1,3 @@ -#!/usr/bin/env python2 # -*- coding: utf-8 -*- import json @@ -132,75 +131,6 @@ class TestPopulateGlobalPkgset(helpers.PungiTestCase): with open(self.pkgset_path) as f: self.assertEqual(f.read(), 'DATA') - @unittest.skipUnless(Modulemd is not None, 'Modulemd not available') # noqa - @mock.patch('six.moves.cPickle.dumps') - @mock.patch('pungi.phases.pkgset.pkgsets.KojiPackageSet') - @mock.patch('pungi.phases.pkgset.sources.source_koji.get_koji_modules') - def test_pdc_log(self, get_koji_modules, KojiPackageSet, pickle_dumps): - - pickle_dumps.return_value = b'DATA' - - modulemd1 = """ -document: modulemd -version: 2 -data: - name: foo - stream: bar - version: 1 - summary: foo - description: foo - license: - module: - - MIT -""" - - modulemd2 = """ -document: modulemd -version: 2 -data: - name: foo - stream: bar - version: 4 - summary: foo - description: foo - license: - module: - - MIT -""" - - get_koji_modules.return_value = [ - { - 'abc': 'def', - 'modulemd': modulemd1, - 'tag': 'taggg', - 'uid': 'modulenamefoo:rhel:1:00000000', - 'name': 'modulenamefoo', - 'stream': 'rhel', - 'version': '1', - 'context': '00000000' - }, - { - 'abc': 'def', - 'modulemd': modulemd2, - 'tag': 'taggg', - 'uid': 'modulenamefoo:rhel:4:00000000', - 'name': 'modulenamefoo', - 'stream': 'rhel', - 'version': '4', - 'context': '00000000' - }, - ] - for name, variant in self.compose.variants.items(): - variant.get_modules = mock.MagicMock() - if name == 'Server': - variant.modules = [{'name': 'modulenamefoo'}] - variant.get_modules.return_value = variant.modules - - source_koji.populate_global_pkgset( - self.compose, self.koji_wrapper, '/prefix', 123456) - mmds = Modulemd.Module.new_all_from_file(self.koji_module_path) - self.assertEqual(mmds[0].get_name(), "foo") - @mock.patch('six.moves.cPickle.dumps') @mock.patch('pungi.phases.pkgset.pkgsets.KojiPackageSet') def test_populate_with_multiple_koji_tags(self, KojiPackageSet, pickle_dumps): @@ -735,5 +665,138 @@ class TestFilterByWhitelist(unittest.TestCase): ) -if __name__ == "__main__": - unittest.main() +class MockModule(object): + def __init__(self, path): + self.path = path + + def __repr__(self): + return "MockModule(%r)" % self.path + + def __eq__(self, other): + return self.path == other.path + + def dup_nsvc(self): + return "module:master:20190318.abcdef" + + +@mock.patch("pungi.Modulemd.Module.new_from_file", new=MockModule) +class TestAddModuleToVariant(unittest.TestCase): + + def setUp(self): + self.koji = mock.Mock() + self.koji.koji_module.pathinfo.typedir.return_value = "/koji" + files = ["modulemd.x86_64.txt", "modulemd.armv7hl.txt", "modulemd.txt"] + self.koji.koji_proxy.listArchives.return_value = [ + {"btype": "module", "filename": fname} for fname in files + ] + [{"btype": "foo"}] + + def test_adding_module(self): + build = {"id": 1234} + variant = mock.Mock( + arches=["armhfp", "x86_64"], mmds=[], arch_mmds={}, modules=[] + ) + + source_koji._add_module_to_variant(self.koji, variant, build) + + self.assertEqual(variant.mmds, [MockModule("/koji/modulemd.txt")]) + self.assertEqual( + variant.arch_mmds, + { + "armhfp": { + "module:master:20190318.abcdef": MockModule("/koji/modulemd.armv7hl.txt"), + }, + "x86_64": { + "module:master:20190318.abcdef": MockModule("/koji/modulemd.x86_64.txt"), + }, + }, + ) + self.assertEqual(variant.modules, []) + + def test_adding_module_to_existing(self): + build = {"id": 1234} + variant = mock.Mock( + arches=["armhfp", "x86_64"], + mmds=[MockModule("/koji/m1.txt")], + arch_mmds={ + "x86_64": {"m1:latest:20190101.cafe": MockModule("/koji/m1.x86_64.txt")} + }, + modules=["m1:latest-20190101.cafe"], + ) + + source_koji._add_module_to_variant(self.koji, variant, build) + + self.assertEqual( + variant.mmds, [MockModule("/koji/m1.txt"), MockModule("/koji/modulemd.txt")] + ) + self.assertEqual( + variant.arch_mmds, + { + "armhfp": { + "module:master:20190318.abcdef": MockModule("/koji/modulemd.armv7hl.txt"), + }, + "x86_64": { + "module:master:20190318.abcdef": MockModule("/koji/modulemd.x86_64.txt"), + "m1:latest:20190101.cafe": MockModule("/koji/m1.x86_64.txt"), + }, + }, + ) + self.assertEqual(variant.modules, ["m1:latest-20190101.cafe"]) + + def test_adding_module_with_add_module(self): + build = {"id": 1234} + variant = mock.Mock( + arches=["armhfp", "x86_64"], mmds=[], arch_mmds={}, modules=[] + ) + + source_koji._add_module_to_variant( + self.koji, variant, build, add_to_variant_modules=True + ) + + self.assertEqual(variant.mmds, [MockModule("/koji/modulemd.txt")]) + self.assertEqual( + variant.arch_mmds, + { + "armhfp": { + "module:master:20190318.abcdef": MockModule("/koji/modulemd.armv7hl.txt"), + }, + "x86_64": { + "module:master:20190318.abcdef": MockModule("/koji/modulemd.x86_64.txt"), + }, + }, + ) + self.assertEqual(variant.modules, ["module:master:20190318.abcdef"]) + + def test_adding_module_to_existing_with_add_module(self): + build = {"id": 1234} + variant = mock.Mock( + arches=["armhfp", "x86_64"], + mmds=[MockModule("/koji/m1.txt")], + arch_mmds={ + "x86_64": {"m1:latest:20190101.cafe": MockModule("/koji/m1.x86_64.txt")} + }, + modules=["m1:latest-20190101.cafe"], + ) + + source_koji._add_module_to_variant( + self.koji, variant, build, add_to_variant_modules=True + ) + + self.assertEqual( + variant.mmds, [MockModule("/koji/m1.txt"), MockModule("/koji/modulemd.txt")] + ) + self.assertEqual( + variant.arch_mmds, + { + "armhfp": { + "module:master:20190318.abcdef": MockModule("/koji/modulemd.armv7hl.txt"), + }, + "x86_64": { + "module:master:20190318.abcdef": MockModule("/koji/modulemd.x86_64.txt"), + "m1:latest:20190101.cafe": MockModule("/koji/m1.x86_64.txt"), + }, + }, + ) + self.assertEqual( + variant.modules, + ["m1:latest-20190101.cafe", "module:master:20190318.abcdef"], + )