diff --git a/pungi/phases/createrepo.py b/pungi/phases/createrepo.py index c9ac4746..83028bb0 100644 --- a/pungi/phases/createrepo.py +++ b/pungi/phases/createrepo.py @@ -31,7 +31,7 @@ from kobo.shortcuts import run, relative_path from ..wrappers.scm import get_dir_from_scm from ..wrappers.createrepo import CreaterepoWrapper from .base import PhaseBase -from ..util import get_arch_variant_data, temp_dir +from ..util import get_arch_variant_data, temp_dir, read_single_module_stream_from_file from ..module_util import Modulemd, collect_module_defaults import productmd.rpms @@ -268,7 +268,7 @@ def create_variant_repo( compose.log_debug("Adding extra modulemd for %s.%s", variant.uid, arch) dirname = compose.paths.work.tmp_dir(variant=variant, create_dir=False) for filepath in glob.glob(os.path.join(dirname, arch) + "/*.yaml"): - module_stream = Modulemd.ModuleStream.read_file(filepath, strict=True) + module_stream = read_single_module_stream_from_file(filepath) if not mod_index.add_module_stream(module_stream): raise RuntimeError( "Failed parsing modulemd data from %s" % filepath diff --git a/pungi/phases/pkgset/sources/source_koji.py b/pungi/phases/pkgset/sources/source_koji.py index 2ee955ce..1219677c 100644 --- a/pungi/phases/pkgset/sources/source_koji.py +++ b/pungi/phases/pkgset/sources/source_koji.py @@ -29,7 +29,13 @@ from pungi.wrappers.comps import CompsWrapper from pungi.wrappers.mbs import MBSWrapper import pungi.phases.pkgset.pkgsets from pungi.arch import getBaseArch -from pungi.util import retry, get_arch_variant_data, get_variant_data +from pungi.util import ( + retry, + get_arch_variant_data, + get_variant_data, + read_single_module_stream_from_file, + read_single_module_stream_from_string, +) from pungi.module_util import Modulemd from pungi.phases.pkgset.common import MaterializedPackageSet, get_all_arches @@ -258,17 +264,12 @@ def _add_module_to_variant( compose.log_debug("Module %s is filtered from %s.%s", nsvc, variant, arch) continue - try: - mmd = Modulemd.ModuleStream.read_file( - mmds["modulemd.%s.txt" % arch], strict=True - ) - variant.arch_mmds.setdefault(arch, {})[nsvc] = mmd + mod_stream = read_single_module_stream_from_file( + mmds["modulemd.%s.txt" % arch], compose, arch, build + ) + if mod_stream: added = True - except KeyError: - # There is no modulemd for this arch. This could mean an arch was - # added to the compose after the module was built. We don't want to - # process this, let's skip this module. - pass + variant.arch_mmds.setdefault(arch, {})[nsvc] = mod_stream if not added: # The module is filtered on all arches of this variant. @@ -348,9 +349,7 @@ def _add_scratch_modules_to_variant( tag_to_mmd.setdefault(tag, {}) for arch in variant.arches: try: - mmd = Modulemd.ModuleStream.read_string( - final_modulemd[arch], strict=True - ) + mmd = read_single_module_stream_from_string(final_modulemd[arch]) variant.arch_mmds.setdefault(arch, {})[nsvc] = mmd except KeyError: continue diff --git a/pungi/util.py b/pungi/util.py index c5717bde..da5e82f2 100644 --- a/pungi/util.py +++ b/pungi/util.py @@ -34,6 +34,7 @@ import kobo.conf from kobo.shortcuts import run, force_list from kobo.threads import WorkerThread, ThreadPool from productmd.common import get_major_version +from pungi.module_util import Modulemd # Patterns that match all names of debuginfo packages DEBUG_PATTERNS = ["*-debuginfo", "*-debuginfo-*", "*-debugsource"] @@ -1034,6 +1035,46 @@ def load_config(file_path, defaults={}): return conf +def _read_single_module_stream( + file_or_string, compose=None, arch=None, build=None, is_file=True +): + try: + mod_index = Modulemd.ModuleIndex.new() + if is_file: + mod_index.update_from_file(file_or_string, True) + else: + mod_index.update_from_string(file_or_string, True) + mod_names = mod_index.get_module_names() + emit_warning = False + if len(mod_names) > 1: + emit_warning = True + mod_streams = mod_index.get_module(mod_names[0]).get_all_streams() + if len(mod_streams) > 1: + emit_warning = True + if emit_warning and compose: + compose.log_warning( + "Multiple modules/streams for arch: %s. Build: %s. " + "Processing first module/stream only.", + arch, + build, + ) + return mod_streams[0] + except (KeyError, IndexError): + # There is no modulemd for this arch. This could mean an arch was + # added to the compose after the module was built. We don't want to + # process this, let's skip this module. + if compose: + compose.log_info("Skipping arch: %s. Build: %s", arch, build) + + +def read_single_module_stream_from_file(*args, **kwargs): + return _read_single_module_stream(*args, is_file=True, **kwargs) + + +def read_single_module_stream_from_string(*args, **kwargs): + return _read_single_module_stream(*args, is_file=False, **kwargs) + + @contextlib.contextmanager def as_local_file(url): """If URL points to a file over HTTP, the file will be downloaded locally diff --git a/tests/fixtures/mmds/m1.x86_64.txt b/tests/fixtures/mmds/m1.x86_64.txt new file mode 100644 index 00000000..e1989733 --- /dev/null +++ b/tests/fixtures/mmds/m1.x86_64.txt @@ -0,0 +1,20 @@ +--- +document: modulemd +version: 2 +data: + name: m1 + stream: latest + version: 20190101 + context: cafe + arch: x86_64 + summary: Dummy module + description: Dummy module + license: + module: + - Beerware + content: + - Beerware + artifacts: + rpms: + - foobar-0:1.0-1.noarch +... diff --git a/tests/fixtures/mmds/modulemd.armv7hl.txt b/tests/fixtures/mmds/modulemd.armv7hl.txt new file mode 100644 index 00000000..e03147d2 --- /dev/null +++ b/tests/fixtures/mmds/modulemd.armv7hl.txt @@ -0,0 +1,20 @@ +--- +document: modulemd +version: 2 +data: + name: module + stream: master + version: 20190318 + context: abcdef + arch: armhfp + summary: Dummy module + description: Dummy module + license: + module: + - Beerware + content: + - Beerware + artifacts: + rpms: + - foobar-0:1.0-1.noarch +... diff --git a/tests/fixtures/mmds/modulemd.x86_64.txt b/tests/fixtures/mmds/modulemd.x86_64.txt new file mode 100644 index 00000000..b7e3761c --- /dev/null +++ b/tests/fixtures/mmds/modulemd.x86_64.txt @@ -0,0 +1,20 @@ +--- +document: modulemd +version: 2 +data: + name: module + stream: master + version: 20190318 + context: abcdef + arch: x86_64 + summary: Dummy module + description: Dummy module + license: + module: + - Beerware + content: + - Beerware + artifacts: + rpms: + - foobar-0:1.0-1.noarch +... diff --git a/tests/fixtures/mmds/scratch-module.x86_64.txt b/tests/fixtures/mmds/scratch-module.x86_64.txt new file mode 100644 index 00000000..8a13926b --- /dev/null +++ b/tests/fixtures/mmds/scratch-module.x86_64.txt @@ -0,0 +1,20 @@ +--- +document: modulemd +version: 2 +data: + name: scratch-module + stream: master + version: 20200710 + context: abcdef + arch: x86_64 + summary: Dummy module + description: Dummy module + license: + module: + - Beerware + content: + - Beerware + artifacts: + rpms: + - foobar-0:1.0-1.noarch +... diff --git a/tests/test_pkgset_source_koji.py b/tests/test_pkgset_source_koji.py index f45a5ac4..c4106808 100644 --- a/tests/test_pkgset_source_koji.py +++ b/tests/test_pkgset_source_koji.py @@ -14,7 +14,9 @@ except ImportError: from pungi.phases.pkgset.sources import source_koji from tests import helpers from pungi.module_util import Modulemd +from pungi.util import read_single_module_stream_from_file +MMDS_DIR = os.path.join(helpers.FIXTURE_DIR, "mmds") EVENT_INFO = {"id": 15681980, "ts": 1460956382.81936} TAG_INFO = { "maven_support": False, @@ -672,24 +674,12 @@ class TestFilterByWhitelist(unittest.TestCase): self.assertEqual(expected, set()) -class MockModule(object): - def __init__(self, path, strict=True): - self.path = path - - def __repr__(self): - return "MockModule(%r)" % self.path - - def __eq__(self, other): - return self.path == other.path - - -@mock.patch("pungi.module_util.Modulemd.ModuleStream.read_file", new=MockModule) @unittest.skipIf(Modulemd is None, "Skipping tests, no module support") class TestAddModuleToVariant(helpers.PungiTestCase): def setUp(self): super(TestAddModuleToVariant, self).setUp() self.koji = mock.Mock() - self.koji.koji_module.pathinfo.typedir.return_value = "/koji" + self.koji.koji_module.pathinfo.typedir.return_value = MMDS_DIR files = ["modulemd.x86_64.txt", "modulemd.armv7hl.txt", "modulemd.txt"] self.koji.koji_proxy.listArchives.return_value = [ {"btype": "module", "filename": fname} for fname in files @@ -713,50 +703,35 @@ class TestAddModuleToVariant(helpers.PungiTestCase): source_koji._add_module_to_variant(self.koji, variant, self.buildinfo) - self.assertEqual( - variant.arch_mmds, - { - "armhfp": { - "module:master:20190318:abcdef": MockModule( - "/koji/modulemd.armv7hl.txt" - ), - }, - "x86_64": { - "module:master:20190318:abcdef": MockModule( - "/koji/modulemd.x86_64.txt" - ), - }, - }, - ) + mod1 = variant.arch_mmds["armhfp"]["module:master:20190318:abcdef"] + self.assertEqual(mod1.get_NSVCA(), "module:master:20190318:abcdef:armhfp") + mod2 = variant.arch_mmds["x86_64"]["module:master:20190318:abcdef"] + self.assertEqual(mod2.get_NSVCA(), "module:master:20190318:abcdef:x86_64") + self.assertEqual(len(variant.arch_mmds), 2) self.assertEqual(variant.modules, []) def test_adding_module_to_existing(self): variant = mock.Mock( arches=["armhfp", "x86_64"], arch_mmds={ - "x86_64": {"m1:latest:20190101:cafe": MockModule("/koji/m1.x86_64.txt")} + "x86_64": { + "m1:latest:20190101:cafe": read_single_module_stream_from_file( + os.path.join(MMDS_DIR, "m1.x86_64.txt") + ) + } }, modules=[{"name": "m1:latest-20190101:cafe", "glob": False}], ) source_koji._add_module_to_variant(self.koji, variant, self.buildinfo) - self.assertEqual( - variant.arch_mmds, - { - "armhfp": { - "module:master:20190318:abcdef": MockModule( - "/koji/modulemd.armv7hl.txt" - ), - }, - "x86_64": { - "module:master:20190318:abcdef": MockModule( - "/koji/modulemd.x86_64.txt" - ), - "m1:latest:20190101:cafe": MockModule("/koji/m1.x86_64.txt"), - }, - }, - ) + mod1 = variant.arch_mmds["armhfp"]["module:master:20190318:abcdef"] + self.assertEqual(mod1.get_NSVCA(), "module:master:20190318:abcdef:armhfp") + mod2 = variant.arch_mmds["x86_64"]["module:master:20190318:abcdef"] + self.assertEqual(mod2.get_NSVCA(), "module:master:20190318:abcdef:x86_64") + mod3 = variant.arch_mmds["x86_64"]["m1:latest:20190101:cafe"] + self.assertEqual(mod3.get_NSVCA(), "m1:latest:20190101:cafe:x86_64") + self.assertEqual( variant.modules, [{"name": "m1:latest-20190101:cafe", "glob": False}] ) @@ -768,21 +743,11 @@ class TestAddModuleToVariant(helpers.PungiTestCase): self.koji, variant, self.buildinfo, add_to_variant_modules=True ) - self.assertEqual( - variant.arch_mmds, - { - "armhfp": { - "module:master:20190318:abcdef": MockModule( - "/koji/modulemd.armv7hl.txt" - ), - }, - "x86_64": { - "module:master:20190318:abcdef": MockModule( - "/koji/modulemd.x86_64.txt" - ), - }, - }, - ) + mod1 = variant.arch_mmds["armhfp"]["module:master:20190318:abcdef"] + self.assertEqual(mod1.get_NSVCA(), "module:master:20190318:abcdef:armhfp") + mod2 = variant.arch_mmds["x86_64"]["module:master:20190318:abcdef"] + self.assertEqual(mod2.get_NSVCA(), "module:master:20190318:abcdef:x86_64") + self.assertEqual( variant.modules, [{"name": "module:master:20190318:abcdef", "glob": False}] ) @@ -791,7 +756,11 @@ class TestAddModuleToVariant(helpers.PungiTestCase): variant = mock.Mock( arches=["armhfp", "x86_64"], arch_mmds={ - "x86_64": {"m1:latest:20190101:cafe": MockModule("/koji/m1.x86_64.txt")} + "x86_64": { + "m1:latest:20190101:cafe": read_single_module_stream_from_file( + os.path.join(MMDS_DIR, "m1.x86_64.txt") + ) + } }, modules=[{"name": "m1:latest-20190101:cafe", "glob": False}], ) @@ -800,22 +769,13 @@ class TestAddModuleToVariant(helpers.PungiTestCase): self.koji, variant, self.buildinfo, add_to_variant_modules=True ) - self.assertEqual( - variant.arch_mmds, - { - "armhfp": { - "module:master:20190318:abcdef": MockModule( - "/koji/modulemd.armv7hl.txt" - ), - }, - "x86_64": { - "module:master:20190318:abcdef": MockModule( - "/koji/modulemd.x86_64.txt" - ), - "m1:latest:20190101:cafe": MockModule("/koji/m1.x86_64.txt"), - }, - }, - ) + mod1 = variant.arch_mmds["armhfp"]["module:master:20190318:abcdef"] + self.assertEqual(mod1.get_NSVCA(), "module:master:20190318:abcdef:armhfp") + mod2 = variant.arch_mmds["x86_64"]["module:master:20190318:abcdef"] + self.assertEqual(mod2.get_NSVCA(), "module:master:20190318:abcdef:x86_64") + mod3 = variant.arch_mmds["x86_64"]["m1:latest:20190101:cafe"] + self.assertEqual(mod3.get_NSVCA(), "m1:latest:20190101:cafe:x86_64") + self.assertEqual( variant.modules, [ @@ -891,12 +851,8 @@ class MockMBS(object): return {"id": 1, "koji_tag": "scratch-module-tag", "name": "scratch-module"} def final_modulemd(self, module_build_id): - return {"x86_64": ""} - - -class MockMmd(object): - def __init__(self, mmd, strict=True): - pass + with open(os.path.join(MMDS_DIR, "scratch-module.x86_64.txt")) as f: + return {"x86_64": f.read()} @mock.patch("pungi.phases.pkgset.sources.source_koji.MBSWrapper", new=MockMBS) @@ -909,10 +865,7 @@ class TestAddScratchModuleToVariant(helpers.PungiTestCase): ) self.nsvc = "scratch-module:master:20200710:abcdef" - @mock.patch( - "pungi.phases.pkgset.sources.source_koji.Modulemd.ModuleStream.read_string" - ) - def test_adding_scratch_module(self, mock_mmd): + def test_adding_scratch_module(self): variant = mock.Mock( arches=["armhfp", "x86_64"], arch_mmds={}, @@ -927,11 +880,16 @@ class TestAddScratchModuleToVariant(helpers.PungiTestCase): self.compose, variant, scratch_modules, variant_tags, tag_to_mmd ) self.assertEqual(variant_tags, {variant: ["scratch-module-tag"]}) + self.assertEqual( - variant.arch_mmds, {"x86_64": {self.nsvc: mock_mmd.return_value}} + variant.arch_mmds["x86_64"][self.nsvc].get_NSVCA(), + "scratch-module:master:20200710:abcdef:x86_64", ) + + self.assertTrue(isinstance(tag_to_mmd["scratch-module-tag"]["x86_64"], set)) self.assertEqual( - tag_to_mmd, {"scratch-module-tag": {"x86_64": set([mock_mmd.return_value])}} + list(tag_to_mmd["scratch-module-tag"]["x86_64"])[0].get_NSVCA(), + "scratch-module:master:20200710:abcdef:x86_64", ) self.assertEqual(variant.modules, [])