From e3aa2f769bb8106cccb1a463057d200df8df515e Mon Sep 17 00:00:00 2001 From: Ondrej Nosek Date: Tue, 20 Mar 2018 08:52:49 +0100 Subject: [PATCH] Write module metadata Signed-off-by: Ondrej Nosek --- pungi/phases/createrepo.py | 86 ++++++++++++++++++---- pungi/phases/pkgset/sources/source_koji.py | 19 ++++- pungi/wrappers/variants.py | 1 + tests/helpers.py | 1 + tests/test_createrepophase.py | 57 +++++++------- tests/test_pkgset_source_koji.py | 2 +- 6 files changed, 124 insertions(+), 42 deletions(-) diff --git a/pungi/phases/createrepo.py b/pungi/phases/createrepo.py index 662fab58..1ce2b171 100644 --- a/pungi/phases/createrepo.py +++ b/pungi/phases/createrepo.py @@ -19,12 +19,13 @@ __all__ = ( ) -import os -import glob -import shutil -import threading import copy import errno +import glob +import os +import shutil +import threading +import xml.dom.minidom from kobo.threads import ThreadPool, WorkerThread from kobo.shortcuts import run, relative_path @@ -36,6 +37,7 @@ from ..util import find_old_compose, temp_dir, get_arch_variant_data from pungi import Modulemd import productmd.rpms +import productmd.modules createrepo_lock = threading.Lock() @@ -48,6 +50,7 @@ class CreaterepoPhase(PhaseBase): def __init__(self, compose): PhaseBase.__init__(self, compose) self.pool = ThreadPool(logger=self.compose._logger) + self.modules_metadata = ModulesMetadata(compose) def validate(self): errors = [] @@ -70,22 +73,26 @@ class CreaterepoPhase(PhaseBase): for variant in self.compose.get_variants(): if variant.is_empty: continue - self.pool.queue_put((self.compose, None, variant, "srpm")) + self.pool.queue_put((self.compose, None, variant, "srpm", self.modules_metadata)) for arch in variant.arches: - self.pool.queue_put((self.compose, arch, variant, "rpm")) - self.pool.queue_put((self.compose, arch, variant, "debuginfo")) + self.pool.queue_put((self.compose, arch, variant, "rpm", self.modules_metadata)) + self.pool.queue_put((self.compose, arch, variant, "debuginfo", self.modules_metadata)) self.pool.start() + def stop(self): + super(CreaterepoPhase, self).stop() + self.modules_metadata.write_modules_metadata() -def create_variant_repo(compose, arch, variant, pkg_type): + +def create_variant_repo(compose, arch, variant, pkg_type, modules_metadata=None): types = { 'rpm': ('binary', - lambda: compose.paths.compose.repository(arch=arch, variant=variant)), + lambda **kwargs: compose.paths.compose.repository(arch=arch, variant=variant, **kwargs)), 'srpm': ('source', - lambda: compose.paths.compose.repository(arch='src', variant=variant)), + lambda **kwargs: compose.paths.compose.repository(arch='src', variant=variant, **kwargs)), 'debuginfo': ('debug', - lambda: compose.paths.compose.debug_repository(arch=arch, variant=variant)), + lambda **kwargs: compose.paths.compose.debug_repository(arch=arch, variant=variant, **kwargs)), } if variant.is_empty or (arch is None and pkg_type != 'srpm'): @@ -186,7 +193,8 @@ def create_variant_repo(compose, arch, variant, pkg_type): # call modifyrepo to inject modulemd if needed if arch in variant.arch_mmds and Modulemd is not None: modules = [] - for mmd in variant.arch_mmds[arch].values(): + metadata = [] + for module_id, mmd in variant.arch_mmds[arch].items(): # Create copy of architecture specific mmd to filter out packages # which are not part of this particular repo. repo_mmd = Modulemd.Module.new_from_string(mmd.dumps()) @@ -196,11 +204,18 @@ def create_variant_repo(compose, arch, variant, pkg_type): if not artifacts or artifacts.size() == 0: continue + module_rpms = set() repo_artifacts = Modulemd.SimpleSet() for rpm_nevra in rpm_nevras: if artifacts.contains(rpm_nevra): repo_artifacts.add(rpm_nevra) + module_rpms.add(rpm_nevra) repo_mmd.set_rpm_artifacts(repo_artifacts) + if module_rpms: # do not create metadata if there is empty rpm list + if modules_metadata: # some unittests call this method without parameter modules_metadata and its default is None + metadata.append((module_id, module_rpms)) + else: + raise AttributeError("module_metadata parameter was not passed and it is needed for module processing") modules.append(repo_mmd) with temp_dir() as tmp_dir: @@ -214,13 +229,26 @@ def create_variant_repo(compose, arch, variant, pkg_type): arch, "modifyrepo-modules-%s" % variant) run(cmd, logfile=log_file, show_cmd=True) + for module_id, module_rpms in metadata: + modulemd_path = os.path.join(types[pkg_type][1](relative=True), find_file_in_repodata(repo_dir, 'modules')) + modules_metadata.prepare_module_metadata(variant, arch, module_id, modulemd_path, types[pkg_type][0], list(module_rpms)) + compose.log_info("[DONE ] %s" % msg) +def find_file_in_repodata(repo_path, type_): + dom = xml.dom.minidom.parse(os.path.join(repo_path, 'repodata', 'repomd.xml')) + for entry in dom.getElementsByTagName('data'): + if entry.getAttribute('type') == type_: + return entry.getElementsByTagName('location')[0].getAttribute('href') + entry.unlink() + raise RuntimeError('No such file in repodata: %s' % type_) + + class CreaterepoThread(WorkerThread): def process(self, item, num): - compose, arch, variant, pkg_type = item - create_variant_repo(compose, arch, variant, pkg_type=pkg_type) + compose, arch, variant, pkg_type, modules_metadata = item + create_variant_repo(compose, arch, variant, pkg_type=pkg_type, modules_metadata=modules_metadata) def get_productids_from_scm(compose): @@ -317,3 +345,33 @@ def _has_deltas(compose, variant, arch): if isinstance(compose.conf.get(key), bool): return compose.conf[key] return any(get_arch_variant_data(compose.conf, key, arch, variant)) + + +class ModulesMetadata(object): + def __init__(self, compose): + # Prepare empty module metadata + self.compose = compose + self.modules_metadata_file = self.compose.paths.compose.metadata("modules.json") + self.productmd_modules_metadata = productmd.modules.Modules() + self.productmd_modules_metadata.compose.id = copy.copy(self.compose.compose_id) + self.productmd_modules_metadata.compose.type = copy.copy(self.compose.compose_type) + self.productmd_modules_metadata.compose.date = copy.copy(self.compose.compose_date) + self.productmd_modules_metadata.compose.respin = copy.copy(self.compose.compose_respin) + + def write_modules_metadata(self): + """ + flush modules metadata into file + """ + self.compose.log_info("Writing modules metadata: %s" % self.modules_metadata_file) + self.productmd_modules_metadata.dump(self.modules_metadata_file) + + def prepare_module_metadata(self, variant, arch, module_id, modulemd_path, category, module_rpms): + """ + find uid/koji_tag which is correstponding with variant object and + add record(s) into module metadata structure + """ + for uid, koji_tag in variant.module_uid_to_koji_tag.items(): + uid_dict = self.productmd_modules_metadata.parse_uid(uid) + if module_id == '{module_name}-{stream}'.format(**uid_dict): + self.productmd_modules_metadata.add(variant.uid, arch, uid, koji_tag, modulemd_path, category, module_rpms) + break diff --git a/pungi/phases/pkgset/sources/source_koji.py b/pungi/phases/pkgset/sources/source_koji.py index eb900ab4..65d0a407 100644 --- a/pungi/phases/pkgset/sources/source_koji.py +++ b/pungi/phases/pkgset/sources/source_koji.py @@ -254,8 +254,13 @@ def _get_modules_from_pdc(compose, session, variant, variant_tags): _add_module_to_variant(variant, mmd, pdc_module["rpms"]) tag = pdc_module["koji_tag"] + uid = pdc_module["variant_uid"] variant_tags[variant].append(tag) + # Store mapping module-uid --> koji_tag into variant. + # This is needed in createrepo phase where metadata is exposed by producmd + variant.module_uid_to_koji_tag[uid] = tag + module_msg = "Module {module} in variant {variant} will use Koji tag {tag}.".format( variant=variant, tag=tag, module=module["name"]) compose.log_info("%s" % module_msg) @@ -334,12 +339,24 @@ def _get_modules_from_koji_tags( mmd.upgrade() _add_module_to_variant(variant, mmd, rpms, True) + # Store mapping module-uid --> koji_tag into variant. + # This is needed in createrepo phase where metadata is exposed by producmd + module_data = build.get("extra", {}).get("typeinfo", {}).get("module", {}) + try: + uid = "{name}:{stream}".format(**module_data) + except KeyError as e: + raise KeyError("Unable to create uid in format name:stream %s" % e) + if module_data.get("version"): + uid += ":{version}".format(**module_data) + if module_data.get("context"): + uid += ":{context}".format(**module_data) + variant.module_uid_to_koji_tag[uid] = module_tag + module_msg = "Module {module} in variant {variant} will use Koji tag {tag}.".format( variant=variant, tag=module_tag, module=build["nvr"]) compose.log_info("%s" % module_msg) - def populate_global_pkgset(compose, koji_wrapper, path_prefix, event_id): all_arches = set(["src"]) for arch in compose.get_arches(): diff --git a/pungi/wrappers/variants.py b/pungi/wrappers/variants.py index de304b2a..87da5d5b 100755 --- a/pungi/wrappers/variants.py +++ b/pungi/wrappers/variants.py @@ -236,6 +236,7 @@ class Variant(object): self.pkgset = None self.mmds = [] self.arch_mmds = {} + self.module_uid_to_koji_tag = {} def __getitem__(self, name): return self.variants[name] diff --git a/tests/helpers.py b/tests/helpers.py index a24a845f..34d34fdc 100644 --- a/tests/helpers.py +++ b/tests/helpers.py @@ -38,6 +38,7 @@ class MockVariant(mock.Mock): self.parent = kwargs.get('parent', None) self.mmds = [] self.arch_mmds = {} + self.module_uid_to_koji_tag = {} self.variants = {} self.pkgset = mock.Mock(rpms_by_arch={}) self.modules = None diff --git a/tests/test_createrepophase.py b/tests/test_createrepophase.py index b9a96c31..6a2847a0 100644 --- a/tests/test_createrepophase.py +++ b/tests/test_createrepophase.py @@ -16,7 +16,8 @@ sys.path.insert(0, os.path.join(os.path.dirname(__file__), "..")) from pungi.phases.createrepo import (CreaterepoPhase, create_variant_repo, - get_productids_from_scm) + get_productids_from_scm, + ModulesMetadata) from tests.helpers import DummyCompose, PungiTestCase, copy_fixture, touch from pungi import Modulemd @@ -75,19 +76,19 @@ class TestCreaterepoPhase(PungiTestCase): self.assertEqual(len(pool.add.mock_calls), 5) self.assertItemsEqual( pool.queue_put.mock_calls, - [mock.call((compose, 'x86_64', compose.variants['Server'], 'rpm')), - mock.call((compose, 'x86_64', compose.variants['Server'], 'debuginfo')), - mock.call((compose, 'amd64', compose.variants['Server'], 'rpm')), - mock.call((compose, 'amd64', compose.variants['Server'], 'debuginfo')), - mock.call((compose, None, compose.variants['Server'], 'srpm')), - mock.call((compose, 'x86_64', compose.variants['Everything'], 'rpm')), - mock.call((compose, 'x86_64', compose.variants['Everything'], 'debuginfo')), - mock.call((compose, 'amd64', compose.variants['Everything'], 'rpm')), - mock.call((compose, 'amd64', compose.variants['Everything'], 'debuginfo')), - mock.call((compose, None, compose.variants['Everything'], 'srpm')), - mock.call((compose, 'amd64', compose.variants['Client'], 'rpm')), - mock.call((compose, 'amd64', compose.variants['Client'], 'debuginfo')), - mock.call((compose, None, compose.variants['Client'], 'srpm'))]) + [mock.call((compose, 'x86_64', compose.variants['Server'], 'rpm', phase.modules_metadata)), + mock.call((compose, 'x86_64', compose.variants['Server'], 'debuginfo', phase.modules_metadata)), + mock.call((compose, 'amd64', compose.variants['Server'], 'rpm', phase.modules_metadata)), + mock.call((compose, 'amd64', compose.variants['Server'], 'debuginfo', phase.modules_metadata)), + mock.call((compose, None, compose.variants['Server'], 'srpm', phase.modules_metadata)), + mock.call((compose, 'x86_64', compose.variants['Everything'], 'rpm', phase.modules_metadata)), + mock.call((compose, 'x86_64', compose.variants['Everything'], 'debuginfo', phase.modules_metadata)), + mock.call((compose, 'amd64', compose.variants['Everything'], 'rpm', phase.modules_metadata)), + mock.call((compose, 'amd64', compose.variants['Everything'], 'debuginfo', phase.modules_metadata)), + mock.call((compose, None, compose.variants['Everything'], 'srpm', phase.modules_metadata)), + mock.call((compose, 'amd64', compose.variants['Client'], 'rpm', phase.modules_metadata)), + mock.call((compose, 'amd64', compose.variants['Client'], 'debuginfo', phase.modules_metadata)), + mock.call((compose, None, compose.variants['Client'], 'srpm', phase.modules_metadata))]) @mock.patch('pungi.checks.get_num_cpus') @mock.patch('pungi.phases.createrepo.ThreadPool') @@ -105,16 +106,16 @@ class TestCreaterepoPhase(PungiTestCase): self.assertEqual(len(pool.add.mock_calls), 5) self.assertItemsEqual( pool.queue_put.mock_calls, - [mock.call((compose, 'x86_64', compose.variants['Server'], 'rpm')), - mock.call((compose, 'x86_64', compose.variants['Server'], 'debuginfo')), - mock.call((compose, 'amd64', compose.variants['Server'], 'rpm')), - mock.call((compose, 'amd64', compose.variants['Server'], 'debuginfo')), - mock.call((compose, None, compose.variants['Server'], 'srpm')), - mock.call((compose, 'x86_64', compose.variants['Everything'], 'rpm')), - mock.call((compose, 'x86_64', compose.variants['Everything'], 'debuginfo')), - mock.call((compose, 'amd64', compose.variants['Everything'], 'rpm')), - mock.call((compose, 'amd64', compose.variants['Everything'], 'debuginfo')), - mock.call((compose, None, compose.variants['Everything'], 'srpm'))]) + [mock.call((compose, 'x86_64', compose.variants['Server'], 'rpm', phase.modules_metadata)), + mock.call((compose, 'x86_64', compose.variants['Server'], 'debuginfo', phase.modules_metadata)), + mock.call((compose, 'amd64', compose.variants['Server'], 'rpm', phase.modules_metadata)), + mock.call((compose, 'amd64', compose.variants['Server'], 'debuginfo', phase.modules_metadata)), + mock.call((compose, None, compose.variants['Server'], 'srpm', phase.modules_metadata)), + mock.call((compose, 'x86_64', compose.variants['Everything'], 'rpm', phase.modules_metadata)), + mock.call((compose, 'x86_64', compose.variants['Everything'], 'debuginfo', phase.modules_metadata)), + mock.call((compose, 'amd64', compose.variants['Everything'], 'rpm', phase.modules_metadata)), + mock.call((compose, 'amd64', compose.variants['Everything'], 'debuginfo', phase.modules_metadata)), + mock.call((compose, None, compose.variants['Everything'], 'srpm', phase.modules_metadata))]) class TestCreateVariantRepo(PungiTestCase): @@ -750,10 +751,11 @@ class TestCreateVariantRepo(PungiTestCase): [mock.call(repodata_dir, ANY, compress_type='gz', mdtype='modules')]) @unittest.skipUnless(Modulemd is not None, 'Skipped test, no module support.') + @mock.patch('pungi.phases.createrepo.find_file_in_repodata') @mock.patch('pungi.phases.createrepo.run') @mock.patch('pungi.phases.createrepo.CreaterepoWrapper') def test_variant_repo_modules_artifacts( - self, CreaterepoWrapperCls, run): + self, CreaterepoWrapperCls, run, modulemd_filename): compose = DummyCompose(self.topdir, { 'createrepo_checksum': 'sha256', }) @@ -784,7 +786,10 @@ class TestCreateVariantRepo(PungiTestCase): compose.paths.compose.os_tree('x86_64', compose.variants['Server']), 'repodata') - create_variant_repo(compose, 'x86_64', compose.variants['Server'], 'rpm') + modules_metadata = ModulesMetadata(compose) + + modulemd_filename.return_value = "Server/x86_64/os/repodata/3511d16a723e1bd69826e591508f07e377d2212769b59178a9-modules.yaml.gz" + create_variant_repo(compose, 'x86_64', compose.variants['Server'], 'rpm', modules_metadata) self.assertItemsEqual( repo.get_modifyrepo_cmd.mock_calls, diff --git a/tests/test_pkgset_source_koji.py b/tests/test_pkgset_source_koji.py index 0c803fcf..fcd8710a 100644 --- a/tests/test_pkgset_source_koji.py +++ b/tests/test_pkgset_source_koji.py @@ -142,7 +142,7 @@ data: - MIT """ - get_module.return_value = {'abc': 'def', 'modulemd': modulemd, 'rpms': [], 'koji_tag': 'taggg'} + get_module.return_value = {'abc': 'def', 'modulemd': modulemd, 'rpms': [], 'koji_tag': 'taggg', 'variant_uid': 'modulenamefoo-rhel-1'} for name, variant in self.compose.variants.items(): variant.get_modules = mock.MagicMock() if name == 'Server':