From f7167fa3b6dea8d09d26105bfb1b62bd36e42f0d Mon Sep 17 00:00:00 2001 From: Haibo Lin Date: Mon, 6 Jul 2020 11:02:55 +0800 Subject: [PATCH] Allow including scratch module builds JIRA: RHELCMP-439 Signed-off-by: Haibo Lin --- doc/configuration.rst | 9 +++ pungi/checks.py | 9 +++ pungi/paths.py | 2 + pungi/phases/pkgset/sources/source_koji.py | 56 ++++++++++++++++++- pungi/wrappers/mbs.py | 43 ++++++++++++++ tests/test_pkgset_source_koji.py | 65 ++++++++++++++++++++++ 6 files changed, 183 insertions(+), 1 deletion(-) create mode 100644 pungi/wrappers/mbs.py diff --git a/doc/configuration.rst b/doc/configuration.rst index 43063c10..c6a817a5 100644 --- a/doc/configuration.rst +++ b/doc/configuration.rst @@ -190,6 +190,11 @@ Options (*str*) -- Allows to set default compose type. Type set via a command-line option overwrites this. +**mbs_api_url** + (*str*) -- URL to Module Build Service (MBS) API. + For example ``https://mbs.example.com/module-build-service/2``. + This is required by ``pkgset_scratch_modules``. + Example ------- :: @@ -548,6 +553,10 @@ Options (*dict*) -- A mapping of architectures to repositories with RPMs: ``{arch: [repo]}``. Only use when ``pkgset_source = "repos"``. +**pkgset_scratch_modules** + (*dict*) -- A mapping of variants to scratch module builds: ``{variant: + [N:S:V:C]}``. Requires ``mbs_api_url``. + **pkgset_exclusive_arch_considers_noarch** = True (*bool*) -- If a package includes ``noarch`` in its ``ExclusiveArch`` tag, it will be included in all architectures since ``noarch`` is compatible diff --git a/pungi/checks.py b/pungi/checks.py index e98db28e..4adf2f64 100644 --- a/pungi/checks.py +++ b/pungi/checks.py @@ -786,6 +786,14 @@ def make_schema(): "type": "boolean", "default": True, }, + "pkgset_scratch_modules": { + "type": "object", + "patternProperties": { + "^.+$": {"$ref": "#/definitions/list_of_strings"} + }, + "additionalProperties": False, + }, + "mbs_api_url": {"type": "string"}, "disc_types": {"type": "object", "default": {}}, "paths_module": {"type": "string"}, "skip_phases": { @@ -1258,6 +1266,7 @@ CONFIG_DEPS = { "conflicts": ((lambda x: not x, ["base_product_name", "base_product_short"]),), }, "product_id": {"conflicts": [(lambda x: not x, ["product_id_allow_missing"])]}, + "pkgset_scratch_modules": {"requires": ((lambda x: x, ["mbs_api_url"]),)}, "pkgset_source": { "requires": [(lambda x: x == "repos", ["pkgset_repos"])], "conflicts": [ diff --git a/pungi/paths.py b/pungi/paths.py index 1531ad05..43b084c7 100644 --- a/pungi/paths.py +++ b/pungi/paths.py @@ -490,6 +490,8 @@ class WorkPaths(object): def module_defaults_dir(self, create_dir=True): """ + Example: + work/global/module_defaults """ path = os.path.join(self.topdir(create_dir=create_dir), "module_defaults") if create_dir: diff --git a/pungi/phases/pkgset/sources/source_koji.py b/pungi/phases/pkgset/sources/source_koji.py index f5709c60..309ed654 100644 --- a/pungi/phases/pkgset/sources/source_koji.py +++ b/pungi/phases/pkgset/sources/source_koji.py @@ -26,9 +26,10 @@ from kobo.shortcuts import force_list import pungi.wrappers.kojiwrapper from pungi.wrappers.comps import CompsWrapper +from pungi.wrappers.mbs import MBSWrapper import pungi.phases.pkgset.pkgsets from pungi.arch import getBaseArch -from pungi.util import retry, get_arch_variant_data +from pungi.util import retry, get_arch_variant_data, get_variant_data from pungi.module_util import Modulemd from pungi.phases.pkgset.common import MaterializedPackageSet, get_all_arches @@ -272,6 +273,51 @@ def _add_module_to_variant( return nsvc +def _add_scratch_modules_to_variant( + compose, variant, scratch_modules, variant_tags, tag_to_mmd +): + if compose.compose_type != "test" and scratch_modules: + compose.log_warning("Only test composes could include scratch module builds") + return + + mbs = MBSWrapper(compose.conf["mbs_api_url"]) + for nsvc in scratch_modules: + module_build = mbs.get_module_build_by_nsvc(nsvc) + if not module_build: + continue + try: + final_modulemd = mbs.final_modulemd(module_build["id"]) + except Exception: + compose.log_error("Unable to get modulemd for build %s" % module_build) + raise + tag = module_build["koji_tag"] + variant_tags[variant].append(tag) + tag_to_mmd.setdefault(tag, {}) + for arch in variant.arches: + try: + mmd = Modulemd.ModuleStream.read_string( + final_modulemd[arch], strict=True + ) + variant.arch_mmds.setdefault(arch, {})[nsvc] = mmd + except KeyError: + continue + tag_to_mmd[tag].setdefault(arch, set()).add(mmd) + + if tag_to_mmd[tag]: + compose.log_info( + "Module '%s' in variant '%s' will use Koji tag '%s' " + "(as a result of querying module '%s')", + nsvc, + variant, + tag, + module_build["name"], + ) + + # Store mapping NSVC --> koji_tag into variant. This is needed + # in createrepo phase where metadata is exposed by productmd + variant.module_uid_to_koji_tag[nsvc] = tag + + def _is_filtered_out(compose, variant, arch, module_name, module_stream): """Check if module with given name and stream is filter out from this stream. """ @@ -618,6 +664,14 @@ def populate_global_pkgset(compose, koji_wrapper, path_prefix, event): compose, koji_wrapper, event, variant, variant_tags, tag_to_mmd ) + variant_scratch_modules = get_variant_data( + compose.conf, "pkgset_scratch_modules", variant + ) + if variant_scratch_modules: + _add_scratch_modules_to_variant( + compose, variant, variant_scratch_modules, variant_tags, tag_to_mmd + ) + # Ensure that every tag added to `variant_tags` is added also to # `compose_tags`. for variant_tag in variant_tags[variant]: diff --git a/pungi/wrappers/mbs.py b/pungi/wrappers/mbs.py new file mode 100644 index 00000000..730ea0e7 --- /dev/null +++ b/pungi/wrappers/mbs.py @@ -0,0 +1,43 @@ +import os +import requests + + +class MBSWrapper(object): + def __init__(self, api_url): + """ + :param string api_url: e.g. https://example.com/module-build-service/2 + """ + self.api_url = api_url + + def _get(self, resource, params=None): + """Get specified resource. + + :param string resource: e.g. module-builds, final-modulemd + :param dict data: + """ + url = os.path.join(self.api_url, resource) + try: + resp = requests.get(url, params=params) + except Exception as e: + raise Exception( + "Failed to query URL %s with params %s - %s" % (url, params, str(e)) + ) + resp.raise_for_status() + return resp + + def module_builds(self, filters=None): + return self._get("module-builds", filters).json() + + def get_module_build_by_nsvc(self, nsvc): + nsvc_list = nsvc.split(":") + if len(nsvc_list) != 4: + raise ValueError("Invalid N:S:V:C - %s" % nsvc) + filters = dict(zip(["name", "stream", "version", "context"], nsvc_list)) + resp = self.module_builds(filters) + if resp["items"]: + return resp["items"][0] + else: + return None + + def final_modulemd(self, module_build_id): + return self._get("final-modulemd/%s" % module_build_id).json() diff --git a/tests/test_pkgset_source_koji.py b/tests/test_pkgset_source_koji.py index 8e3ef136..26f9d914 100644 --- a/tests/test_pkgset_source_koji.py +++ b/tests/test_pkgset_source_koji.py @@ -881,3 +881,68 @@ class TestIsModuleFiltered(helpers.PungiTestCase): self.assertIsFiltered("foo", "master") self.assertIsNotFiltered("bar", "master") self.assertIsNotFiltered("foo", "stable") + + +class MockMBS(object): + def __init__(self, api_url): + self.api_url = api_url + + def get_module_build_by_nsvc(self, nsvc): + return {"id": 1, "koji_tag": "scratch-module-tag", "name": "scratch-module"} + + def final_modulemd(self, module_build_id): + return {"x86_64": ""} + + +class MockMmd(object): + def __init__(self, mmd, strict=True): + pass + + +@mock.patch("pungi.phases.pkgset.sources.source_koji.MBSWrapper", new=MockMBS) +@unittest.skipIf(Modulemd is None, "Skipping tests, no module support") +class TestAddScratchModuleToVariant(helpers.PungiTestCase): + def setUp(self): + super(TestAddScratchModuleToVariant, self).setUp() + self.compose = helpers.DummyCompose( + self.topdir, {"mbs_api_url": "http://mbs.local/module-build-service/2"} + ) + self.nsvc = "scratch-module:master:20200710:abcdef" + + @mock.patch( + "pungi.phases.pkgset.sources.source_koji.Modulemd.ModuleStream.read_string" + ) + def test_adding_scratch_module(self, mock_mmd): + variant = mock.Mock( + arches=["armhfp", "x86_64"], + arch_mmds={}, + modules=[], + module_uid_to_koji_tag={}, + ) + variant_tags = {variant: []} + tag_to_mmd = {} + scratch_modules = [self.nsvc] + + source_koji._add_scratch_modules_to_variant( + self.compose, variant, scratch_modules, variant_tags, tag_to_mmd + ) + self.assertEqual(variant_tags, {variant: ["scratch-module-tag"]}) + self.assertEqual( + variant.arch_mmds, {"x86_64": {self.nsvc: mock_mmd.return_value}} + ) + self.assertEqual( + tag_to_mmd, {"scratch-module-tag": {"x86_64": {mock_mmd.return_value}}} + ) + + self.assertEqual(variant.modules, []) + + def test_adding_scratch_module_nontest_compose(self): + self.compose.compose_type = "production" + scratch_modules = [self.nsvc] + + source_koji._add_scratch_modules_to_variant( + self.compose, mock.Mock(), scratch_modules, {}, {} + ) + self.compose.log_warning.assert_called_once_with( + "Only test composes could include scratch module builds" + )