From 9fcd71f8317292c67344ac8e126170001ea2eca0 Mon Sep 17 00:00:00 2001 From: Jan Kaluza Date: Tue, 28 Feb 2017 14:03:36 +0100 Subject: [PATCH] Add support for modular composes Signed-off-by: Jan Kaluza --- pungi/checks.py | 6 +- pungi/phases/createrepo.py | 18 ++ pungi/phases/gather/sources/source_module.py | 44 +++++ pungi/phases/pkgset/pkgsets.py | 9 +- pungi/phases/pkgset/sources/source_koji.py | 185 +++++++++++++++---- pungi/wrappers/variants.py | 37 +++- share/variants.dtd | 9 +- tests/helpers.py | 1 + tests/test_pkgset_source_koji.py | 54 +----- 9 files changed, 273 insertions(+), 90 deletions(-) create mode 100644 pungi/phases/gather/sources/source_module.py diff --git a/pungi/checks.py b/pungi/checks.py index 631c7499..60185d7d 100644 --- a/pungi/checks.py +++ b/pungi/checks.py @@ -493,7 +493,7 @@ def _make_schema(): }, "gather_source": { "type": "string", - "enum": ["json", "comps", "none"], + "enum": ["module", "json", "comps", "none"], }, "gather_fulltree": { "type": "boolean", @@ -614,6 +614,10 @@ def _make_schema(): "global_target": {"type": "string"}, "global_release": {"$ref": "#/definitions/optional_string"}, + "pdc_url": {"type": "string"}, + "pdc_develop": {"type": "boolean", "default": False}, + "pdc_insecure": {"type": "boolean", "default": False}, + "koji_profile": {"type": "string"}, "pkgset_koji_tag": {"type": "string"}, diff --git a/pungi/phases/createrepo.py b/pungi/phases/createrepo.py index 05ffe4da..9d580363 100644 --- a/pungi/phases/createrepo.py +++ b/pungi/phases/createrepo.py @@ -181,6 +181,24 @@ def create_variant_repo(compose, arch, variant, pkg_type): # this is a HACK to make CDN happy (dmach: at least I think, need to confirm with dgregor) shutil.copy2(product_id_path, os.path.join(repo_dir, "repodata", "productid")) + # call modifyrepo to inject modulemd if needed + if variant.mmds: + import yaml + modules = {"modules": []} + for mmd in variant.mmds: + modules["modules"].append(mmd.dumps()) + tmp_dir = compose.mkdtemp(prefix="pungi_") + modules_path = os.path.join(tmp_dir, "modules.yaml") + with open(modules_path, "w") as outfile: + outfile.write(yaml.safe_dump(modules)) + cmd = repo.get_modifyrepo_cmd(os.path.join(repo_dir, "repodata"), + modules_path, mdtype="modules", + compress_type="gz") + log_file = compose.paths.log.log_file( + arch, "modifyrepo-modules-%s" % variant) + run(cmd, logfile=log_file, show_cmd=True) + shutil.rmtree(tmp_dir) + compose.log_info("[DONE ] %s" % msg) diff --git a/pungi/phases/gather/sources/source_module.py b/pungi/phases/gather/sources/source_module.py new file mode 100644 index 00000000..dd7eaffa --- /dev/null +++ b/pungi/phases/gather/sources/source_module.py @@ -0,0 +1,44 @@ +# -*- coding: utf-8 -*- + + +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; version 2 of the License. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Library General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, see . + + +""" +Get a package list based on modulemd metadata loaded in pkgset phase. +""" + + +from pungi.wrappers.comps import CompsWrapper +import pungi.phases.gather.source +import kobo.rpmlib + + +class GatherSourceModule(pungi.phases.gather.source.GatherSourceBase): + enabled = True + + def __call__(self, arch, variant): + groups = set() + packages = set() + + if variant is not None and variant.modules: + rpms = variant.pkgset.rpms_by_arch[arch] + \ + variant.pkgset.rpms_by_arch["noarch"] + for rpm_obj in rpms: + for mmd in variant.mmds: + srpm = kobo.rpmlib.parse_nvr(rpm_obj.sourcerpm)["name"] + if (srpm in mmd.components.rpms.keys() and + rpm_obj.name not in mmd.filter.rpms): + packages.add((rpm_obj.name, None)) + + return packages, groups diff --git a/pungi/phases/pkgset/pkgsets.py b/pungi/phases/pkgset/pkgsets.py index 496d99f2..03c6e458 100644 --- a/pungi/phases/pkgset/pkgsets.py +++ b/pungi/phases/pkgset/pkgsets.py @@ -148,15 +148,18 @@ class PackageSetBase(kobo.log.LoggingBase): seen_sourcerpms = set() # {Exclude,Exclusive}Arch must match *tree* arch + compatible native # arches (excluding multilib arches) - exclusivearch_list = get_valid_arches(primary_arch, multilib=False, - add_noarch=False, add_src=False) + if primary_arch: + exclusivearch_list = get_valid_arches( + primary_arch, multilib=False, add_noarch=False, add_src=False) + else: + exclusivearch_list = None for arch in arch_list: self.rpms_by_arch.setdefault(arch, []) for i in other.rpms_by_arch.get(arch, []): if i.file_path in self.file_cache: # TODO: test if it really works continue - if arch == "noarch": + if exclusivearch_list and arch == "noarch": if i.excludearch and set(i.excludearch) & set(exclusivearch_list): self.log_debug("Excluding (EXCLUDEARCH: %s): %s" % (sorted(set(i.excludearch)), i.file_name)) diff --git a/pungi/phases/pkgset/sources/source_koji.py b/pungi/phases/pkgset/sources/source_koji.py index 50cd6b95..15d7d88d 100644 --- a/pungi/phases/pkgset/sources/source_koji.py +++ b/pungi/phases/pkgset/sources/source_koji.py @@ -28,6 +28,84 @@ from pungi.phases.pkgset.common import create_arch_repos, create_global_repo, po import pungi.phases.pkgset.source +try: + from pdc_client import PDCClient + import modulemd + WITH_MODULES = True +except: + WITH_MODULES = False + + +def get_pdc_client_session(compose): + if not WITH_MODULES: + compose.log_warning("pdc_client module is not installed, " + "support for modules is disabled") + return None + try: + return PDCClient( + server=compose.conf['pdc_url'], + develop=compose.conf['pdc_develop'], + ssl_verify=not compose.conf['pdc_insecure'], + ) + except KeyError: + return None + + +def variant_dict_from_str(module_str): + module_info = {} + + release_start = module_str.rfind('-') + module_info['variant_version'] = module_str[release_start+1:] + module_info['variant_id'] = module_str[:release_start] + module_info['variant_type'] = 'module' + + return module_info + + +def get_module(session, module_info, strict=False): + """ + :param session : PDCClient instance + :param module_info: pdc variant_dict, str, mmd or module dict + :param strict: Normally this function returns None if no module can be + found. If strict=True, then a ValueError is raised. + + :return final list of module_info which pass repoclosure + """ + + module_info = variant_dict_from_str(module_info) + + query = dict( + variant_id=module_info['variant_id'], + variant_version=module_info['variant_version'], + ) + if module_info.get('variant_release'): + query['variant_release'] = module_info['variant_release'] + + retval = session['unreleasedvariants'](page_size=-1, **query) + + # Error handling + if not retval: + if strict: + raise ValueError("Failed to find module in PDC %r" % query) + else: + return None + + module = None + # If we specify 'variant_release', we expect only single module to be + # returned, but otherwise we have to pick the one with the highest + # release ourselves. + if 'variant_release' in query: + assert len(retval) <= 1, compose.log_error( + "More than one module returned from PDC: %s" % str(retval)) + module = retval[0] + else: + module = retval[0] + for m in retval: + if int(m['variant_release']) > int(module['variant_release']): + module = m + + return module + class PkgsetSourceKoji(pungi.phases.pkgset.source.PkgsetSourceBase): enabled = True @@ -44,9 +122,7 @@ class PkgsetSourceKoji(pungi.phases.pkgset.source.PkgsetSourceBase): def get_pkgset_from_koji(compose, koji_wrapper, path_prefix): event_info = get_koji_event_info(compose, koji_wrapper) - tag_info = get_koji_tag_info(compose, koji_wrapper) - - pkgset_global = populate_global_pkgset(compose, koji_wrapper, path_prefix, tag_info, event_info) + pkgset_global = populate_global_pkgset(compose, koji_wrapper, path_prefix, event_info) package_sets = populate_arch_pkgsets(compose, path_prefix, pkgset_global) package_sets["global"] = pkgset_global @@ -58,32 +134,90 @@ def get_pkgset_from_koji(compose, koji_wrapper, path_prefix): return package_sets -def populate_global_pkgset(compose, koji_wrapper, path_prefix, compose_tag, event_id): +def populate_global_pkgset(compose, koji_wrapper, path_prefix, event_id): all_arches = set(["src"]) for arch in compose.get_arches(): is_multilib = is_arch_multilib(compose.conf, arch) arches = get_valid_arches(arch, is_multilib) all_arches.update(arches) - compose_tag = compose.conf["pkgset_koji_tag"] + # List of compose tags from which we create this compose + compose_tags = [] + + # List of compose_tags per variant + variant_tags = {} + + session = get_pdc_client_session(compose) + for variant in compose.all_variants.values(): + variant.pkgset = pungi.phases.pkgset.pkgsets.KojiPackageSet( + koji_wrapper, compose.conf["sigkeys"], logger=compose._logger, + arches=all_arches) + variant_tags[variant] = [] + + # Find out all modules in every variant and add their compose tags + # to compose_tags list. + if session: + for module in variant.get_modules(): + pdc_module = get_module(session, module["name"]) + mmd = modulemd.ModuleMetadata() + mmd.loads(pdc_module["modulemd"]) + tag = pdc_module["koji_tag"] + variant.mmds.append(mmd) + variant_tags[variant].append(tag) + if tag not in compose_tags: + compose_tags.append(tag) + + if not variant_tags[variant]: + variant_tags[variant].append(compose.conf["pkgset_koji_tag"]) + + # In case we have no compose tag from module, use the default + # one from config. + if not compose_tags: + compose_tags.append(compose.conf["pkgset_koji_tag"]) + inherit = compose.conf["pkgset_koji_inherit"] - msg = "Populating the global package set from tag '%s'" % compose_tag - global_pkgset_path = os.path.join(compose.paths.work.topdir(arch="global"), "pkgset_global.pickle") + global_pkgset_path = os.path.join( + compose.paths.work.topdir(arch="global"), "pkgset_global.pickle") if compose.DEBUG and os.path.isfile(global_pkgset_path): + msg = "Populating the global package set from tag '%s'" % compose_tags compose.log_warning("[SKIP ] %s" % msg) - pkgset = pickle.load(open(global_pkgset_path, "r")) + global_pkgset = pickle.load(open(global_pkgset_path, "r")) else: - compose.log_info(msg) - pkgset = pungi.phases.pkgset.pkgsets.KojiPackageSet(koji_wrapper, compose.conf["sigkeys"], - logger=compose._logger, arches=all_arches) - pkgset.populate(compose_tag, event_id, inherit=inherit) + global_pkgset = pungi.phases.pkgset.pkgsets.KojiPackageSet( + koji_wrapper, compose.conf["sigkeys"], logger=compose._logger, + arches=all_arches) + # Get package set for each compose tag and merge it to global package + # list. Also prepare per-variant pkgset, because we do not have list + # of binary RPMs in module definition - there is just list of SRPMs. + for compose_tag in compose_tags: + compose.log_info("Populating the global package set from tag " + "'%s'" % compose_tag) + pkgset = pungi.phases.pkgset.pkgsets.KojiPackageSet( + koji_wrapper, compose.conf["sigkeys"], logger=compose._logger, + arches=all_arches) + pkgset.populate(compose_tag, event_id, inherit=inherit) + for variant in compose.all_variants.values(): + if compose_tag in variant_tags[variant]: + # Optimization for case where we have just single compose + # tag - we do not have to merge in this case... + if len(compose_tags) == 1: + variant.pkgset = pkgset + else: + variant.pkgset.merge(pkgset, None, list(all_arches)) + # Optimization for case where we have just single compose + # tag - we do not have to merge in this case... + if len(compose_tags) == 1: + global_pkgset = pkgset + else: + global_pkgset.merge(pkgset, None, list(all_arches)) with open(global_pkgset_path, 'w') as f: - f.write(pickle.dumps(pkgset)) + f.write(pickle.dumps(global_pkgset)) # write global package list - pkgset.save_file_list(compose.paths.work.package_list(arch="global"), - remove_path_prefix=path_prefix) - return pkgset + global_pkgset.save_file_list( + compose.paths.work.package_list(arch="global"), + remove_path_prefix=path_prefix) + return global_pkgset def get_koji_event_info(compose, koji_wrapper): @@ -106,22 +240,3 @@ def get_koji_event_info(compose, koji_wrapper): json.dump(result, open(event_file, "w")) compose.log_info("Koji event: %s" % result["id"]) return result - - -def get_koji_tag_info(compose, koji_wrapper): - koji_proxy = koji_wrapper.koji_proxy - tag_file = os.path.join(compose.paths.work.topdir(arch="global"), "koji-tag") - msg = "Getting a koji tag info" - if compose.DEBUG and os.path.exists(tag_file): - compose.log_warning("[SKIP ] %s" % msg) - result = json.load(open(tag_file, "r")) - else: - compose.log_info(msg) - tag_name = compose.conf["pkgset_koji_tag"] - result = koji_proxy.getTag(tag_name) - if result is None: - raise ValueError("Unknown koji tag: %s" % tag_name) - result["name"] = tag_name - json.dump(result, open(tag_file, "w")) - compose.log_info("Koji compose tag: %(name)s (ID: %(id)s)" % result) - return result diff --git a/pungi/wrappers/variants.py b/pungi/wrappers/variants.py index fb2cdb74..92e53b61 100755 --- a/pungi/wrappers/variants.py +++ b/pungi/wrappers/variants.py @@ -77,6 +77,7 @@ class VariantsXmlParser(object): "type": str(variant_node.attrib["type"]), "arches": [str(i) for i in variant_node.xpath("arches/arch/text()")], "groups": [], + "modules": [], "environments": [], "buildinstallpackages": [], "is_empty": bool(variant_node.attrib.get("is_empty", False)), @@ -108,6 +109,15 @@ class VariantsXmlParser(object): variant_dict["groups"].append(group) + for modulelist_node in variant_node.xpath("modules"): + for module_node in modulelist_node.xpath("module"): + module = { + "name": str(module_node.text), + "glob": self._is_true(module_node.attrib.get("glob", "false")) + } + + variant_dict["modules"].append(module) + for environments_node in variant_node.xpath("environments"): for environment_node in environments_node.xpath("environment"): environment = { @@ -198,9 +208,11 @@ class VariantsXmlParser(object): class Variant(object): def __init__(self, id, name, type, arches, groups, environments=None, - buildinstallpackages=None, is_empty=False, parent=None): + buildinstallpackages=None, is_empty=False, parent=None, + modules=None): environments = environments or [] + modules = modules or [] buildinstallpackages = buildinstallpackages or [] self.id = id @@ -209,11 +221,16 @@ class Variant(object): self.arches = sorted(copy.deepcopy(arches)) self.groups = sorted(copy.deepcopy(groups), lambda x, y: cmp(x["name"], y["name"])) self.environments = sorted(copy.deepcopy(environments), lambda x, y: cmp(x["name"], y["name"])) + self.modules = sorted(copy.deepcopy(modules), + lambda x, y: cmp(x["name"], y["name"])) self.buildinstallpackages = sorted(buildinstallpackages) self.variants = {} self.parent = parent self.is_empty = is_empty + self.pkgset = None + self.mmds = [] + def __getitem__(self, name): return self.variants[name] @@ -274,6 +291,22 @@ class Variant(object): result.append(group) return result + def get_modules(self, arch=None, types=None, recursive=False): + """Return list of groups, default types is ["self"]""" + + types = types or ["self"] + result = copy.deepcopy(self.modules) + for variant in self.get_variants(arch=arch, types=types, + recursive=recursive): + if variant == self: + # XXX + continue + for module in variant.get_modules(arch=arch, types=types, + recursive=recursive): + if module not in result: + result.append(module) + return result + def get_variants(self, arch=None, types=None, recursive=False): """ Return all variants of given arch and types. @@ -339,6 +372,8 @@ def main(argv): print(" ARCHES: %s" % ", ".join(sorted(i.arches))) for group in i.groups: print(" GROUP: %(name)-40s GLOB: %(glob)-5s DEFAULT: %(default)-5s USERVISIBLE: %(uservisible)-5s" % group) + for module in i.modules: + print(" MODULE: %(name)-40s GLOB: %(glob)-5s DEFAULT: %(default)-5s USERVISIBLE: %(uservisible)-5s" % module) for env in i.environments: print(" ENV: %(name)-40s DISPLAY_ORDER: %(display_order)s" % env) print() diff --git a/share/variants.dtd b/share/variants.dtd index 19ef3430..197e4c4b 100644 --- a/share/variants.dtd +++ b/share/variants.dtd @@ -1,6 +1,6 @@ - + + + + + + diff --git a/tests/helpers.py b/tests/helpers.py index b459ac58..b66d8d7e 100644 --- a/tests/helpers.py +++ b/tests/helpers.py @@ -33,6 +33,7 @@ class MockVariant(mock.Mock): def __init__(self, *args, **kwargs): super(MockVariant, self).__init__(*args, **kwargs) self.parent = kwargs.get('parent', None) + self.mmds = [] def __str__(self): return self.uid diff --git a/tests/test_pkgset_source_koji.py b/tests/test_pkgset_source_koji.py index 4696787d..ce97600d 100644 --- a/tests/test_pkgset_source_koji.py +++ b/tests/test_pkgset_source_koji.py @@ -81,50 +81,6 @@ class TestGetKojiEvent(helpers.PungiTestCase): with open(self.event_file) as f: self.assertEqual(json.load(f), EVENT_INFO) - -class TestGetKojiTagInfo(helpers.PungiTestCase): - def setUp(self): - super(TestGetKojiTagInfo, self).setUp() - self.compose = helpers.DummyCompose(self.topdir, { - 'pkgset_koji_tag': 'f25' - }) - self.compose.DEBUG = False - self.tag_file = os.path.join(self.topdir, 'work', 'global', 'koji-tag') - self.koji_wrapper = mock.Mock() - - def test_get_tag_info(self): - self.koji_wrapper.koji_proxy.getTag.return_value = TAG_INFO - - tag_info = source_koji.get_koji_tag_info(self.compose, self.koji_wrapper) - - self.assertEqual(tag_info, TAG_INFO) - self.assertItemsEqual( - self.koji_wrapper.mock_calls, - [mock.call.koji_proxy.getTag('f25')] - ) - with open(self.tag_file) as f: - self.assertEqual(json.load(f), TAG_INFO) - - def test_bad_tag_name(self): - self.koji_wrapper.koji_proxy.getTag.return_value = None - - with self.assertRaises(ValueError) as ctx: - source_koji.get_koji_tag_info(self.compose, self.koji_wrapper) - - self.assertIn('Unknown', str(ctx.exception)) - - def test_get_tag_info_in_debug_mode(self): - self.compose.DEBUG = True - helpers.touch(self.tag_file, json.dumps(TAG_INFO)) - - tag_info = source_koji.get_koji_tag_info(self.compose, self.koji_wrapper) - - self.assertEqual(tag_info, TAG_INFO) - self.assertItemsEqual(self.koji_wrapper.mock_calls, []) - with open(self.tag_file) as f: - self.assertEqual(json.load(f), TAG_INFO) - - class TestPopulateGlobalPkgset(helpers.PungiTestCase): def setUp(self): super(TestPopulateGlobalPkgset, self).setUp() @@ -145,7 +101,7 @@ class TestPopulateGlobalPkgset(helpers.PungiTestCase): orig_pkgset = KojiPackageSet.return_value pkgset = source_koji.populate_global_pkgset( - self.compose, self.koji_wrapper, '/prefix', 'f25', 123456) + self.compose, self.koji_wrapper, '/prefix', 123456) self.assertIs(pkgset, orig_pkgset) self.assertEqual( @@ -168,7 +124,7 @@ class TestPopulateGlobalPkgset(helpers.PungiTestCase): with mock.patch('pungi.phases.pkgset.sources.source_koji.open', mock.mock_open(), create=True) as m: pkgset = source_koji.populate_global_pkgset( - self.compose, self.koji_wrapper, '/prefix', 'f25', 123456) + self.compose, self.koji_wrapper, '/prefix', 123456) self.assertEqual(pickle_load.call_args_list, [mock.call(m.return_value)]) @@ -204,12 +160,12 @@ class TestGetPackageSetFromKoji(helpers.PungiTestCase): self.assertItemsEqual( self.koji_wrapper.koji_proxy.mock_calls, - [mock.call.getLastEvent(), - mock.call.getTag('f25')] + [mock.call.getLastEvent()] ) + self.assertEqual(pgp.call_args_list, [mock.call(self.compose, self.koji_wrapper, '/prefix', - TAG_INFO, EVENT_INFO)]) + EVENT_INFO)]) self.assertEqual(pap.call_args_list, [mock.call(self.compose, '/prefix', pgp.return_value)]) self.assertEqual(cgr.call_args_list,