From f0bd1af9999c30aa6aa0784a6698533cbf69cb41 Mon Sep 17 00:00:00 2001 From: soksanichenko Date: Sun, 19 Mar 2023 18:21:33 +0200 Subject: [PATCH 1/5] ALBS-1030: Generate Devel section in packages.json - Also the tool can combine (remove and add) packages in a variant from different sources according to an url's type of source --- pungi/scripts/create_packages_json.py | 26 +++++++++++++++++++++----- 1 file changed, 21 insertions(+), 5 deletions(-) diff --git a/pungi/scripts/create_packages_json.py b/pungi/scripts/create_packages_json.py index f54b7a02..1274945f 100644 --- a/pungi/scripts/create_packages_json.py +++ b/pungi/scripts/create_packages_json.py @@ -14,7 +14,7 @@ import os import re import tempfile from collections import defaultdict -from typing import AnyStr, Dict, List, Optional, Any, Iterator +from typing import AnyStr, Dict, List, Any, Iterator import binascii import createrepo_c as cr @@ -65,7 +65,7 @@ class RepoInfo: # Only layout of specific package (which don't exist # in a reference repository) will be taken as example is_reference: bool = False - strict_arch: bool = False + repo_type: str = 'present' class PackagesGenerator: @@ -262,7 +262,11 @@ class PackagesGenerator: ) ) all_packages = defaultdict(lambda: {'variants': list()}) - for repo_info in self.repos: + for repo_info in sorted( + self.repos, + key=lambda i: i.repo_type, + reverse=True, + ): repomd_records = self._get_repomd_records( repo_info=repo_info, ) @@ -298,6 +302,8 @@ class PackagesGenerator: all_packages[package_key]['arch'] = package_arch all_packages[package_key]['package'] = package all_packages[package_key]['type'] = repo_info.is_reference + elif repo_info.repo_type == 'absent' and (repo_info.name, repo_info.arch) in all_packages[package_key]['variants']: + all_packages[package_key]['variants'].remove((repo_info.name, repo_info.arch)) # replace an older package if it's not reference or # a newer package is from reference repo elif (not all_packages[package_key]['type'] or @@ -404,6 +410,14 @@ def create_parser(): choices=['yes', 'no'], required=True, ) + parser.add_argument( + '--repo-type', + action='append', + type=str, + help='Packages from repository will be removed or added to variant', + choices=['present', 'absent'], + required=True, + ) parser.add_argument( '--excluded-packages', nargs='+', @@ -436,13 +450,14 @@ def cli_main(): args = create_parser().parse_args() repos = [] for repo_path, repo_folder, repo_name, \ - repo_arch, is_remote, is_reference in zip( + repo_arch, is_remote, is_reference, repo_type in zip( args.repo_path, args.repo_folder, args.repo_name, args.repo_arch, args.is_remote, args.is_reference, + args.repo_type, ): repos.append(RepoInfo( path=repo_path, @@ -450,7 +465,8 @@ def cli_main(): name=repo_name, arch=repo_arch, is_remote=True if is_remote == 'yes' else False, - is_reference=True if is_reference == 'yes' else False + is_reference=True if is_reference == 'yes' else False, + repo_type=repo_type, )) pg = PackagesGenerator( repos=repos, From d32c293bcaa384f948a8cb42793a1f4307dd136e Mon Sep 17 00:00:00 2001 From: soksanichenko Date: Sun, 19 Mar 2023 21:11:12 +0200 Subject: [PATCH 2/5] ALBS-1030: Generate Devel section in packages.json - Some upstream changes to KojiMock parts --- .../phases/pkgset/sources/source_kojimock.py | 191 ++++++++++++------ 1 file changed, 132 insertions(+), 59 deletions(-) diff --git a/pungi/phases/pkgset/sources/source_kojimock.py b/pungi/phases/pkgset/sources/source_kojimock.py index aa063a18..b8b749d7 100644 --- a/pungi/phases/pkgset/sources/source_kojimock.py +++ b/pungi/phases/pkgset/sources/source_kojimock.py @@ -35,10 +35,12 @@ import pungi.wrappers.kojiwrapper from pungi.wrappers.comps import CompsWrapper from pungi.wrappers.mbs import MBSWrapper import pungi.phases.pkgset.pkgsets + from pungi.util import ( retry, get_arch_variant_data, get_variant_data, + read_single_module_stream_from_string, read_single_module_stream_from_file, ) @@ -101,12 +103,12 @@ def variant_dict_from_str(compose, module_str): release_regex = re.compile(r"^(\d){14}$") section_start = module_str.rfind("-") - module_str_first_part = module_str[section_start + 1 :] + module_str_first_part = module_str[section_start + 1:] if release_regex.match(module_str_first_part): module_info["version"] = module_str_first_part module_str = module_str[:section_start] section_start = module_str.rfind("-") - module_info["stream"] = module_str[section_start + 1 :] + module_info["stream"] = module_str[section_start + 1:] else: module_info["stream"] = module_str_first_part module_info["name"] = module_str[:section_start] @@ -160,14 +162,16 @@ def get_koji_modules(compose, koji_wrapper, event, module_info_str): # Store module versioning information into the dict, but make sure # not to overwrite any existing keys. md["module_stream"] = md["extra"]["typeinfo"]["module"]["stream"] - md["module_version"] = int(md["extra"]["typeinfo"]["module"]["version"]) + md["module_version"] = int( + md["extra"]["typeinfo"]["module"]["version"]) md["module_context"] = md["extra"]["typeinfo"]["module"]["context"] except KeyError: continue if md["state"] == pungi.wrappers.kojiwrapper.KOJI_BUILD_DELETED: compose.log_debug( - "Module build %s has been deleted, ignoring it." % build["name"] + "Module build %s has been deleted, ignoring it." % build[ + "name"] ) continue @@ -189,7 +193,8 @@ def get_koji_modules(compose, koji_wrapper, event, module_info_str): ) latest_version = sorted_modules[0]["module_version"] modules = [ - module for module in modules if latest_version == module["module_version"] + module for module in modules if + latest_version == module["module_version"] ] return modules @@ -205,7 +210,8 @@ class PkgsetSourceKojiMock(pungi.phases.pkgset.source.PkgsetSourceBase): get_all_arches(compose), ) # path prefix must contain trailing '/' - path_prefix = self.koji_wrapper.koji_module.config.topdir.rstrip("/") + "/" + path_prefix = self.koji_wrapper.koji_module.config.topdir.rstrip( + "/") + "/" package_sets = get_pkgset_from_koji( self.compose, self.koji_wrapper, path_prefix ) @@ -214,16 +220,17 @@ class PkgsetSourceKojiMock(pungi.phases.pkgset.source.PkgsetSourceBase): def get_pkgset_from_koji(compose, koji_wrapper, path_prefix): event_info = get_koji_event_info(compose, koji_wrapper) - return populate_global_pkgset(compose, koji_wrapper, path_prefix, event_info) + return populate_global_pkgset(compose, koji_wrapper, path_prefix, + event_info) def _add_module_to_variant( - koji_wrapper, - variant, - build, - add_to_variant_modules=False, - compose=None, - exclude_module_ns=None, + koji_wrapper, + variant, + build, + add_to_variant_modules=False, + compose=None, + exclude_module_ns=None, ): """ Adds module defined by Koji build info to variant. @@ -241,13 +248,16 @@ def _add_module_to_variant( if archive["btype"] != "module": # Skip non module archives continue + filename = archive["filename"] file_path = os.path.join( koji_wrapper.koji_module.pathinfo.topdir, 'modules', build['arch'], build['extra']['typeinfo']['module']['content_koji_tag'] + ) + mmds[filename] = file_path if len(mmds) <= 1: @@ -266,17 +276,22 @@ def _add_module_to_variant( added = False for arch in variant.arches: - if _is_filtered_out(compose, variant, arch, info["name"], info["stream"]): - compose.log_debug("Module %s is filtered from %s.%s", nsvc, variant, arch) + if _is_filtered_out(compose, variant, arch, info["name"], + info["stream"]): + compose.log_debug("Module %s is filtered from %s.%s", nsvc, + variant, arch) continue + filename = "modulemd.%s.txt" % arch try: mod_stream = read_single_module_stream_from_file( mmds[filename], compose, arch, build + ) if mod_stream: added = True variant.arch_mmds.setdefault(arch, {})[nsvc] = mod_stream + added = True except KeyError: # There is no modulemd for this arch. This could mean an arch was @@ -295,10 +310,11 @@ def _add_module_to_variant( def _add_extra_modules_to_variant( - compose, koji_wrapper, variant, extra_modules, variant_tags, tag_to_mmd + compose, koji_wrapper, variant, extra_modules, variant_tags, tag_to_mmd ): for nsvc in extra_modules: - msg = "Adding extra module build '%s' to variant '%s'" % (nsvc, variant) + msg = "Adding extra module build '%s' to variant '%s'" % ( + nsvc, variant) compose.log_info(msg) nsvc_info = nsvc.split(":") @@ -341,10 +357,11 @@ def _add_extra_modules_to_variant( def _add_scratch_modules_to_variant( - compose, variant, scratch_modules, variant_tags, tag_to_mmd + compose, variant, scratch_modules, variant_tags, tag_to_mmd ): if compose.compose_type != "test" and scratch_modules: - compose.log_warning("Only test composes could include scratch module builds") + compose.log_warning( + "Only test composes could include scratch module builds") return mbs = MBSWrapper(compose.conf["mbs_api_url"]) @@ -355,7 +372,8 @@ def _add_scratch_modules_to_variant( try: final_modulemd = mbs.final_modulemd(module_build["id"]) except Exception: - compose.log_error("Unable to get modulemd for build %s" % module_build) + compose.log_error( + "Unable to get modulemd for build %s" % module_build) raise tag = module_build["koji_tag"] variant_tags[variant].append(tag) @@ -363,8 +381,7 @@ def _add_scratch_modules_to_variant( for arch in variant.arches: try: mmd = read_single_module_stream_from_string( - final_modulemd[arch] - ) + final_modulemd[arch]) variant.arch_mmds.setdefault(arch, {})[nsvc] = mmd except KeyError: continue @@ -390,21 +407,24 @@ def _is_filtered_out(compose, variant, arch, module_name, module_stream): if not compose: return False - for filter in get_arch_variant_data(compose.conf, "filter_modules", arch, variant): + for filter in get_arch_variant_data(compose.conf, "filter_modules", arch, + variant): if ":" not in filter: name_filter = filter stream_filter = "*" else: name_filter, stream_filter = filter.split(":", 1) - if fnmatch(module_name, name_filter) and fnmatch(module_stream, stream_filter): + if fnmatch(module_name, name_filter) and fnmatch(module_stream, + stream_filter): return True return False def _get_modules_from_koji( - compose, koji_wrapper, event, variant, variant_tags, tag_to_mmd + compose, koji_wrapper, event, variant, variant_tags, tag_to_mmd, + exclude_module_ns ): """ Loads modules for given `variant` from koji `session`, adds them to @@ -415,15 +435,21 @@ def _get_modules_from_koji( :param Variant variant: Variant with modules to find. :param dict variant_tags: Dict populated by this method. Key is `variant` and value is list of Koji tags to get the RPMs from. + :param list exclude_module_ns: Module name:stream which will be excluded. """ # Find out all modules in every variant and add their Koji tags # to variant and variant_tags list. for module in variant.get_modules(): - koji_modules = get_koji_modules(compose, koji_wrapper, event, module["name"]) + koji_modules = get_koji_modules(compose, koji_wrapper, event, + module["name"]) for koji_module in koji_modules: nsvc = _add_module_to_variant( - koji_wrapper, variant, koji_module, compose=compose + koji_wrapper, + variant, + koji_module, + compose=compose, + exclude_module_ns=exclude_module_ns, ) if not nsvc: continue @@ -462,7 +488,8 @@ def filter_inherited(koji_proxy, event, module_builds, top_tag): does not understand streams, so we have to reimplement it here. """ inheritance = [ - tag["name"] for tag in koji_proxy.getFullInheritance(top_tag, event=event["id"]) + tag["name"] for tag in + koji_proxy.getFullInheritance(top_tag, event=event["id"]) ] def keyfunc(mb): @@ -487,7 +514,8 @@ def filter_inherited(koji_proxy, event, module_builds, top_tag): return result -def filter_by_whitelist(compose, module_builds, input_modules, expected_modules): +def filter_by_whitelist(compose, module_builds, input_modules, + expected_modules): """ Exclude modules from the list that do not match any pattern specified in input_modules. Order may not be preserved. The last argument is a set of @@ -511,6 +539,7 @@ def filter_by_whitelist(compose, module_builds, input_modules, expected_modules) info.get("context"), ) nvr_patterns.add((pattern, spec["name"])) + modules_to_keep = [] for mb in sorted(module_builds, key=lambda i: i['name']): @@ -521,13 +550,13 @@ def filter_by_whitelist(compose, module_builds, input_modules, expected_modules) # and context. for (n, s, v, c), spec in sorted(nvr_patterns): if ( - # We always have a name and stream... - mb["name"] == n - and mb["version"] == s - # ...but version and context can be missing, in which case we - # don't want to check them. - and (not v or ver == v) - and (not c or ctx == c) + # We always have a name and stream... + mb["name"] == n + and mb["version"] == s + # ...but version and context can be missing, in which case we + # don't want to check them. + and (not v or ver == v) + and (not c or ctx == c) ): modules_to_keep.append(mb) expected_modules.discard(spec) @@ -575,7 +604,13 @@ def _filter_expected_modules( def _get_modules_from_koji_tags( - compose, koji_wrapper, event_id, variant, variant_tags, tag_to_mmd + compose, + koji_wrapper, + event_id, + variant, + variant_tags, + tag_to_mmd, + exclude_module_ns, ): """ Loads modules for given `variant` from Koji, adds them to @@ -587,10 +622,12 @@ def _get_modules_from_koji_tags( :param Variant variant: Variant with modules to find. :param dict variant_tags: Dict populated by this method. Key is `variant` and value is list of Koji tags to get the RPMs from. + :param list exclude_module_ns: Module name:stream which will be excluded. """ # Compose tags from configuration compose_tags = [ - {"name": tag} for tag in force_list(compose.conf["pkgset_koji_module_tag"]) + {"name": tag} for tag in + force_list(compose.conf["pkgset_koji_module_tag"]) ] # Get set of configured module names for this variant. If nothing is # configured, the set is empty. @@ -617,7 +654,8 @@ def _get_modules_from_koji_tags( ) # Filter out builds inherited from non-top tag - module_builds = filter_inherited(koji_proxy, event_id, module_builds, tag) + module_builds = filter_inherited(koji_proxy, event_id, module_builds, + tag) # Apply whitelist of modules if specified. variant_modules = variant.get_modules() @@ -625,6 +663,7 @@ def _get_modules_from_koji_tags( module_builds = filter_by_whitelist( compose, module_builds, variant_modules, expected_modules ) + # Find the latest builds of all modules. This does following: # - Sorts the module_builds descending by Koji NVR (which maps to NSV # for modules). Split release into modular version and context, and @@ -645,14 +684,14 @@ def _get_modules_from_koji_tags( latest_builds = [] module_builds = sorted(module_builds, key=_key, reverse=True) for ns, ns_builds in groupby( - module_builds, key=lambda x: ":".join([ + module_builds, key=lambda x: ":".join([ x["name"], x["version"], x['arch'], ]) ): for nsv, nsv_builds in groupby( - ns_builds, key=lambda x: x["release"].split(".")[0] + ns_builds, key=lambda x: x["release"].split(".")[0] ): latest_builds += list(nsv_builds) break @@ -662,6 +701,18 @@ def _get_modules_from_koji_tags( for build in latest_builds: # Get the Build from Koji to get modulemd and module_tag. build = koji_proxy.getBuild(build["build_id"]) + + nsvc = _add_module_to_variant( + koji_wrapper, + variant, + build, + True, + compose=compose, + exclude_module_ns=exclude_module_ns, + ) + if not nsvc: + continue + module_tag = ( build.get("extra", {}) .get("typeinfo", {}) @@ -671,12 +722,6 @@ def _get_modules_from_koji_tags( variant_tags[variant].append(module_tag) - nsvc = _add_module_to_variant( - koji_wrapper, variant, build, True, compose=compose - ) - if not nsvc: - continue - tag_to_mmd.setdefault(module_tag, {}) for arch in variant.arch_mmds: try: @@ -736,7 +781,7 @@ def populate_global_pkgset(compose, koji_wrapper, path_prefix, event): for group in groups: packages_to_gather += comps.get_packages(group) if compose.conf["gather_method"] == "nodeps" and not compose.conf.get( - "buildinstall_method" + "buildinstall_method" ): populate_only_packages_to_gather = True else: @@ -767,26 +812,48 @@ def populate_global_pkgset(compose, koji_wrapper, path_prefix, event): "modules." ) + extra_modules = get_variant_data( + compose.conf, "pkgset_koji_module_builds", variant + ) + + # When adding extra modules, other modules of the same name:stream available + # in brew tag should be excluded. + exclude_module_ns = [] + if extra_modules: + exclude_module_ns = [ + ":".join(nsvc.split(":")[:2]) for nsvc in extra_modules + ] + if modular_koji_tags or ( - compose.conf["pkgset_koji_module_tag"] and variant.modules + compose.conf["pkgset_koji_module_tag"] and variant.modules ): # List modules tagged in particular tags. _get_modules_from_koji_tags( - compose, koji_wrapper, event, variant, variant_tags, tag_to_mmd + compose, + koji_wrapper, + event, + variant, + variant_tags, + tag_to_mmd, + exclude_module_ns, ) elif variant.modules: # Search each module in Koji separately. Tagging does not come into # play here. _get_modules_from_koji( - compose, koji_wrapper, event, variant, variant_tags, tag_to_mmd + compose, + koji_wrapper, + event, + variant, + variant_tags, + tag_to_mmd, + exclude_module_ns, ) - extra_modules = get_variant_data( - compose.conf, "pkgset_koji_module_builds", variant - ) if extra_modules: _add_extra_modules_to_variant( - compose, koji_wrapper, variant, extra_modules, variant_tags, tag_to_mmd + compose, koji_wrapper, variant, extra_modules, variant_tags, + tag_to_mmd ) variant_scratch_modules = get_variant_data( @@ -794,7 +861,8 @@ def populate_global_pkgset(compose, koji_wrapper, path_prefix, event): ) if variant_scratch_modules: _add_scratch_modules_to_variant( - compose, variant, variant_scratch_modules, variant_tags, tag_to_mmd + compose, variant, variant_scratch_modules, variant_tags, + tag_to_mmd ) # Ensure that every tag added to `variant_tags` is added also to @@ -819,8 +887,10 @@ def populate_global_pkgset(compose, koji_wrapper, path_prefix, event): for compose_tag in compose_tags: compose.log_info("Loading package set for tag %s", compose_tag) if compose_tag in pkgset_koji_tags: - extra_builds = force_list(compose.conf.get("pkgset_koji_builds", [])) - extra_tasks = force_list(compose.conf.get("pkgset_koji_scratch_tasks", [])) + extra_builds = force_list( + compose.conf.get("pkgset_koji_builds", [])) + extra_tasks = force_list( + compose.conf.get("pkgset_koji_scratch_tasks", [])) else: extra_builds = [] extra_tasks = [] @@ -837,6 +907,7 @@ def populate_global_pkgset(compose, koji_wrapper, path_prefix, event): cache_region=compose.cache_region, extra_builds=extra_builds, extra_tasks=extra_tasks, + ) # Check if we have cache for this tag from previous compose. If so, use @@ -847,6 +918,7 @@ def populate_global_pkgset(compose, koji_wrapper, path_prefix, event): if old_cache_path: pkgset.set_old_file_cache( pungi.phases.pkgset.pkgsets.KojiPackageSet.load_old_file_cache( + old_cache_path ) ) @@ -926,7 +998,8 @@ def populate_global_pkgset(compose, koji_wrapper, path_prefix, event): def get_koji_event_info(compose, koji_wrapper): - event_file = os.path.join(compose.paths.work.topdir(arch="global"), "koji-event") + event_file = os.path.join(compose.paths.work.topdir(arch="global"), + "koji-event") compose.log_info("Getting koji event") result = get_koji_event_raw(koji_wrapper, compose.koji_event, event_file) From fcc9b4f1cad392c0f69e3594adf729c40b333705 Mon Sep 17 00:00:00 2001 From: soksanichenko Date: Mon, 20 Mar 2023 13:25:45 +0200 Subject: [PATCH 3/5] ALBS-1030: Generate Devel section in packages.json - Skip verifying an RPM signature if sigkeys are empty --- pungi/phases/pkgset/pkgsets.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/pungi/phases/pkgset/pkgsets.py b/pungi/phases/pkgset/pkgsets.py index a556f139..c97d49fd 100644 --- a/pungi/phases/pkgset/pkgsets.py +++ b/pungi/phases/pkgset/pkgsets.py @@ -889,6 +889,8 @@ class KojiMockPackageSet(KojiPackageSet): sigkey.lower() for sigkey in self.sigkey_ordering if sigkey is not None ] + if not sigkeys: + return True with open(rpm_path, 'rb') as fd: header = ts.hdrFromFdno(fd) signature = header[rpm.RPMTAG_SIGGPG] or header[rpm.RPMTAG_SIGPGP] From ed43f0038ec162f9b99ab693713732a3f16b609b Mon Sep 17 00:00:00 2001 From: soksanichenko Date: Mon, 20 Mar 2023 13:55:06 +0200 Subject: [PATCH 4/5] ALBS-1030: Generate Devel section in packages.json - Style fix --- pungi/phases/pkgset/sources/source_kojimock.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pungi/phases/pkgset/sources/source_kojimock.py b/pungi/phases/pkgset/sources/source_kojimock.py index b8b749d7..52e4272e 100644 --- a/pungi/phases/pkgset/sources/source_kojimock.py +++ b/pungi/phases/pkgset/sources/source_kojimock.py @@ -193,8 +193,8 @@ def get_koji_modules(compose, koji_wrapper, event, module_info_str): ) latest_version = sorted_modules[0]["module_version"] modules = [ - module for module in modules if - latest_version == module["module_version"] + module for module in modules + if latest_version == module["module_version"] ] return modules From 53ed7386f3fee99d5b34f7106999d3f221705820 Mon Sep 17 00:00:00 2001 From: soksanichenko Date: Mon, 20 Mar 2023 13:56:44 +0200 Subject: [PATCH 5/5] ALBS-1030: Generate Devel section in packages.json - Redundant empty lines are removed --- pungi/phases/pkgset/sources/source_kojimock.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/pungi/phases/pkgset/sources/source_kojimock.py b/pungi/phases/pkgset/sources/source_kojimock.py index 52e4272e..6391ad4f 100644 --- a/pungi/phases/pkgset/sources/source_kojimock.py +++ b/pungi/phases/pkgset/sources/source_kojimock.py @@ -907,7 +907,6 @@ def populate_global_pkgset(compose, koji_wrapper, path_prefix, event): cache_region=compose.cache_region, extra_builds=extra_builds, extra_tasks=extra_tasks, - ) # Check if we have cache for this tag from previous compose. If so, use @@ -918,7 +917,6 @@ def populate_global_pkgset(compose, koji_wrapper, path_prefix, event): if old_cache_path: pkgset.set_old_file_cache( pungi.phases.pkgset.pkgsets.KojiPackageSet.load_old_file_cache( - old_cache_path ) )