diff --git a/pungi/phases/pkgset/sources/source_kojimock.py b/pungi/phases/pkgset/sources/source_kojimock.py index aa063a18..b8b749d7 100644 --- a/pungi/phases/pkgset/sources/source_kojimock.py +++ b/pungi/phases/pkgset/sources/source_kojimock.py @@ -35,10 +35,12 @@ import pungi.wrappers.kojiwrapper from pungi.wrappers.comps import CompsWrapper from pungi.wrappers.mbs import MBSWrapper import pungi.phases.pkgset.pkgsets + from pungi.util import ( retry, get_arch_variant_data, get_variant_data, + read_single_module_stream_from_string, read_single_module_stream_from_file, ) @@ -101,12 +103,12 @@ def variant_dict_from_str(compose, module_str): release_regex = re.compile(r"^(\d){14}$") section_start = module_str.rfind("-") - module_str_first_part = module_str[section_start + 1 :] + module_str_first_part = module_str[section_start + 1:] if release_regex.match(module_str_first_part): module_info["version"] = module_str_first_part module_str = module_str[:section_start] section_start = module_str.rfind("-") - module_info["stream"] = module_str[section_start + 1 :] + module_info["stream"] = module_str[section_start + 1:] else: module_info["stream"] = module_str_first_part module_info["name"] = module_str[:section_start] @@ -160,14 +162,16 @@ def get_koji_modules(compose, koji_wrapper, event, module_info_str): # Store module versioning information into the dict, but make sure # not to overwrite any existing keys. md["module_stream"] = md["extra"]["typeinfo"]["module"]["stream"] - md["module_version"] = int(md["extra"]["typeinfo"]["module"]["version"]) + md["module_version"] = int( + md["extra"]["typeinfo"]["module"]["version"]) md["module_context"] = md["extra"]["typeinfo"]["module"]["context"] except KeyError: continue if md["state"] == pungi.wrappers.kojiwrapper.KOJI_BUILD_DELETED: compose.log_debug( - "Module build %s has been deleted, ignoring it." % build["name"] + "Module build %s has been deleted, ignoring it." % build[ + "name"] ) continue @@ -189,7 +193,8 @@ def get_koji_modules(compose, koji_wrapper, event, module_info_str): ) latest_version = sorted_modules[0]["module_version"] modules = [ - module for module in modules if latest_version == module["module_version"] + module for module in modules if + latest_version == module["module_version"] ] return modules @@ -205,7 +210,8 @@ class PkgsetSourceKojiMock(pungi.phases.pkgset.source.PkgsetSourceBase): get_all_arches(compose), ) # path prefix must contain trailing '/' - path_prefix = self.koji_wrapper.koji_module.config.topdir.rstrip("/") + "/" + path_prefix = self.koji_wrapper.koji_module.config.topdir.rstrip( + "/") + "/" package_sets = get_pkgset_from_koji( self.compose, self.koji_wrapper, path_prefix ) @@ -214,16 +220,17 @@ class PkgsetSourceKojiMock(pungi.phases.pkgset.source.PkgsetSourceBase): def get_pkgset_from_koji(compose, koji_wrapper, path_prefix): event_info = get_koji_event_info(compose, koji_wrapper) - return populate_global_pkgset(compose, koji_wrapper, path_prefix, event_info) + return populate_global_pkgset(compose, koji_wrapper, path_prefix, + event_info) def _add_module_to_variant( - koji_wrapper, - variant, - build, - add_to_variant_modules=False, - compose=None, - exclude_module_ns=None, + koji_wrapper, + variant, + build, + add_to_variant_modules=False, + compose=None, + exclude_module_ns=None, ): """ Adds module defined by Koji build info to variant. @@ -241,13 +248,16 @@ def _add_module_to_variant( if archive["btype"] != "module": # Skip non module archives continue + filename = archive["filename"] file_path = os.path.join( koji_wrapper.koji_module.pathinfo.topdir, 'modules', build['arch'], build['extra']['typeinfo']['module']['content_koji_tag'] + ) + mmds[filename] = file_path if len(mmds) <= 1: @@ -266,17 +276,22 @@ def _add_module_to_variant( added = False for arch in variant.arches: - if _is_filtered_out(compose, variant, arch, info["name"], info["stream"]): - compose.log_debug("Module %s is filtered from %s.%s", nsvc, variant, arch) + if _is_filtered_out(compose, variant, arch, info["name"], + info["stream"]): + compose.log_debug("Module %s is filtered from %s.%s", nsvc, + variant, arch) continue + filename = "modulemd.%s.txt" % arch try: mod_stream = read_single_module_stream_from_file( mmds[filename], compose, arch, build + ) if mod_stream: added = True variant.arch_mmds.setdefault(arch, {})[nsvc] = mod_stream + added = True except KeyError: # There is no modulemd for this arch. This could mean an arch was @@ -295,10 +310,11 @@ def _add_module_to_variant( def _add_extra_modules_to_variant( - compose, koji_wrapper, variant, extra_modules, variant_tags, tag_to_mmd + compose, koji_wrapper, variant, extra_modules, variant_tags, tag_to_mmd ): for nsvc in extra_modules: - msg = "Adding extra module build '%s' to variant '%s'" % (nsvc, variant) + msg = "Adding extra module build '%s' to variant '%s'" % ( + nsvc, variant) compose.log_info(msg) nsvc_info = nsvc.split(":") @@ -341,10 +357,11 @@ def _add_extra_modules_to_variant( def _add_scratch_modules_to_variant( - compose, variant, scratch_modules, variant_tags, tag_to_mmd + compose, variant, scratch_modules, variant_tags, tag_to_mmd ): if compose.compose_type != "test" and scratch_modules: - compose.log_warning("Only test composes could include scratch module builds") + compose.log_warning( + "Only test composes could include scratch module builds") return mbs = MBSWrapper(compose.conf["mbs_api_url"]) @@ -355,7 +372,8 @@ def _add_scratch_modules_to_variant( try: final_modulemd = mbs.final_modulemd(module_build["id"]) except Exception: - compose.log_error("Unable to get modulemd for build %s" % module_build) + compose.log_error( + "Unable to get modulemd for build %s" % module_build) raise tag = module_build["koji_tag"] variant_tags[variant].append(tag) @@ -363,8 +381,7 @@ def _add_scratch_modules_to_variant( for arch in variant.arches: try: mmd = read_single_module_stream_from_string( - final_modulemd[arch] - ) + final_modulemd[arch]) variant.arch_mmds.setdefault(arch, {})[nsvc] = mmd except KeyError: continue @@ -390,21 +407,24 @@ def _is_filtered_out(compose, variant, arch, module_name, module_stream): if not compose: return False - for filter in get_arch_variant_data(compose.conf, "filter_modules", arch, variant): + for filter in get_arch_variant_data(compose.conf, "filter_modules", arch, + variant): if ":" not in filter: name_filter = filter stream_filter = "*" else: name_filter, stream_filter = filter.split(":", 1) - if fnmatch(module_name, name_filter) and fnmatch(module_stream, stream_filter): + if fnmatch(module_name, name_filter) and fnmatch(module_stream, + stream_filter): return True return False def _get_modules_from_koji( - compose, koji_wrapper, event, variant, variant_tags, tag_to_mmd + compose, koji_wrapper, event, variant, variant_tags, tag_to_mmd, + exclude_module_ns ): """ Loads modules for given `variant` from koji `session`, adds them to @@ -415,15 +435,21 @@ def _get_modules_from_koji( :param Variant variant: Variant with modules to find. :param dict variant_tags: Dict populated by this method. Key is `variant` and value is list of Koji tags to get the RPMs from. + :param list exclude_module_ns: Module name:stream which will be excluded. """ # Find out all modules in every variant and add their Koji tags # to variant and variant_tags list. for module in variant.get_modules(): - koji_modules = get_koji_modules(compose, koji_wrapper, event, module["name"]) + koji_modules = get_koji_modules(compose, koji_wrapper, event, + module["name"]) for koji_module in koji_modules: nsvc = _add_module_to_variant( - koji_wrapper, variant, koji_module, compose=compose + koji_wrapper, + variant, + koji_module, + compose=compose, + exclude_module_ns=exclude_module_ns, ) if not nsvc: continue @@ -462,7 +488,8 @@ def filter_inherited(koji_proxy, event, module_builds, top_tag): does not understand streams, so we have to reimplement it here. """ inheritance = [ - tag["name"] for tag in koji_proxy.getFullInheritance(top_tag, event=event["id"]) + tag["name"] for tag in + koji_proxy.getFullInheritance(top_tag, event=event["id"]) ] def keyfunc(mb): @@ -487,7 +514,8 @@ def filter_inherited(koji_proxy, event, module_builds, top_tag): return result -def filter_by_whitelist(compose, module_builds, input_modules, expected_modules): +def filter_by_whitelist(compose, module_builds, input_modules, + expected_modules): """ Exclude modules from the list that do not match any pattern specified in input_modules. Order may not be preserved. The last argument is a set of @@ -511,6 +539,7 @@ def filter_by_whitelist(compose, module_builds, input_modules, expected_modules) info.get("context"), ) nvr_patterns.add((pattern, spec["name"])) + modules_to_keep = [] for mb in sorted(module_builds, key=lambda i: i['name']): @@ -521,13 +550,13 @@ def filter_by_whitelist(compose, module_builds, input_modules, expected_modules) # and context. for (n, s, v, c), spec in sorted(nvr_patterns): if ( - # We always have a name and stream... - mb["name"] == n - and mb["version"] == s - # ...but version and context can be missing, in which case we - # don't want to check them. - and (not v or ver == v) - and (not c or ctx == c) + # We always have a name and stream... + mb["name"] == n + and mb["version"] == s + # ...but version and context can be missing, in which case we + # don't want to check them. + and (not v or ver == v) + and (not c or ctx == c) ): modules_to_keep.append(mb) expected_modules.discard(spec) @@ -575,7 +604,13 @@ def _filter_expected_modules( def _get_modules_from_koji_tags( - compose, koji_wrapper, event_id, variant, variant_tags, tag_to_mmd + compose, + koji_wrapper, + event_id, + variant, + variant_tags, + tag_to_mmd, + exclude_module_ns, ): """ Loads modules for given `variant` from Koji, adds them to @@ -587,10 +622,12 @@ def _get_modules_from_koji_tags( :param Variant variant: Variant with modules to find. :param dict variant_tags: Dict populated by this method. Key is `variant` and value is list of Koji tags to get the RPMs from. + :param list exclude_module_ns: Module name:stream which will be excluded. """ # Compose tags from configuration compose_tags = [ - {"name": tag} for tag in force_list(compose.conf["pkgset_koji_module_tag"]) + {"name": tag} for tag in + force_list(compose.conf["pkgset_koji_module_tag"]) ] # Get set of configured module names for this variant. If nothing is # configured, the set is empty. @@ -617,7 +654,8 @@ def _get_modules_from_koji_tags( ) # Filter out builds inherited from non-top tag - module_builds = filter_inherited(koji_proxy, event_id, module_builds, tag) + module_builds = filter_inherited(koji_proxy, event_id, module_builds, + tag) # Apply whitelist of modules if specified. variant_modules = variant.get_modules() @@ -625,6 +663,7 @@ def _get_modules_from_koji_tags( module_builds = filter_by_whitelist( compose, module_builds, variant_modules, expected_modules ) + # Find the latest builds of all modules. This does following: # - Sorts the module_builds descending by Koji NVR (which maps to NSV # for modules). Split release into modular version and context, and @@ -645,14 +684,14 @@ def _get_modules_from_koji_tags( latest_builds = [] module_builds = sorted(module_builds, key=_key, reverse=True) for ns, ns_builds in groupby( - module_builds, key=lambda x: ":".join([ + module_builds, key=lambda x: ":".join([ x["name"], x["version"], x['arch'], ]) ): for nsv, nsv_builds in groupby( - ns_builds, key=lambda x: x["release"].split(".")[0] + ns_builds, key=lambda x: x["release"].split(".")[0] ): latest_builds += list(nsv_builds) break @@ -662,6 +701,18 @@ def _get_modules_from_koji_tags( for build in latest_builds: # Get the Build from Koji to get modulemd and module_tag. build = koji_proxy.getBuild(build["build_id"]) + + nsvc = _add_module_to_variant( + koji_wrapper, + variant, + build, + True, + compose=compose, + exclude_module_ns=exclude_module_ns, + ) + if not nsvc: + continue + module_tag = ( build.get("extra", {}) .get("typeinfo", {}) @@ -671,12 +722,6 @@ def _get_modules_from_koji_tags( variant_tags[variant].append(module_tag) - nsvc = _add_module_to_variant( - koji_wrapper, variant, build, True, compose=compose - ) - if not nsvc: - continue - tag_to_mmd.setdefault(module_tag, {}) for arch in variant.arch_mmds: try: @@ -736,7 +781,7 @@ def populate_global_pkgset(compose, koji_wrapper, path_prefix, event): for group in groups: packages_to_gather += comps.get_packages(group) if compose.conf["gather_method"] == "nodeps" and not compose.conf.get( - "buildinstall_method" + "buildinstall_method" ): populate_only_packages_to_gather = True else: @@ -767,26 +812,48 @@ def populate_global_pkgset(compose, koji_wrapper, path_prefix, event): "modules." ) + extra_modules = get_variant_data( + compose.conf, "pkgset_koji_module_builds", variant + ) + + # When adding extra modules, other modules of the same name:stream available + # in brew tag should be excluded. + exclude_module_ns = [] + if extra_modules: + exclude_module_ns = [ + ":".join(nsvc.split(":")[:2]) for nsvc in extra_modules + ] + if modular_koji_tags or ( - compose.conf["pkgset_koji_module_tag"] and variant.modules + compose.conf["pkgset_koji_module_tag"] and variant.modules ): # List modules tagged in particular tags. _get_modules_from_koji_tags( - compose, koji_wrapper, event, variant, variant_tags, tag_to_mmd + compose, + koji_wrapper, + event, + variant, + variant_tags, + tag_to_mmd, + exclude_module_ns, ) elif variant.modules: # Search each module in Koji separately. Tagging does not come into # play here. _get_modules_from_koji( - compose, koji_wrapper, event, variant, variant_tags, tag_to_mmd + compose, + koji_wrapper, + event, + variant, + variant_tags, + tag_to_mmd, + exclude_module_ns, ) - extra_modules = get_variant_data( - compose.conf, "pkgset_koji_module_builds", variant - ) if extra_modules: _add_extra_modules_to_variant( - compose, koji_wrapper, variant, extra_modules, variant_tags, tag_to_mmd + compose, koji_wrapper, variant, extra_modules, variant_tags, + tag_to_mmd ) variant_scratch_modules = get_variant_data( @@ -794,7 +861,8 @@ def populate_global_pkgset(compose, koji_wrapper, path_prefix, event): ) if variant_scratch_modules: _add_scratch_modules_to_variant( - compose, variant, variant_scratch_modules, variant_tags, tag_to_mmd + compose, variant, variant_scratch_modules, variant_tags, + tag_to_mmd ) # Ensure that every tag added to `variant_tags` is added also to @@ -819,8 +887,10 @@ def populate_global_pkgset(compose, koji_wrapper, path_prefix, event): for compose_tag in compose_tags: compose.log_info("Loading package set for tag %s", compose_tag) if compose_tag in pkgset_koji_tags: - extra_builds = force_list(compose.conf.get("pkgset_koji_builds", [])) - extra_tasks = force_list(compose.conf.get("pkgset_koji_scratch_tasks", [])) + extra_builds = force_list( + compose.conf.get("pkgset_koji_builds", [])) + extra_tasks = force_list( + compose.conf.get("pkgset_koji_scratch_tasks", [])) else: extra_builds = [] extra_tasks = [] @@ -837,6 +907,7 @@ def populate_global_pkgset(compose, koji_wrapper, path_prefix, event): cache_region=compose.cache_region, extra_builds=extra_builds, extra_tasks=extra_tasks, + ) # Check if we have cache for this tag from previous compose. If so, use @@ -847,6 +918,7 @@ def populate_global_pkgset(compose, koji_wrapper, path_prefix, event): if old_cache_path: pkgset.set_old_file_cache( pungi.phases.pkgset.pkgsets.KojiPackageSet.load_old_file_cache( + old_cache_path ) ) @@ -926,7 +998,8 @@ def populate_global_pkgset(compose, koji_wrapper, path_prefix, event): def get_koji_event_info(compose, koji_wrapper): - event_file = os.path.join(compose.paths.work.topdir(arch="global"), "koji-event") + event_file = os.path.join(compose.paths.work.topdir(arch="global"), + "koji-event") compose.log_info("Getting koji event") result = get_koji_event_raw(koji_wrapper, compose.koji_event, event_file)