almalinux/pungi
14
1
Fork 0
Code Issues Pull Requests Releases Activity

ALBS-1030: Generate Devel section in packages.json

Browse Source 
- Some upstream changes to KojiMock parts
This commit is contained in:
soksanichenko 2023-03-19 21:11:12 +02:00
parent f0bd1af999
commit d32c293bca
1 changed files with 132 additions and 59 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

191
pungi/phases/pkgset/sources/source_kojimock.py
View File

@ -35,10 +35,12 @@ import pungi.wrappers.kojiwrapper
from pungi.wrappers.comps import CompsWrapper from pungi.wrappers.comps import CompsWrapper
from pungi.wrappers.mbs import MBSWrapper from pungi.wrappers.mbs import MBSWrapper
import pungi.phases.pkgset.pkgsets import pungi.phases.pkgset.pkgsets
from pungi.util import ( from pungi.util import (
retry, retry,
get_arch_variant_data, get_arch_variant_data,
get_variant_data, get_variant_data,
read_single_module_stream_from_string, read_single_module_stream_from_string,
read_single_module_stream_from_file, read_single_module_stream_from_file,
) )
@ -101,12 +103,12 @@ def variant_dict_from_str(compose, module_str):
release_regex = re.compile(r"^(\d){14}$") release_regex = re.compile(r"^(\d){14}$")
section_start = module_str.rfind("-") section_start = module_str.rfind("-")
module_str_first_part = module_str[section_start + 1 :] module_str_first_part = module_str[section_start + 1:]
if release_regex.match(module_str_first_part): if release_regex.match(module_str_first_part):
module_info["version"] = module_str_first_part module_info["version"] = module_str_first_part
module_str = module_str[:section_start] module_str = module_str[:section_start]
section_start = module_str.rfind("-") section_start = module_str.rfind("-")
module_info["stream"] = module_str[section_start + 1 :] module_info["stream"] = module_str[section_start + 1:]
else: else:
module_info["stream"] = module_str_first_part module_info["stream"] = module_str_first_part
module_info["name"] = module_str[:section_start] module_info["name"] = module_str[:section_start]
@ -160,14 +162,16 @@ def get_koji_modules(compose, koji_wrapper, event, module_info_str):
# Store module versioning information into the dict, but make sure # Store module versioning information into the dict, but make sure
# not to overwrite any existing keys. # not to overwrite any existing keys.
md["module_stream"] = md["extra"]["typeinfo"]["module"]["stream"] md["module_stream"] = md["extra"]["typeinfo"]["module"]["stream"]
md["module_version"] = int(md["extra"]["typeinfo"]["module"]["version"]) md["module_version"] = int(
md["extra"]["typeinfo"]["module"]["version"])
md["module_context"] = md["extra"]["typeinfo"]["module"]["context"] md["module_context"] = md["extra"]["typeinfo"]["module"]["context"]
except KeyError: except KeyError:
continue continue
if md["state"] == pungi.wrappers.kojiwrapper.KOJI_BUILD_DELETED: if md["state"] == pungi.wrappers.kojiwrapper.KOJI_BUILD_DELETED:
compose.log_debug( compose.log_debug(
"Module build %s has been deleted, ignoring it." % build["name"] "Module build %s has been deleted, ignoring it." % build[
"name"]
) )
continue continue
@ -189,7 +193,8 @@ def get_koji_modules(compose, koji_wrapper, event, module_info_str):
) )
latest_version = sorted_modules[0]["module_version"] latest_version = sorted_modules[0]["module_version"]
modules = [ modules = [
module for module in modules if latest_version == module["module_version"] module for module in modules if
latest_version == module["module_version"]
] ]
return modules return modules
@ -205,7 +210,8 @@ class PkgsetSourceKojiMock(pungi.phases.pkgset.source.PkgsetSourceBase):
get_all_arches(compose), get_all_arches(compose),
) )
# path prefix must contain trailing '/' # path prefix must contain trailing '/'
path_prefix = self.koji_wrapper.koji_module.config.topdir.rstrip("/") + "/" path_prefix = self.koji_wrapper.koji_module.config.topdir.rstrip(
"/") + "/"
package_sets = get_pkgset_from_koji( package_sets = get_pkgset_from_koji(
self.compose, self.koji_wrapper, path_prefix self.compose, self.koji_wrapper, path_prefix
) )
@ -214,16 +220,17 @@ class PkgsetSourceKojiMock(pungi.phases.pkgset.source.PkgsetSourceBase):
def get_pkgset_from_koji(compose, koji_wrapper, path_prefix): def get_pkgset_from_koji(compose, koji_wrapper, path_prefix):
event_info = get_koji_event_info(compose, koji_wrapper) event_info = get_koji_event_info(compose, koji_wrapper)
return populate_global_pkgset(compose, koji_wrapper, path_prefix, event_info) return populate_global_pkgset(compose, koji_wrapper, path_prefix,
event_info)
def _add_module_to_variant( def _add_module_to_variant(
koji_wrapper, koji_wrapper,
variant, variant,
build, build,
add_to_variant_modules=False, add_to_variant_modules=False,
compose=None, compose=None,
exclude_module_ns=None, exclude_module_ns=None,
): ):
""" """
Adds module defined by Koji build info to variant. Adds module defined by Koji build info to variant.
@ -241,13 +248,16 @@ def _add_module_to_variant(
if archive["btype"] != "module": if archive["btype"] != "module":
# Skip non module archives # Skip non module archives
continue continue
filename = archive["filename"] filename = archive["filename"]
file_path = os.path.join( file_path = os.path.join(
koji_wrapper.koji_module.pathinfo.topdir, koji_wrapper.koji_module.pathinfo.topdir,
'modules', 'modules',
build['arch'], build['arch'],
build['extra']['typeinfo']['module']['content_koji_tag'] build['extra']['typeinfo']['module']['content_koji_tag']
) )
mmds[filename] = file_path mmds[filename] = file_path
if len(mmds) <= 1: if len(mmds) <= 1:
@ -266,17 +276,22 @@ def _add_module_to_variant(
added = False added = False
for arch in variant.arches: for arch in variant.arches:
if _is_filtered_out(compose, variant, arch, info["name"], info["stream"]): if _is_filtered_out(compose, variant, arch, info["name"],
compose.log_debug("Module %s is filtered from %s.%s", nsvc, variant, arch) info["stream"]):
compose.log_debug("Module %s is filtered from %s.%s", nsvc,
variant, arch)
continue continue
filename = "modulemd.%s.txt" % arch filename = "modulemd.%s.txt" % arch
try: try:
mod_stream = read_single_module_stream_from_file( mod_stream = read_single_module_stream_from_file(
mmds[filename], compose, arch, build mmds[filename], compose, arch, build
) )
if mod_stream: if mod_stream:
added = True added = True
variant.arch_mmds.setdefault(arch, {})[nsvc] = mod_stream variant.arch_mmds.setdefault(arch, {})[nsvc] = mod_stream
added = True added = True
except KeyError: except KeyError:
# There is no modulemd for this arch. This could mean an arch was # There is no modulemd for this arch. This could mean an arch was
@ -295,10 +310,11 @@ def _add_module_to_variant(
def _add_extra_modules_to_variant( def _add_extra_modules_to_variant(
compose, koji_wrapper, variant, extra_modules, variant_tags, tag_to_mmd compose, koji_wrapper, variant, extra_modules, variant_tags, tag_to_mmd
): ):
for nsvc in extra_modules: for nsvc in extra_modules:
msg = "Adding extra module build '%s' to variant '%s'" % (nsvc, variant) msg = "Adding extra module build '%s' to variant '%s'" % (
nsvc, variant)
compose.log_info(msg) compose.log_info(msg)
nsvc_info = nsvc.split(":") nsvc_info = nsvc.split(":")
@ -341,10 +357,11 @@ def _add_extra_modules_to_variant(
def _add_scratch_modules_to_variant( def _add_scratch_modules_to_variant(
compose, variant, scratch_modules, variant_tags, tag_to_mmd compose, variant, scratch_modules, variant_tags, tag_to_mmd
): ):
if compose.compose_type != "test" and scratch_modules: if compose.compose_type != "test" and scratch_modules:
compose.log_warning("Only test composes could include scratch module builds") compose.log_warning(
"Only test composes could include scratch module builds")
return return
mbs = MBSWrapper(compose.conf["mbs_api_url"]) mbs = MBSWrapper(compose.conf["mbs_api_url"])
@ -355,7 +372,8 @@ def _add_scratch_modules_to_variant(
try: try:
final_modulemd = mbs.final_modulemd(module_build["id"]) final_modulemd = mbs.final_modulemd(module_build["id"])
except Exception: except Exception:
compose.log_error("Unable to get modulemd for build %s" % module_build) compose.log_error(
"Unable to get modulemd for build %s" % module_build)
raise raise
tag = module_build["koji_tag"] tag = module_build["koji_tag"]
variant_tags[variant].append(tag) variant_tags[variant].append(tag)
@ -363,8 +381,7 @@ def _add_scratch_modules_to_variant(
for arch in variant.arches: for arch in variant.arches:
try: try:
mmd = read_single_module_stream_from_string( mmd = read_single_module_stream_from_string(
final_modulemd[arch] final_modulemd[arch])
)
variant.arch_mmds.setdefault(arch, {})[nsvc] = mmd variant.arch_mmds.setdefault(arch, {})[nsvc] = mmd
except KeyError: except KeyError:
continue continue
@ -390,21 +407,24 @@ def _is_filtered_out(compose, variant, arch, module_name, module_stream):
if not compose: if not compose:
return False return False
for filter in get_arch_variant_data(compose.conf, "filter_modules", arch, variant): for filter in get_arch_variant_data(compose.conf, "filter_modules", arch,
variant):
if ":" not in filter: if ":" not in filter:
name_filter = filter name_filter = filter
stream_filter = "*" stream_filter = "*"
else: else:
name_filter, stream_filter = filter.split(":", 1) name_filter, stream_filter = filter.split(":", 1)
if fnmatch(module_name, name_filter) and fnmatch(module_stream, stream_filter): if fnmatch(module_name, name_filter) and fnmatch(module_stream,
stream_filter):
return True return True
return False return False
def _get_modules_from_koji( def _get_modules_from_koji(
compose, koji_wrapper, event, variant, variant_tags, tag_to_mmd compose, koji_wrapper, event, variant, variant_tags, tag_to_mmd,
exclude_module_ns
): ):
""" """
Loads modules for given `variant` from koji `session`, adds them to Loads modules for given `variant` from koji `session`, adds them to
@ -415,15 +435,21 @@ def _get_modules_from_koji(
:param Variant variant: Variant with modules to find. :param Variant variant: Variant with modules to find.
:param dict variant_tags: Dict populated by this method. Key is `variant` :param dict variant_tags: Dict populated by this method. Key is `variant`
and value is list of Koji tags to get the RPMs from. and value is list of Koji tags to get the RPMs from.
:param list exclude_module_ns: Module name:stream which will be excluded.
""" """
# Find out all modules in every variant and add their Koji tags # Find out all modules in every variant and add their Koji tags
# to variant and variant_tags list. # to variant and variant_tags list.
for module in variant.get_modules(): for module in variant.get_modules():
koji_modules = get_koji_modules(compose, koji_wrapper, event, module["name"]) koji_modules = get_koji_modules(compose, koji_wrapper, event,
module["name"])
for koji_module in koji_modules: for koji_module in koji_modules:
nsvc = _add_module_to_variant( nsvc = _add_module_to_variant(
koji_wrapper, variant, koji_module, compose=compose koji_wrapper,
variant,
koji_module,
compose=compose,
exclude_module_ns=exclude_module_ns,
) )
if not nsvc: if not nsvc:
continue continue
@ -462,7 +488,8 @@ def filter_inherited(koji_proxy, event, module_builds, top_tag):
does not understand streams, so we have to reimplement it here. does not understand streams, so we have to reimplement it here.
""" """
inheritance = [ inheritance = [
tag["name"] for tag in koji_proxy.getFullInheritance(top_tag, event=event["id"]) tag["name"] for tag in
koji_proxy.getFullInheritance(top_tag, event=event["id"])
] ]
def keyfunc(mb): def keyfunc(mb):
@ -487,7 +514,8 @@ def filter_inherited(koji_proxy, event, module_builds, top_tag):
return result return result
def filter_by_whitelist(compose, module_builds, input_modules, expected_modules): def filter_by_whitelist(compose, module_builds, input_modules,
expected_modules):
""" """
Exclude modules from the list that do not match any pattern specified in Exclude modules from the list that do not match any pattern specified in
input_modules. Order may not be preserved. The last argument is a set of input_modules. Order may not be preserved. The last argument is a set of
@ -511,6 +539,7 @@ def filter_by_whitelist(compose, module_builds, input_modules, expected_modules)
info.get("context"), info.get("context"),
) )
nvr_patterns.add((pattern, spec["name"])) nvr_patterns.add((pattern, spec["name"]))
modules_to_keep = [] modules_to_keep = []
for mb in sorted(module_builds, key=lambda i: i['name']): for mb in sorted(module_builds, key=lambda i: i['name']):
@ -521,13 +550,13 @@ def filter_by_whitelist(compose, module_builds, input_modules, expected_modules)
# and context. # and context.
for (n, s, v, c), spec in sorted(nvr_patterns): for (n, s, v, c), spec in sorted(nvr_patterns):
if ( if (
# We always have a name and stream... # We always have a name and stream...
mb["name"] == n mb["name"] == n
and mb["version"] == s and mb["version"] == s
# ...but version and context can be missing, in which case we # ...but version and context can be missing, in which case we
# don't want to check them. # don't want to check them.
and (not v or ver == v) and (not v or ver == v)
and (not c or ctx == c) and (not c or ctx == c)
): ):
modules_to_keep.append(mb) modules_to_keep.append(mb)
expected_modules.discard(spec) expected_modules.discard(spec)
@ -575,7 +604,13 @@ def _filter_expected_modules(
def _get_modules_from_koji_tags( def _get_modules_from_koji_tags(
compose, koji_wrapper, event_id, variant, variant_tags, tag_to_mmd compose,
koji_wrapper,
event_id,
variant,
variant_tags,
tag_to_mmd,
exclude_module_ns,
): ):
""" """
Loads modules for given `variant` from Koji, adds them to Loads modules for given `variant` from Koji, adds them to
@ -587,10 +622,12 @@ def _get_modules_from_koji_tags(
:param Variant variant: Variant with modules to find. :param Variant variant: Variant with modules to find.
:param dict variant_tags: Dict populated by this method. Key is `variant` :param dict variant_tags: Dict populated by this method. Key is `variant`
and value is list of Koji tags to get the RPMs from. and value is list of Koji tags to get the RPMs from.
:param list exclude_module_ns: Module name:stream which will be excluded.
""" """
# Compose tags from configuration # Compose tags from configuration
compose_tags = [ compose_tags = [
{"name": tag} for tag in force_list(compose.conf["pkgset_koji_module_tag"]) {"name": tag} for tag in
force_list(compose.conf["pkgset_koji_module_tag"])
] ]
# Get set of configured module names for this variant. If nothing is # Get set of configured module names for this variant. If nothing is
# configured, the set is empty. # configured, the set is empty.
@ -617,7 +654,8 @@ def _get_modules_from_koji_tags(
) )
# Filter out builds inherited from non-top tag # Filter out builds inherited from non-top tag
module_builds = filter_inherited(koji_proxy, event_id, module_builds, tag) module_builds = filter_inherited(koji_proxy, event_id, module_builds,
tag)
# Apply whitelist of modules if specified. # Apply whitelist of modules if specified.
variant_modules = variant.get_modules() variant_modules = variant.get_modules()
@ -625,6 +663,7 @@ def _get_modules_from_koji_tags(
module_builds = filter_by_whitelist( module_builds = filter_by_whitelist(
compose, module_builds, variant_modules, expected_modules compose, module_builds, variant_modules, expected_modules
) )
# Find the latest builds of all modules. This does following: # Find the latest builds of all modules. This does following:
# - Sorts the module_builds descending by Koji NVR (which maps to NSV # - Sorts the module_builds descending by Koji NVR (which maps to NSV
# for modules). Split release into modular version and context, and # for modules). Split release into modular version and context, and
@ -645,14 +684,14 @@ def _get_modules_from_koji_tags(
latest_builds = [] latest_builds = []
module_builds = sorted(module_builds, key=_key, reverse=True) module_builds = sorted(module_builds, key=_key, reverse=True)
for ns, ns_builds in groupby( for ns, ns_builds in groupby(
module_builds, key=lambda x: ":".join([ module_builds, key=lambda x: ":".join([
x["name"], x["name"],
x["version"], x["version"],
x['arch'], x['arch'],
]) ])
): ):
for nsv, nsv_builds in groupby( for nsv, nsv_builds in groupby(
ns_builds, key=lambda x: x["release"].split(".")[0] ns_builds, key=lambda x: x["release"].split(".")[0]
): ):
latest_builds += list(nsv_builds) latest_builds += list(nsv_builds)
break break
@ -662,6 +701,18 @@ def _get_modules_from_koji_tags(
for build in latest_builds: for build in latest_builds:
# Get the Build from Koji to get modulemd and module_tag. # Get the Build from Koji to get modulemd and module_tag.
build = koji_proxy.getBuild(build["build_id"]) build = koji_proxy.getBuild(build["build_id"])
nsvc = _add_module_to_variant(
koji_wrapper,
variant,
build,
True,
compose=compose,
exclude_module_ns=exclude_module_ns,
)
if not nsvc:
continue
module_tag = ( module_tag = (
build.get("extra", {}) build.get("extra", {})
.get("typeinfo", {}) .get("typeinfo", {})
@ -671,12 +722,6 @@ def _get_modules_from_koji_tags(
variant_tags[variant].append(module_tag) variant_tags[variant].append(module_tag)
nsvc = _add_module_to_variant(
koji_wrapper, variant, build, True, compose=compose
)
if not nsvc:
continue
tag_to_mmd.setdefault(module_tag, {}) tag_to_mmd.setdefault(module_tag, {})
for arch in variant.arch_mmds: for arch in variant.arch_mmds:
try: try:
@ -736,7 +781,7 @@ def populate_global_pkgset(compose, koji_wrapper, path_prefix, event):
for group in groups: for group in groups:
packages_to_gather += comps.get_packages(group) packages_to_gather += comps.get_packages(group)
if compose.conf["gather_method"] == "nodeps" and not compose.conf.get( if compose.conf["gather_method"] == "nodeps" and not compose.conf.get(
"buildinstall_method" "buildinstall_method"
): ):
populate_only_packages_to_gather = True populate_only_packages_to_gather = True
else: else:
@ -767,26 +812,48 @@ def populate_global_pkgset(compose, koji_wrapper, path_prefix, event):
"modules." "modules."
) )
extra_modules = get_variant_data(
compose.conf, "pkgset_koji_module_builds", variant
)
# When adding extra modules, other modules of the same name:stream available
# in brew tag should be excluded.
exclude_module_ns = []
if extra_modules:
exclude_module_ns = [
":".join(nsvc.split(":")[:2]) for nsvc in extra_modules
]
if modular_koji_tags or ( if modular_koji_tags or (
compose.conf["pkgset_koji_module_tag"] and variant.modules compose.conf["pkgset_koji_module_tag"] and variant.modules
): ):
# List modules tagged in particular tags. # List modules tagged in particular tags.
_get_modules_from_koji_tags( _get_modules_from_koji_tags(
compose, koji_wrapper, event, variant, variant_tags, tag_to_mmd compose,
koji_wrapper,
event,
variant,
variant_tags,
tag_to_mmd,
exclude_module_ns,
) )
elif variant.modules: elif variant.modules:
# Search each module in Koji separately. Tagging does not come into # Search each module in Koji separately. Tagging does not come into
# play here. # play here.
_get_modules_from_koji( _get_modules_from_koji(
compose, koji_wrapper, event, variant, variant_tags, tag_to_mmd compose,
koji_wrapper,
event,
variant,
variant_tags,
tag_to_mmd,
exclude_module_ns,
) )
extra_modules = get_variant_data(
compose.conf, "pkgset_koji_module_builds", variant
)
if extra_modules: if extra_modules:
_add_extra_modules_to_variant( _add_extra_modules_to_variant(
compose, koji_wrapper, variant, extra_modules, variant_tags, tag_to_mmd compose, koji_wrapper, variant, extra_modules, variant_tags,
tag_to_mmd
) )
variant_scratch_modules = get_variant_data( variant_scratch_modules = get_variant_data(
@ -794,7 +861,8 @@ def populate_global_pkgset(compose, koji_wrapper, path_prefix, event):
) )
if variant_scratch_modules: if variant_scratch_modules:
_add_scratch_modules_to_variant( _add_scratch_modules_to_variant(
compose, variant, variant_scratch_modules, variant_tags, tag_to_mmd compose, variant, variant_scratch_modules, variant_tags,
tag_to_mmd
) )
# Ensure that every tag added to `variant_tags` is added also to # Ensure that every tag added to `variant_tags` is added also to
@ -819,8 +887,10 @@ def populate_global_pkgset(compose, koji_wrapper, path_prefix, event):
for compose_tag in compose_tags: for compose_tag in compose_tags:
compose.log_info("Loading package set for tag %s", compose_tag) compose.log_info("Loading package set for tag %s", compose_tag)
if compose_tag in pkgset_koji_tags: if compose_tag in pkgset_koji_tags:
extra_builds = force_list(compose.conf.get("pkgset_koji_builds", [])) extra_builds = force_list(
extra_tasks = force_list(compose.conf.get("pkgset_koji_scratch_tasks", [])) compose.conf.get("pkgset_koji_builds", []))
extra_tasks = force_list(
compose.conf.get("pkgset_koji_scratch_tasks", []))
else: else:
extra_builds = [] extra_builds = []
extra_tasks = [] extra_tasks = []
@ -837,6 +907,7 @@ def populate_global_pkgset(compose, koji_wrapper, path_prefix, event):
cache_region=compose.cache_region, cache_region=compose.cache_region,
extra_builds=extra_builds, extra_builds=extra_builds,
extra_tasks=extra_tasks, extra_tasks=extra_tasks,
) )
# Check if we have cache for this tag from previous compose. If so, use # Check if we have cache for this tag from previous compose. If so, use
@ -847,6 +918,7 @@ def populate_global_pkgset(compose, koji_wrapper, path_prefix, event):
if old_cache_path: if old_cache_path:
pkgset.set_old_file_cache( pkgset.set_old_file_cache(
pungi.phases.pkgset.pkgsets.KojiPackageSet.load_old_file_cache( pungi.phases.pkgset.pkgsets.KojiPackageSet.load_old_file_cache(
old_cache_path old_cache_path
) )
) )
@ -926,7 +998,8 @@ def populate_global_pkgset(compose, koji_wrapper, path_prefix, event):
def get_koji_event_info(compose, koji_wrapper): def get_koji_event_info(compose, koji_wrapper):
event_file = os.path.join(compose.paths.work.topdir(arch="global"), "koji-event") event_file = os.path.join(compose.paths.work.topdir(arch="global"),
"koji-event")
compose.log_info("Getting koji event") compose.log_info("Getting koji event")
result = get_koji_event_raw(koji_wrapper, compose.koji_event, event_file) result = get_koji_event_raw(koji_wrapper, compose.koji_event, event_file)