ALBS-1030: Generate Devel section in packages.json
- Some upstream changes to KojiMock parts
This commit is contained in:
soksanichenko
parent
f0bd1af999
commit
d32c293bca
|
|
@ -35,10 +35,12 @@ import pungi.wrappers.kojiwrapper
|
|||
from pungi.wrappers.comps import CompsWrapper
|
||||
from pungi.wrappers.mbs import MBSWrapper
|
||||
import pungi.phases.pkgset.pkgsets
|
||||
|
||||
from pungi.util import (
|
||||
retry,
|
||||
get_arch_variant_data,
|
||||
get_variant_data,
|
||||
|
||||
read_single_module_stream_from_string,
|
||||
read_single_module_stream_from_file,
|
||||
)
|
||||
|
|
@ -101,12 +103,12 @@ def variant_dict_from_str(compose, module_str):
|
|||
release_regex = re.compile(r"^(\d){14}$")
|
||||
|
||||
section_start = module_str.rfind("-")
|
||||
module_str_first_part = module_str[section_start + 1 :]
|
||||
module_str_first_part = module_str[section_start + 1:]
|
||||
if release_regex.match(module_str_first_part):
|
||||
module_info["version"] = module_str_first_part
|
||||
module_str = module_str[:section_start]
|
||||
section_start = module_str.rfind("-")
|
||||
module_info["stream"] = module_str[section_start + 1 :]
|
||||
module_info["stream"] = module_str[section_start + 1:]
|
||||
else:
|
||||
module_info["stream"] = module_str_first_part
|
||||
module_info["name"] = module_str[:section_start]
|
||||
|
|
@ -160,14 +162,16 @@ def get_koji_modules(compose, koji_wrapper, event, module_info_str):
|
|||
# Store module versioning information into the dict, but make sure
|
||||
# not to overwrite any existing keys.
|
||||
md["module_stream"] = md["extra"]["typeinfo"]["module"]["stream"]
|
||||
md["module_version"] = int(md["extra"]["typeinfo"]["module"]["version"])
|
||||
md["module_version"] = int(
|
||||
md["extra"]["typeinfo"]["module"]["version"])
|
||||
md["module_context"] = md["extra"]["typeinfo"]["module"]["context"]
|
||||
except KeyError:
|
||||
continue
|
||||
|
||||
if md["state"] == pungi.wrappers.kojiwrapper.KOJI_BUILD_DELETED:
|
||||
compose.log_debug(
|
||||
"Module build %s has been deleted, ignoring it." % build["name"]
|
||||
"Module build %s has been deleted, ignoring it." % build[
|
||||
"name"]
|
||||
)
|
||||
continue
|
||||
|
||||
|
|
@ -189,7 +193,8 @@ def get_koji_modules(compose, koji_wrapper, event, module_info_str):
|
|||
)
|
||||
latest_version = sorted_modules[0]["module_version"]
|
||||
modules = [
|
||||
module for module in modules if latest_version == module["module_version"]
|
||||
module for module in modules if
|
||||
latest_version == module["module_version"]
|
||||
]
|
||||
|
||||
return modules
|
||||
|
|
@ -205,7 +210,8 @@ class PkgsetSourceKojiMock(pungi.phases.pkgset.source.PkgsetSourceBase):
|
|||
get_all_arches(compose),
|
||||
)
|
||||
# path prefix must contain trailing '/'
|
||||
path_prefix = self.koji_wrapper.koji_module.config.topdir.rstrip("/") + "/"
|
||||
path_prefix = self.koji_wrapper.koji_module.config.topdir.rstrip(
|
||||
"/") + "/"
|
||||
package_sets = get_pkgset_from_koji(
|
||||
self.compose, self.koji_wrapper, path_prefix
|
||||
)
|
||||
|
|
@ -214,16 +220,17 @@ class PkgsetSourceKojiMock(pungi.phases.pkgset.source.PkgsetSourceBase):
|
|||
|
||||
def get_pkgset_from_koji(compose, koji_wrapper, path_prefix):
|
||||
event_info = get_koji_event_info(compose, koji_wrapper)
|
||||
return populate_global_pkgset(compose, koji_wrapper, path_prefix, event_info)
|
||||
return populate_global_pkgset(compose, koji_wrapper, path_prefix,
|
||||
event_info)
|
||||
|
||||
|
||||
def _add_module_to_variant(
|
||||
koji_wrapper,
|
||||
variant,
|
||||
build,
|
||||
add_to_variant_modules=False,
|
||||
compose=None,
|
||||
exclude_module_ns=None,
|
||||
koji_wrapper,
|
||||
variant,
|
||||
build,
|
||||
add_to_variant_modules=False,
|
||||
compose=None,
|
||||
exclude_module_ns=None,
|
||||
):
|
||||
"""
|
||||
Adds module defined by Koji build info to variant.
|
||||
|
|
@ -241,13 +248,16 @@ def _add_module_to_variant(
|
|||
if archive["btype"] != "module":
|
||||
# Skip non module archives
|
||||
continue
|
||||
|
||||
filename = archive["filename"]
|
||||
file_path = os.path.join(
|
||||
koji_wrapper.koji_module.pathinfo.topdir,
|
||||
'modules',
|
||||
build['arch'],
|
||||
build['extra']['typeinfo']['module']['content_koji_tag']
|
||||
|
||||
)
|
||||
|
||||
mmds[filename] = file_path
|
||||
|
||||
if len(mmds) <= 1:
|
||||
|
|
@ -266,17 +276,22 @@ def _add_module_to_variant(
|
|||
added = False
|
||||
|
||||
for arch in variant.arches:
|
||||
if _is_filtered_out(compose, variant, arch, info["name"], info["stream"]):
|
||||
compose.log_debug("Module %s is filtered from %s.%s", nsvc, variant, arch)
|
||||
if _is_filtered_out(compose, variant, arch, info["name"],
|
||||
info["stream"]):
|
||||
compose.log_debug("Module %s is filtered from %s.%s", nsvc,
|
||||
variant, arch)
|
||||
continue
|
||||
|
||||
filename = "modulemd.%s.txt" % arch
|
||||
try:
|
||||
mod_stream = read_single_module_stream_from_file(
|
||||
mmds[filename], compose, arch, build
|
||||
|
||||
)
|
||||
if mod_stream:
|
||||
added = True
|
||||
variant.arch_mmds.setdefault(arch, {})[nsvc] = mod_stream
|
||||
|
||||
added = True
|
||||
except KeyError:
|
||||
# There is no modulemd for this arch. This could mean an arch was
|
||||
|
|
@ -295,10 +310,11 @@ def _add_module_to_variant(
|
|||
|
||||
|
||||
def _add_extra_modules_to_variant(
|
||||
compose, koji_wrapper, variant, extra_modules, variant_tags, tag_to_mmd
|
||||
compose, koji_wrapper, variant, extra_modules, variant_tags, tag_to_mmd
|
||||
):
|
||||
for nsvc in extra_modules:
|
||||
msg = "Adding extra module build '%s' to variant '%s'" % (nsvc, variant)
|
||||
msg = "Adding extra module build '%s' to variant '%s'" % (
|
||||
nsvc, variant)
|
||||
compose.log_info(msg)
|
||||
|
||||
nsvc_info = nsvc.split(":")
|
||||
|
|
@ -341,10 +357,11 @@ def _add_extra_modules_to_variant(
|
|||
|
||||
|
||||
def _add_scratch_modules_to_variant(
|
||||
compose, variant, scratch_modules, variant_tags, tag_to_mmd
|
||||
compose, variant, scratch_modules, variant_tags, tag_to_mmd
|
||||
):
|
||||
if compose.compose_type != "test" and scratch_modules:
|
||||
compose.log_warning("Only test composes could include scratch module builds")
|
||||
compose.log_warning(
|
||||
"Only test composes could include scratch module builds")
|
||||
return
|
||||
|
||||
mbs = MBSWrapper(compose.conf["mbs_api_url"])
|
||||
|
|
@ -355,7 +372,8 @@ def _add_scratch_modules_to_variant(
|
|||
try:
|
||||
final_modulemd = mbs.final_modulemd(module_build["id"])
|
||||
except Exception:
|
||||
compose.log_error("Unable to get modulemd for build %s" % module_build)
|
||||
compose.log_error(
|
||||
"Unable to get modulemd for build %s" % module_build)
|
||||
raise
|
||||
tag = module_build["koji_tag"]
|
||||
variant_tags[variant].append(tag)
|
||||
|
|
@ -363,8 +381,7 @@ def _add_scratch_modules_to_variant(
|
|||
for arch in variant.arches:
|
||||
try:
|
||||
mmd = read_single_module_stream_from_string(
|
||||
final_modulemd[arch]
|
||||
)
|
||||
final_modulemd[arch])
|
||||
variant.arch_mmds.setdefault(arch, {})[nsvc] = mmd
|
||||
except KeyError:
|
||||
continue
|
||||
|
|
@ -390,21 +407,24 @@ def _is_filtered_out(compose, variant, arch, module_name, module_stream):
|
|||
if not compose:
|
||||
return False
|
||||
|
||||
for filter in get_arch_variant_data(compose.conf, "filter_modules", arch, variant):
|
||||
for filter in get_arch_variant_data(compose.conf, "filter_modules", arch,
|
||||
variant):
|
||||
if ":" not in filter:
|
||||
name_filter = filter
|
||||
stream_filter = "*"
|
||||
else:
|
||||
name_filter, stream_filter = filter.split(":", 1)
|
||||
|
||||
if fnmatch(module_name, name_filter) and fnmatch(module_stream, stream_filter):
|
||||
if fnmatch(module_name, name_filter) and fnmatch(module_stream,
|
||||
stream_filter):
|
||||
return True
|
||||
|
||||
return False
|
||||
|
||||
|
||||
def _get_modules_from_koji(
|
||||
compose, koji_wrapper, event, variant, variant_tags, tag_to_mmd
|
||||
compose, koji_wrapper, event, variant, variant_tags, tag_to_mmd,
|
||||
exclude_module_ns
|
||||
):
|
||||
"""
|
||||
Loads modules for given `variant` from koji `session`, adds them to
|
||||
|
|
@ -415,15 +435,21 @@ def _get_modules_from_koji(
|
|||
:param Variant variant: Variant with modules to find.
|
||||
:param dict variant_tags: Dict populated by this method. Key is `variant`
|
||||
and value is list of Koji tags to get the RPMs from.
|
||||
:param list exclude_module_ns: Module name:stream which will be excluded.
|
||||
"""
|
||||
|
||||
# Find out all modules in every variant and add their Koji tags
|
||||
# to variant and variant_tags list.
|
||||
for module in variant.get_modules():
|
||||
koji_modules = get_koji_modules(compose, koji_wrapper, event, module["name"])
|
||||
koji_modules = get_koji_modules(compose, koji_wrapper, event,
|
||||
module["name"])
|
||||
for koji_module in koji_modules:
|
||||
nsvc = _add_module_to_variant(
|
||||
koji_wrapper, variant, koji_module, compose=compose
|
||||
koji_wrapper,
|
||||
variant,
|
||||
koji_module,
|
||||
compose=compose,
|
||||
exclude_module_ns=exclude_module_ns,
|
||||
)
|
||||
if not nsvc:
|
||||
continue
|
||||
|
|
@ -462,7 +488,8 @@ def filter_inherited(koji_proxy, event, module_builds, top_tag):
|
|||
does not understand streams, so we have to reimplement it here.
|
||||
"""
|
||||
inheritance = [
|
||||
tag["name"] for tag in koji_proxy.getFullInheritance(top_tag, event=event["id"])
|
||||
tag["name"] for tag in
|
||||
koji_proxy.getFullInheritance(top_tag, event=event["id"])
|
||||
]
|
||||
|
||||
def keyfunc(mb):
|
||||
|
|
@ -487,7 +514,8 @@ def filter_inherited(koji_proxy, event, module_builds, top_tag):
|
|||
return result
|
||||
|
||||
|
||||
def filter_by_whitelist(compose, module_builds, input_modules, expected_modules):
|
||||
def filter_by_whitelist(compose, module_builds, input_modules,
|
||||
expected_modules):
|
||||
"""
|
||||
Exclude modules from the list that do not match any pattern specified in
|
||||
input_modules. Order may not be preserved. The last argument is a set of
|
||||
|
|
@ -511,6 +539,7 @@ def filter_by_whitelist(compose, module_builds, input_modules, expected_modules)
|
|||
info.get("context"),
|
||||
)
|
||||
nvr_patterns.add((pattern, spec["name"]))
|
||||
|
||||
modules_to_keep = []
|
||||
|
||||
for mb in sorted(module_builds, key=lambda i: i['name']):
|
||||
|
|
@ -521,13 +550,13 @@ def filter_by_whitelist(compose, module_builds, input_modules, expected_modules)
|
|||
# and context.
|
||||
for (n, s, v, c), spec in sorted(nvr_patterns):
|
||||
if (
|
||||
# We always have a name and stream...
|
||||
mb["name"] == n
|
||||
and mb["version"] == s
|
||||
# ...but version and context can be missing, in which case we
|
||||
# don't want to check them.
|
||||
and (not v or ver == v)
|
||||
and (not c or ctx == c)
|
||||
# We always have a name and stream...
|
||||
mb["name"] == n
|
||||
and mb["version"] == s
|
||||
# ...but version and context can be missing, in which case we
|
||||
# don't want to check them.
|
||||
and (not v or ver == v)
|
||||
and (not c or ctx == c)
|
||||
):
|
||||
modules_to_keep.append(mb)
|
||||
expected_modules.discard(spec)
|
||||
|
|
@ -575,7 +604,13 @@ def _filter_expected_modules(
|
|||
|
||||
|
||||
def _get_modules_from_koji_tags(
|
||||
compose, koji_wrapper, event_id, variant, variant_tags, tag_to_mmd
|
||||
compose,
|
||||
koji_wrapper,
|
||||
event_id,
|
||||
variant,
|
||||
variant_tags,
|
||||
tag_to_mmd,
|
||||
exclude_module_ns,
|
||||
):
|
||||
"""
|
||||
Loads modules for given `variant` from Koji, adds them to
|
||||
|
|
@ -587,10 +622,12 @@ def _get_modules_from_koji_tags(
|
|||
:param Variant variant: Variant with modules to find.
|
||||
:param dict variant_tags: Dict populated by this method. Key is `variant`
|
||||
and value is list of Koji tags to get the RPMs from.
|
||||
:param list exclude_module_ns: Module name:stream which will be excluded.
|
||||
"""
|
||||
# Compose tags from configuration
|
||||
compose_tags = [
|
||||
{"name": tag} for tag in force_list(compose.conf["pkgset_koji_module_tag"])
|
||||
{"name": tag} for tag in
|
||||
force_list(compose.conf["pkgset_koji_module_tag"])
|
||||
]
|
||||
# Get set of configured module names for this variant. If nothing is
|
||||
# configured, the set is empty.
|
||||
|
|
@ -617,7 +654,8 @@ def _get_modules_from_koji_tags(
|
|||
)
|
||||
|
||||
# Filter out builds inherited from non-top tag
|
||||
module_builds = filter_inherited(koji_proxy, event_id, module_builds, tag)
|
||||
module_builds = filter_inherited(koji_proxy, event_id, module_builds,
|
||||
tag)
|
||||
|
||||
# Apply whitelist of modules if specified.
|
||||
variant_modules = variant.get_modules()
|
||||
|
|
@ -625,6 +663,7 @@ def _get_modules_from_koji_tags(
|
|||
module_builds = filter_by_whitelist(
|
||||
compose, module_builds, variant_modules, expected_modules
|
||||
)
|
||||
|
||||
# Find the latest builds of all modules. This does following:
|
||||
# - Sorts the module_builds descending by Koji NVR (which maps to NSV
|
||||
# for modules). Split release into modular version and context, and
|
||||
|
|
@ -645,14 +684,14 @@ def _get_modules_from_koji_tags(
|
|||
latest_builds = []
|
||||
module_builds = sorted(module_builds, key=_key, reverse=True)
|
||||
for ns, ns_builds in groupby(
|
||||
module_builds, key=lambda x: ":".join([
|
||||
module_builds, key=lambda x: ":".join([
|
||||
x["name"],
|
||||
x["version"],
|
||||
x['arch'],
|
||||
])
|
||||
):
|
||||
for nsv, nsv_builds in groupby(
|
||||
ns_builds, key=lambda x: x["release"].split(".")[0]
|
||||
ns_builds, key=lambda x: x["release"].split(".")[0]
|
||||
):
|
||||
latest_builds += list(nsv_builds)
|
||||
break
|
||||
|
|
@ -662,6 +701,18 @@ def _get_modules_from_koji_tags(
|
|||
for build in latest_builds:
|
||||
# Get the Build from Koji to get modulemd and module_tag.
|
||||
build = koji_proxy.getBuild(build["build_id"])
|
||||
|
||||
nsvc = _add_module_to_variant(
|
||||
koji_wrapper,
|
||||
variant,
|
||||
build,
|
||||
True,
|
||||
compose=compose,
|
||||
exclude_module_ns=exclude_module_ns,
|
||||
)
|
||||
if not nsvc:
|
||||
continue
|
||||
|
||||
module_tag = (
|
||||
build.get("extra", {})
|
||||
.get("typeinfo", {})
|
||||
|
|
@ -671,12 +722,6 @@ def _get_modules_from_koji_tags(
|
|||
|
||||
variant_tags[variant].append(module_tag)
|
||||
|
||||
nsvc = _add_module_to_variant(
|
||||
koji_wrapper, variant, build, True, compose=compose
|
||||
)
|
||||
if not nsvc:
|
||||
continue
|
||||
|
||||
tag_to_mmd.setdefault(module_tag, {})
|
||||
for arch in variant.arch_mmds:
|
||||
try:
|
||||
|
|
@ -736,7 +781,7 @@ def populate_global_pkgset(compose, koji_wrapper, path_prefix, event):
|
|||
for group in groups:
|
||||
packages_to_gather += comps.get_packages(group)
|
||||
if compose.conf["gather_method"] == "nodeps" and not compose.conf.get(
|
||||
"buildinstall_method"
|
||||
"buildinstall_method"
|
||||
):
|
||||
populate_only_packages_to_gather = True
|
||||
else:
|
||||
|
|
@ -767,26 +812,48 @@ def populate_global_pkgset(compose, koji_wrapper, path_prefix, event):
|
|||
"modules."
|
||||
)
|
||||
|
||||
extra_modules = get_variant_data(
|
||||
compose.conf, "pkgset_koji_module_builds", variant
|
||||
)
|
||||
|
||||
# When adding extra modules, other modules of the same name:stream available
|
||||
# in brew tag should be excluded.
|
||||
exclude_module_ns = []
|
||||
if extra_modules:
|
||||
exclude_module_ns = [
|
||||
":".join(nsvc.split(":")[:2]) for nsvc in extra_modules
|
||||
]
|
||||
|
||||
if modular_koji_tags or (
|
||||
compose.conf["pkgset_koji_module_tag"] and variant.modules
|
||||
compose.conf["pkgset_koji_module_tag"] and variant.modules
|
||||
):
|
||||
# List modules tagged in particular tags.
|
||||
_get_modules_from_koji_tags(
|
||||
compose, koji_wrapper, event, variant, variant_tags, tag_to_mmd
|
||||
compose,
|
||||
koji_wrapper,
|
||||
event,
|
||||
variant,
|
||||
variant_tags,
|
||||
tag_to_mmd,
|
||||
exclude_module_ns,
|
||||
)
|
||||
elif variant.modules:
|
||||
# Search each module in Koji separately. Tagging does not come into
|
||||
# play here.
|
||||
_get_modules_from_koji(
|
||||
compose, koji_wrapper, event, variant, variant_tags, tag_to_mmd
|
||||
compose,
|
||||
koji_wrapper,
|
||||
event,
|
||||
variant,
|
||||
variant_tags,
|
||||
tag_to_mmd,
|
||||
exclude_module_ns,
|
||||
)
|
||||
|
||||
extra_modules = get_variant_data(
|
||||
compose.conf, "pkgset_koji_module_builds", variant
|
||||
)
|
||||
if extra_modules:
|
||||
_add_extra_modules_to_variant(
|
||||
compose, koji_wrapper, variant, extra_modules, variant_tags, tag_to_mmd
|
||||
compose, koji_wrapper, variant, extra_modules, variant_tags,
|
||||
tag_to_mmd
|
||||
)
|
||||
|
||||
variant_scratch_modules = get_variant_data(
|
||||
|
|
@ -794,7 +861,8 @@ def populate_global_pkgset(compose, koji_wrapper, path_prefix, event):
|
|||
)
|
||||
if variant_scratch_modules:
|
||||
_add_scratch_modules_to_variant(
|
||||
compose, variant, variant_scratch_modules, variant_tags, tag_to_mmd
|
||||
compose, variant, variant_scratch_modules, variant_tags,
|
||||
tag_to_mmd
|
||||
)
|
||||
|
||||
# Ensure that every tag added to `variant_tags` is added also to
|
||||
|
|
@ -819,8 +887,10 @@ def populate_global_pkgset(compose, koji_wrapper, path_prefix, event):
|
|||
for compose_tag in compose_tags:
|
||||
compose.log_info("Loading package set for tag %s", compose_tag)
|
||||
if compose_tag in pkgset_koji_tags:
|
||||
extra_builds = force_list(compose.conf.get("pkgset_koji_builds", []))
|
||||
extra_tasks = force_list(compose.conf.get("pkgset_koji_scratch_tasks", []))
|
||||
extra_builds = force_list(
|
||||
compose.conf.get("pkgset_koji_builds", []))
|
||||
extra_tasks = force_list(
|
||||
compose.conf.get("pkgset_koji_scratch_tasks", []))
|
||||
else:
|
||||
extra_builds = []
|
||||
extra_tasks = []
|
||||
|
|
@ -837,6 +907,7 @@ def populate_global_pkgset(compose, koji_wrapper, path_prefix, event):
|
|||
cache_region=compose.cache_region,
|
||||
extra_builds=extra_builds,
|
||||
extra_tasks=extra_tasks,
|
||||
|
||||
)
|
||||
|
||||
# Check if we have cache for this tag from previous compose. If so, use
|
||||
|
|
@ -847,6 +918,7 @@ def populate_global_pkgset(compose, koji_wrapper, path_prefix, event):
|
|||
if old_cache_path:
|
||||
pkgset.set_old_file_cache(
|
||||
pungi.phases.pkgset.pkgsets.KojiPackageSet.load_old_file_cache(
|
||||
|
||||
old_cache_path
|
||||
)
|
||||
)
|
||||
|
|
@ -926,7 +998,8 @@ def populate_global_pkgset(compose, koji_wrapper, path_prefix, event):
|
|||
|
||||
|
||||
def get_koji_event_info(compose, koji_wrapper):
|
||||
event_file = os.path.join(compose.paths.work.topdir(arch="global"), "koji-event")
|
||||
event_file = os.path.join(compose.paths.work.topdir(arch="global"),
|
||||
"koji-event")
|
||||
|
||||
compose.log_info("Getting koji event")
|
||||
result = get_koji_event_raw(koji_wrapper, compose.koji_event, event_file)
|
||||
|
|
|
|||
Loading…
Reference in New Issue