almalinux/pungi
14
1
Fork 0
Code Issues Pull Requests Releases Activity

ALBS-1030: Generate Devel section in packages.json #14

Merged
soksanichenko merged 5 commits from ALBS-1030 into al_master 2023-03-22 10:06:59 +00:00
Conversation 0 Commits 5 Files Changed 3 +153 -64
3 changed files with 153 additions and 64 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
pungi/phases/pkgset/pkgsets.py
View File

@ -889,6 +889,8 @@ class KojiMockPackageSet(KojiPackageSet):
sigkey.lower() for sigkey in self.sigkey_ordering
if sigkey is not None
]
if not sigkeys:
return True
with open(rpm_path, 'rb') as fd:
header = ts.hdrFromFdno(fd)
signature = header[rpm.RPMTAG_SIGGPG] or header[rpm.RPMTAG_SIGPGP]

189
pungi/phases/pkgset/sources/source_kojimock.py
View File

@ -35,10 +35,12 @@ import pungi.wrappers.kojiwrapper
from pungi.wrappers.comps import CompsWrapper
from pungi.wrappers.mbs import MBSWrapper
import pungi.phases.pkgset.pkgsets
from pungi.util import (
retry,
get_arch_variant_data,
get_variant_data,
jhernandez commented 2023-03-20 12:01:51 +00:00
Review
Copy Link

extra empty line

extra empty line
read_single_module_stream_from_string,
read_single_module_stream_from_file,
)
@ -101,12 +103,12 @@ def variant_dict_from_str(compose, module_str):
release_regex = re.compile(r"^(\d){14}$")
section_start = module_str.rfind("-")
module_str_first_part = module_str[section_start + 1 :]
module_str_first_part = module_str[section_start + 1:]
if release_regex.match(module_str_first_part):
module_info["version"] = module_str_first_part
module_str = module_str[:section_start]
section_start = module_str.rfind("-")
module_info["stream"] = module_str[section_start + 1 :]
module_info["stream"] = module_str[section_start + 1:]
else:
module_info["stream"] = module_str_first_part
module_info["name"] = module_str[:section_start]
@ -160,14 +162,16 @@ def get_koji_modules(compose, koji_wrapper, event, module_info_str):
# Store module versioning information into the dict, but make sure
# not to overwrite any existing keys.
md["module_stream"] = md["extra"]["typeinfo"]["module"]["stream"]
md["module_version"] = int(md["extra"]["typeinfo"]["module"]["version"])
md["module_version"] = int(
md["extra"]["typeinfo"]["module"]["version"])
md["module_context"] = md["extra"]["typeinfo"]["module"]["context"]
except KeyError:
continue
if md["state"] == pungi.wrappers.kojiwrapper.KOJI_BUILD_DELETED:
compose.log_debug(
"Module build %s has been deleted, ignoring it." % build["name"]
"Module build %s has been deleted, ignoring it." % build[
"name"]
)
continue
@ -189,7 +193,8 @@ def get_koji_modules(compose, koji_wrapper, event, module_info_str):
)
latest_version = sorted_modules[0]["module_version"]
modules = [
module for module in modules if latest_version == module["module_version"]
module for module in modules
soksanichenko marked this conversation as resolved Outdated
jhernandez commented 2023-03-20 11:38:17 +00:00
Outdated
Review
Copy Link

Just a personal taste for readability, I'd move if into the next line

Just a personal taste for readability, I'd move `if` into the next line
if latest_version == module["module_version"]
]
return modules
@ -205,7 +210,8 @@ class PkgsetSourceKojiMock(pungi.phases.pkgset.source.PkgsetSourceBase):
get_all_arches(compose),
)
# path prefix must contain trailing '/'
path_prefix = self.koji_wrapper.koji_module.config.topdir.rstrip("/") + "/"
path_prefix = self.koji_wrapper.koji_module.config.topdir.rstrip(
"/") + "/"
package_sets = get_pkgset_from_koji(
self.compose, self.koji_wrapper, path_prefix
)
@ -214,16 +220,17 @@ class PkgsetSourceKojiMock(pungi.phases.pkgset.source.PkgsetSourceBase):
def get_pkgset_from_koji(compose, koji_wrapper, path_prefix):
event_info = get_koji_event_info(compose, koji_wrapper)
return populate_global_pkgset(compose, koji_wrapper, path_prefix, event_info)
return populate_global_pkgset(compose, koji_wrapper, path_prefix,
event_info)
def _add_module_to_variant(
koji_wrapper,
variant,
build,
add_to_variant_modules=False,
compose=None,
exclude_module_ns=None,
koji_wrapper,
variant,
build,
add_to_variant_modules=False,
compose=None,
exclude_module_ns=None,
):
"""
Adds module defined by Koji build info to variant.
@ -241,13 +248,16 @@ def _add_module_to_variant(
if archive["btype"] != "module":
# Skip non module archives
continue
filename = archive["filename"]
file_path = os.path.join(
koji_wrapper.koji_module.pathinfo.topdir,
'modules',
build['arch'],
build['extra']['typeinfo']['module']['content_koji_tag']
)
mmds[filename] = file_path
if len(mmds) <= 1:
@ -266,17 +276,22 @@ def _add_module_to_variant(
added = False
for arch in variant.arches:
if _is_filtered_out(compose, variant, arch, info["name"], info["stream"]):
compose.log_debug("Module %s is filtered from %s.%s", nsvc, variant, arch)
if _is_filtered_out(compose, variant, arch, info["name"],
info["stream"]):
compose.log_debug("Module %s is filtered from %s.%s", nsvc,
variant, arch)
continue
filename = "modulemd.%s.txt" % arch
try:
mod_stream = read_single_module_stream_from_file(
mmds[filename], compose, arch, build
)
if mod_stream:
added = True
variant.arch_mmds.setdefault(arch, {})[nsvc] = mod_stream
added = True
except KeyError:
# There is no modulemd for this arch. This could mean an arch was
@ -295,10 +310,11 @@ def _add_module_to_variant(
def _add_extra_modules_to_variant(
compose, koji_wrapper, variant, extra_modules, variant_tags, tag_to_mmd
compose, koji_wrapper, variant, extra_modules, variant_tags, tag_to_mmd
):
for nsvc in extra_modules:
msg = "Adding extra module build '%s' to variant '%s'" % (nsvc, variant)
msg = "Adding extra module build '%s' to variant '%s'" % (
nsvc, variant)
compose.log_info(msg)
nsvc_info = nsvc.split(":")
@ -341,10 +357,11 @@ def _add_extra_modules_to_variant(
def _add_scratch_modules_to_variant(
compose, variant, scratch_modules, variant_tags, tag_to_mmd
compose, variant, scratch_modules, variant_tags, tag_to_mmd
):
if compose.compose_type != "test" and scratch_modules:
compose.log_warning("Only test composes could include scratch module builds")
compose.log_warning(
"Only test composes could include scratch module builds")
return
mbs = MBSWrapper(compose.conf["mbs_api_url"])
@ -355,7 +372,8 @@ def _add_scratch_modules_to_variant(
try:
final_modulemd = mbs.final_modulemd(module_build["id"])
except Exception:
compose.log_error("Unable to get modulemd for build %s" % module_build)
compose.log_error(
"Unable to get modulemd for build %s" % module_build)
raise
tag = module_build["koji_tag"]
variant_tags[variant].append(tag)
@ -363,8 +381,7 @@ def _add_scratch_modules_to_variant(
for arch in variant.arches:
try:
mmd = read_single_module_stream_from_string(
final_modulemd[arch]
)
final_modulemd[arch])
variant.arch_mmds.setdefault(arch, {})[nsvc] = mmd
except KeyError:
continue
@ -390,21 +407,24 @@ def _is_filtered_out(compose, variant, arch, module_name, module_stream):
if not compose:
return False
for filter in get_arch_variant_data(compose.conf, "filter_modules", arch, variant):
for filter in get_arch_variant_data(compose.conf, "filter_modules", arch,
variant):
if ":" not in filter:
name_filter = filter
stream_filter = "*"
else:
name_filter, stream_filter = filter.split(":", 1)
if fnmatch(module_name, name_filter) and fnmatch(module_stream, stream_filter):
if fnmatch(module_name, name_filter) and fnmatch(module_stream,
stream_filter):
return True
return False
def _get_modules_from_koji(
compose, koji_wrapper, event, variant, variant_tags, tag_to_mmd
compose, koji_wrapper, event, variant, variant_tags, tag_to_mmd,
exclude_module_ns
):
"""
Loads modules for given `variant` from koji `session`, adds them to
@ -415,15 +435,21 @@ def _get_modules_from_koji(
:param Variant variant: Variant with modules to find.
:param dict variant_tags: Dict populated by this method. Key is `variant`
and value is list of Koji tags to get the RPMs from.
:param list exclude_module_ns: Module name:stream which will be excluded.
"""
# Find out all modules in every variant and add their Koji tags
# to variant and variant_tags list.
for module in variant.get_modules():
koji_modules = get_koji_modules(compose, koji_wrapper, event, module["name"])
koji_modules = get_koji_modules(compose, koji_wrapper, event,
module["name"])
for koji_module in koji_modules:
nsvc = _add_module_to_variant(
koji_wrapper, variant, koji_module, compose=compose
koji_wrapper,
variant,
koji_module,
compose=compose,
exclude_module_ns=exclude_module_ns,
)
if not nsvc:
continue
@ -462,7 +488,8 @@ def filter_inherited(koji_proxy, event, module_builds, top_tag):
does not understand streams, so we have to reimplement it here.
"""
inheritance = [
tag["name"] for tag in koji_proxy.getFullInheritance(top_tag, event=event["id"])
tag["name"] for tag in
koji_proxy.getFullInheritance(top_tag, event=event["id"])
]
def keyfunc(mb):
@ -487,7 +514,8 @@ def filter_inherited(koji_proxy, event, module_builds, top_tag):
return result
def filter_by_whitelist(compose, module_builds, input_modules, expected_modules):
def filter_by_whitelist(compose, module_builds, input_modules,
expected_modules):
"""
Exclude modules from the list that do not match any pattern specified in
input_modules. Order may not be preserved. The last argument is a set of
@ -511,6 +539,7 @@ def filter_by_whitelist(compose, module_builds, input_modules, expected_modules)
info.get("context"),
)
nvr_patterns.add((pattern, spec["name"]))
modules_to_keep = []
for mb in sorted(module_builds, key=lambda i: i['name']):
@ -521,13 +550,13 @@ def filter_by_whitelist(compose, module_builds, input_modules, expected_modules)
# and context.
for (n, s, v, c), spec in sorted(nvr_patterns):
if (
# We always have a name and stream...
mb["name"] == n
and mb["version"] == s
# ...but version and context can be missing, in which case we
# don't want to check them.
and (not v or ver == v)
and (not c or ctx == c)
# We always have a name and stream...
mb["name"] == n
and mb["version"] == s
# ...but version and context can be missing, in which case we
# don't want to check them.
and (not v or ver == v)
and (not c or ctx == c)
):
modules_to_keep.append(mb)
expected_modules.discard(spec)
@ -575,7 +604,13 @@ def _filter_expected_modules(
def _get_modules_from_koji_tags(
compose, koji_wrapper, event_id, variant, variant_tags, tag_to_mmd
compose,
koji_wrapper,
event_id,
variant,
variant_tags,
tag_to_mmd,
exclude_module_ns,
):
"""
Loads modules for given `variant` from Koji, adds them to
@ -587,10 +622,12 @@ def _get_modules_from_koji_tags(
:param Variant variant: Variant with modules to find.
:param dict variant_tags: Dict populated by this method. Key is `variant`
and value is list of Koji tags to get the RPMs from.
:param list exclude_module_ns: Module name:stream which will be excluded.
"""
# Compose tags from configuration
compose_tags = [
{"name": tag} for tag in force_list(compose.conf["pkgset_koji_module_tag"])
{"name": tag} for tag in
force_list(compose.conf["pkgset_koji_module_tag"])
]
# Get set of configured module names for this variant. If nothing is
# configured, the set is empty.
@ -617,7 +654,8 @@ def _get_modules_from_koji_tags(
)
# Filter out builds inherited from non-top tag
module_builds = filter_inherited(koji_proxy, event_id, module_builds, tag)
module_builds = filter_inherited(koji_proxy, event_id, module_builds,
tag)
# Apply whitelist of modules if specified.
variant_modules = variant.get_modules()
@ -625,6 +663,7 @@ def _get_modules_from_koji_tags(
module_builds = filter_by_whitelist(
compose, module_builds, variant_modules, expected_modules
)
# Find the latest builds of all modules. This does following:
# - Sorts the module_builds descending by Koji NVR (which maps to NSV
# for modules). Split release into modular version and context, and
@ -645,14 +684,14 @@ def _get_modules_from_koji_tags(
latest_builds = []
module_builds = sorted(module_builds, key=_key, reverse=True)
for ns, ns_builds in groupby(
module_builds, key=lambda x: ":".join([
module_builds, key=lambda x: ":".join([
x["name"],
x["version"],
x['arch'],
])
):
for nsv, nsv_builds in groupby(
ns_builds, key=lambda x: x["release"].split(".")[0]
ns_builds, key=lambda x: x["release"].split(".")[0]
):
latest_builds += list(nsv_builds)
break
@ -662,6 +701,18 @@ def _get_modules_from_koji_tags(
for build in latest_builds:
# Get the Build from Koji to get modulemd and module_tag.
build = koji_proxy.getBuild(build["build_id"])
nsvc = _add_module_to_variant(
koji_wrapper,
variant,
build,
True,
compose=compose,
exclude_module_ns=exclude_module_ns,
)
if not nsvc:
continue
module_tag = (
build.get("extra", {})
.get("typeinfo", {})
@ -671,12 +722,6 @@ def _get_modules_from_koji_tags(
variant_tags[variant].append(module_tag)
nsvc = _add_module_to_variant(
koji_wrapper, variant, build, True, compose=compose
)
if not nsvc:
continue
tag_to_mmd.setdefault(module_tag, {})
for arch in variant.arch_mmds:
try:
@ -736,7 +781,7 @@ def populate_global_pkgset(compose, koji_wrapper, path_prefix, event):
for group in groups:
packages_to_gather += comps.get_packages(group)
if compose.conf["gather_method"] == "nodeps" and not compose.conf.get(
"buildinstall_method"
"buildinstall_method"
):
populate_only_packages_to_gather = True
else:
@ -767,26 +812,48 @@ def populate_global_pkgset(compose, koji_wrapper, path_prefix, event):
"modules."
)
extra_modules = get_variant_data(
compose.conf, "pkgset_koji_module_builds", variant
)
# When adding extra modules, other modules of the same name:stream available
# in brew tag should be excluded.
exclude_module_ns = []
if extra_modules:
exclude_module_ns = [
":".join(nsvc.split(":")[:2]) for nsvc in extra_modules
]
if modular_koji_tags or (
compose.conf["pkgset_koji_module_tag"] and variant.modules
compose.conf["pkgset_koji_module_tag"] and variant.modules
):
# List modules tagged in particular tags.
_get_modules_from_koji_tags(
compose, koji_wrapper, event, variant, variant_tags, tag_to_mmd
compose,
koji_wrapper,
event,
variant,
variant_tags,
tag_to_mmd,
exclude_module_ns,
)
elif variant.modules:
# Search each module in Koji separately. Tagging does not come into
# play here.
_get_modules_from_koji(
compose, koji_wrapper, event, variant, variant_tags, tag_to_mmd
compose,
koji_wrapper,
event,
variant,
variant_tags,
tag_to_mmd,
exclude_module_ns,
)
extra_modules = get_variant_data(
compose.conf, "pkgset_koji_module_builds", variant
)
if extra_modules:
_add_extra_modules_to_variant(
compose, koji_wrapper, variant, extra_modules, variant_tags, tag_to_mmd
compose, koji_wrapper, variant, extra_modules, variant_tags,
tag_to_mmd
)
variant_scratch_modules = get_variant_data(
@ -794,7 +861,8 @@ def populate_global_pkgset(compose, koji_wrapper, path_prefix, event):
)
if variant_scratch_modules:
_add_scratch_modules_to_variant(
compose, variant, variant_scratch_modules, variant_tags, tag_to_mmd
compose, variant, variant_scratch_modules, variant_tags,
tag_to_mmd
)
# Ensure that every tag added to `variant_tags` is added also to
@ -819,8 +887,10 @@ def populate_global_pkgset(compose, koji_wrapper, path_prefix, event):
for compose_tag in compose_tags:
compose.log_info("Loading package set for tag %s", compose_tag)
if compose_tag in pkgset_koji_tags:
extra_builds = force_list(compose.conf.get("pkgset_koji_builds", []))
extra_tasks = force_list(compose.conf.get("pkgset_koji_scratch_tasks", []))
extra_builds = force_list(
compose.conf.get("pkgset_koji_builds", []))
extra_tasks = force_list(
compose.conf.get("pkgset_koji_scratch_tasks", []))
else:
extra_builds = []
extra_tasks = []
@ -926,7 +996,8 @@ def populate_global_pkgset(compose, koji_wrapper, path_prefix, event):
def get_koji_event_info(compose, koji_wrapper):
event_file = os.path.join(compose.paths.work.topdir(arch="global"), "koji-event")
event_file = os.path.join(compose.paths.work.topdir(arch="global"),
"koji-event")
compose.log_info("Getting koji event")
result = get_koji_event_raw(koji_wrapper, compose.koji_event, event_file)

26
pungi/scripts/create_packages_json.py
View File

@ -14,7 +14,7 @@ import os
import re
import tempfile
from collections import defaultdict
from typing import AnyStr, Dict, List, Optional, Any, Iterator
from typing import AnyStr, Dict, List, Any, Iterator
import binascii
import createrepo_c as cr
@ -65,7 +65,7 @@ class RepoInfo:
# Only layout of specific package (which don't exist
# in a reference repository) will be taken as example
is_reference: bool = False
strict_arch: bool = False
repo_type: str = 'present'
class PackagesGenerator:
@ -262,7 +262,11 @@ class PackagesGenerator:
)
)
all_packages = defaultdict(lambda: {'variants': list()})
for repo_info in self.repos:
for repo_info in sorted(
self.repos,
key=lambda i: i.repo_type,
reverse=True,
):
repomd_records = self._get_repomd_records(
repo_info=repo_info,
)
@ -298,6 +302,8 @@ class PackagesGenerator:
all_packages[package_key]['arch'] = package_arch
all_packages[package_key]['package'] = package
all_packages[package_key]['type'] = repo_info.is_reference
elif repo_info.repo_type == 'absent' and (repo_info.name, repo_info.arch) in all_packages[package_key]['variants']:
all_packages[package_key]['variants'].remove((repo_info.name, repo_info.arch))
# replace an older package if it's not reference or
# a newer package is from reference repo
elif (not all_packages[package_key]['type'] or
@ -404,6 +410,14 @@ def create_parser():
choices=['yes', 'no'],
required=True,
)
parser.add_argument(
'--repo-type',
action='append',
type=str,
help='Packages from repository will be removed or added to variant',
choices=['present', 'absent'],
required=True,
)
parser.add_argument(
'--excluded-packages',
nargs='+',
@ -436,13 +450,14 @@ def cli_main():
args = create_parser().parse_args()
repos = []
for repo_path, repo_folder, repo_name, \
repo_arch, is_remote, is_reference in zip(
repo_arch, is_remote, is_reference, repo_type in zip(
args.repo_path,
args.repo_folder,
args.repo_name,
args.repo_arch,
args.is_remote,
args.is_reference,
args.repo_type,
):
repos.append(RepoInfo(
path=repo_path,
@ -450,7 +465,8 @@ def cli_main():
name=repo_name,
arch=repo_arch,
is_remote=True if is_remote == 'yes' else False,
is_reference=True if is_reference == 'yes' else False
is_reference=True if is_reference == 'yes' else False,
repo_type=repo_type,
))
pg = PackagesGenerator(
repos=repos,