almalinux/pungi
14
1
Fork 0
Code Issues Pull Requests Releases Activity

ALBS-1030: Generate Devel section in packages.json #14

Merged
soksanichenko merged 5 commits from ALBS-1030 into al_master 2023-03-22 10:06:59 +00:00
Conversation 0 Commits 5 Files Changed 3 +153 -64
3 changed files with 153 additions and 64 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
pungi/phases/pkgset/pkgsets.py
View File

@ -889,6 +889,8 @@ class KojiMockPackageSet(KojiPackageSet):
sigkey.lower() for sigkey in self.sigkey_ordering sigkey.lower() for sigkey in self.sigkey_ordering
if sigkey is not None if sigkey is not None
] ]
if not sigkeys:
return True
with open(rpm_path, 'rb') as fd: with open(rpm_path, 'rb') as fd:
header = ts.hdrFromFdno(fd) header = ts.hdrFromFdno(fd)
signature = header[rpm.RPMTAG_SIGGPG] or header[rpm.RPMTAG_SIGPGP] signature = header[rpm.RPMTAG_SIGGPG] or header[rpm.RPMTAG_SIGPGP]

151
pungi/phases/pkgset/sources/source_kojimock.py
View File

@ -35,10 +35,12 @@ import pungi.wrappers.kojiwrapper
from pungi.wrappers.comps import CompsWrapper from pungi.wrappers.comps import CompsWrapper
from pungi.wrappers.mbs import MBSWrapper from pungi.wrappers.mbs import MBSWrapper
import pungi.phases.pkgset.pkgsets import pungi.phases.pkgset.pkgsets
from pungi.util import ( from pungi.util import (
retry, retry,
get_arch_variant_data, get_arch_variant_data,
get_variant_data, get_variant_data,
jhernandez commented 2023-03-20 12:01:51 +00:00
Review
Copy Link

extra empty line

extra empty line
read_single_module_stream_from_string, read_single_module_stream_from_string,
read_single_module_stream_from_file, read_single_module_stream_from_file,
) )
@ -101,12 +103,12 @@ def variant_dict_from_str(compose, module_str):
release_regex = re.compile(r"^(\d){14}$") release_regex = re.compile(r"^(\d){14}$")
section_start = module_str.rfind("-") section_start = module_str.rfind("-")
module_str_first_part = module_str[section_start + 1 :] module_str_first_part = module_str[section_start + 1:]
if release_regex.match(module_str_first_part): if release_regex.match(module_str_first_part):
module_info["version"] = module_str_first_part module_info["version"] = module_str_first_part
module_str = module_str[:section_start] module_str = module_str[:section_start]
section_start = module_str.rfind("-") section_start = module_str.rfind("-")
module_info["stream"] = module_str[section_start + 1 :] module_info["stream"] = module_str[section_start + 1:]
else: else:
module_info["stream"] = module_str_first_part module_info["stream"] = module_str_first_part
module_info["name"] = module_str[:section_start] module_info["name"] = module_str[:section_start]
@ -160,14 +162,16 @@ def get_koji_modules(compose, koji_wrapper, event, module_info_str):
# Store module versioning information into the dict, but make sure # Store module versioning information into the dict, but make sure
# not to overwrite any existing keys. # not to overwrite any existing keys.
md["module_stream"] = md["extra"]["typeinfo"]["module"]["stream"] md["module_stream"] = md["extra"]["typeinfo"]["module"]["stream"]
md["module_version"] = int(md["extra"]["typeinfo"]["module"]["version"]) md["module_version"] = int(
md["extra"]["typeinfo"]["module"]["version"])
md["module_context"] = md["extra"]["typeinfo"]["module"]["context"] md["module_context"] = md["extra"]["typeinfo"]["module"]["context"]
except KeyError: except KeyError:
continue continue
if md["state"] == pungi.wrappers.kojiwrapper.KOJI_BUILD_DELETED: if md["state"] == pungi.wrappers.kojiwrapper.KOJI_BUILD_DELETED:
compose.log_debug( compose.log_debug(
"Module build %s has been deleted, ignoring it." % build["name"] "Module build %s has been deleted, ignoring it." % build[
"name"]
) )
continue continue
@ -189,7 +193,8 @@ def get_koji_modules(compose, koji_wrapper, event, module_info_str):
) )
latest_version = sorted_modules[0]["module_version"] latest_version = sorted_modules[0]["module_version"]
modules = [ modules = [
module for module in modules if latest_version == module["module_version"] module for module in modules
if latest_version == module["module_version"]
] ]
return modules return modules
@ -205,7 +210,8 @@ class PkgsetSourceKojiMock(pungi.phases.pkgset.source.PkgsetSourceBase):
get_all_arches(compose), get_all_arches(compose),
) )
# path prefix must contain trailing '/' # path prefix must contain trailing '/'
path_prefix = self.koji_wrapper.koji_module.config.topdir.rstrip("/") + "/" path_prefix = self.koji_wrapper.koji_module.config.topdir.rstrip(
"/") + "/"
package_sets = get_pkgset_from_koji( package_sets = get_pkgset_from_koji(
self.compose, self.koji_wrapper, path_prefix self.compose, self.koji_wrapper, path_prefix
) )
@ -214,7 +220,8 @@ class PkgsetSourceKojiMock(pungi.phases.pkgset.source.PkgsetSourceBase):
def get_pkgset_from_koji(compose, koji_wrapper, path_prefix): def get_pkgset_from_koji(compose, koji_wrapper, path_prefix):
event_info = get_koji_event_info(compose, koji_wrapper) event_info = get_koji_event_info(compose, koji_wrapper)
return populate_global_pkgset(compose, koji_wrapper, path_prefix, event_info) return populate_global_pkgset(compose, koji_wrapper, path_prefix,
event_info)
def _add_module_to_variant( def _add_module_to_variant(
@ -241,13 +248,16 @@ def _add_module_to_variant(
if archive["btype"] != "module": if archive["btype"] != "module":
# Skip non module archives # Skip non module archives
continue continue
filename = archive["filename"] filename = archive["filename"]
file_path = os.path.join( file_path = os.path.join(
koji_wrapper.koji_module.pathinfo.topdir, koji_wrapper.koji_module.pathinfo.topdir,
'modules', 'modules',
build['arch'], build['arch'],
build['extra']['typeinfo']['module']['content_koji_tag'] build['extra']['typeinfo']['module']['content_koji_tag']
) )
mmds[filename] = file_path mmds[filename] = file_path
if len(mmds) <= 1: if len(mmds) <= 1:
@ -266,17 +276,22 @@ def _add_module_to_variant(
added = False added = False
for arch in variant.arches: for arch in variant.arches:
if _is_filtered_out(compose, variant, arch, info["name"], info["stream"]): if _is_filtered_out(compose, variant, arch, info["name"],
compose.log_debug("Module %s is filtered from %s.%s", nsvc, variant, arch) info["stream"]):
compose.log_debug("Module %s is filtered from %s.%s", nsvc,
variant, arch)
continue continue
filename = "modulemd.%s.txt" % arch filename = "modulemd.%s.txt" % arch
try: try:
mod_stream = read_single_module_stream_from_file( mod_stream = read_single_module_stream_from_file(
mmds[filename], compose, arch, build mmds[filename], compose, arch, build
) )
if mod_stream: if mod_stream:
added = True added = True
variant.arch_mmds.setdefault(arch, {})[nsvc] = mod_stream variant.arch_mmds.setdefault(arch, {})[nsvc] = mod_stream
added = True added = True
except KeyError: except KeyError:
# There is no modulemd for this arch. This could mean an arch was # There is no modulemd for this arch. This could mean an arch was
@ -298,7 +313,8 @@ def _add_extra_modules_to_variant(
compose, koji_wrapper, variant, extra_modules, variant_tags, tag_to_mmd compose, koji_wrapper, variant, extra_modules, variant_tags, tag_to_mmd
): ):
for nsvc in extra_modules: for nsvc in extra_modules:
msg = "Adding extra module build '%s' to variant '%s'" % (nsvc, variant) msg = "Adding extra module build '%s' to variant '%s'" % (
nsvc, variant)
compose.log_info(msg) compose.log_info(msg)
nsvc_info = nsvc.split(":") nsvc_info = nsvc.split(":")
@ -344,7 +360,8 @@ def _add_scratch_modules_to_variant(
compose, variant, scratch_modules, variant_tags, tag_to_mmd compose, variant, scratch_modules, variant_tags, tag_to_mmd
): ):
if compose.compose_type != "test" and scratch_modules: if compose.compose_type != "test" and scratch_modules:
compose.log_warning("Only test composes could include scratch module builds") compose.log_warning(
"Only test composes could include scratch module builds")
return return
mbs = MBSWrapper(compose.conf["mbs_api_url"]) mbs = MBSWrapper(compose.conf["mbs_api_url"])
@ -355,7 +372,8 @@ def _add_scratch_modules_to_variant(
try: try:
final_modulemd = mbs.final_modulemd(module_build["id"]) final_modulemd = mbs.final_modulemd(module_build["id"])
except Exception: except Exception:
compose.log_error("Unable to get modulemd for build %s" % module_build) compose.log_error(
"Unable to get modulemd for build %s" % module_build)
raise raise
tag = module_build["koji_tag"] tag = module_build["koji_tag"]
variant_tags[variant].append(tag) variant_tags[variant].append(tag)
@ -363,8 +381,7 @@ def _add_scratch_modules_to_variant(
for arch in variant.arches: for arch in variant.arches:
try: try:
mmd = read_single_module_stream_from_string( mmd = read_single_module_stream_from_string(
final_modulemd[arch] final_modulemd[arch])
)
variant.arch_mmds.setdefault(arch, {})[nsvc] = mmd variant.arch_mmds.setdefault(arch, {})[nsvc] = mmd
except KeyError: except KeyError:
continue continue
@ -390,21 +407,24 @@ def _is_filtered_out(compose, variant, arch, module_name, module_stream):
if not compose: if not compose:
return False return False
for filter in get_arch_variant_data(compose.conf, "filter_modules", arch, variant): for filter in get_arch_variant_data(compose.conf, "filter_modules", arch,
variant):
if ":" not in filter: if ":" not in filter:
name_filter = filter name_filter = filter
stream_filter = "*" stream_filter = "*"
else: else:
name_filter, stream_filter = filter.split(":", 1) name_filter, stream_filter = filter.split(":", 1)
if fnmatch(module_name, name_filter) and fnmatch(module_stream, stream_filter): if fnmatch(module_name, name_filter) and fnmatch(module_stream,
stream_filter):
return True return True
return False return False
def _get_modules_from_koji( def _get_modules_from_koji(
compose, koji_wrapper, event, variant, variant_tags, tag_to_mmd compose, koji_wrapper, event, variant, variant_tags, tag_to_mmd,
exclude_module_ns
): ):
""" """
Loads modules for given `variant` from koji `session`, adds them to Loads modules for given `variant` from koji `session`, adds them to
@ -415,15 +435,21 @@ def _get_modules_from_koji(
:param Variant variant: Variant with modules to find. :param Variant variant: Variant with modules to find.
:param dict variant_tags: Dict populated by this method. Key is `variant` :param dict variant_tags: Dict populated by this method. Key is `variant`
and value is list of Koji tags to get the RPMs from. and value is list of Koji tags to get the RPMs from.
:param list exclude_module_ns: Module name:stream which will be excluded.
""" """
# Find out all modules in every variant and add their Koji tags # Find out all modules in every variant and add their Koji tags
# to variant and variant_tags list. # to variant and variant_tags list.
for module in variant.get_modules(): for module in variant.get_modules():
koji_modules = get_koji_modules(compose, koji_wrapper, event, module["name"]) koji_modules = get_koji_modules(compose, koji_wrapper, event,
module["name"])
for koji_module in koji_modules: for koji_module in koji_modules:
nsvc = _add_module_to_variant( nsvc = _add_module_to_variant(
koji_wrapper, variant, koji_module, compose=compose koji_wrapper,
variant,
koji_module,
compose=compose,
exclude_module_ns=exclude_module_ns,
) )
if not nsvc: if not nsvc:
continue continue
@ -462,7 +488,8 @@ def filter_inherited(koji_proxy, event, module_builds, top_tag):
does not understand streams, so we have to reimplement it here. does not understand streams, so we have to reimplement it here.
""" """
inheritance = [ inheritance = [
tag["name"] for tag in koji_proxy.getFullInheritance(top_tag, event=event["id"]) tag["name"] for tag in
koji_proxy.getFullInheritance(top_tag, event=event["id"])
] ]
def keyfunc(mb): def keyfunc(mb):
@ -487,7 +514,8 @@ def filter_inherited(koji_proxy, event, module_builds, top_tag):
return result return result
def filter_by_whitelist(compose, module_builds, input_modules, expected_modules): def filter_by_whitelist(compose, module_builds, input_modules,
expected_modules):
""" """
Exclude modules from the list that do not match any pattern specified in Exclude modules from the list that do not match any pattern specified in
input_modules. Order may not be preserved. The last argument is a set of input_modules. Order may not be preserved. The last argument is a set of
@ -511,6 +539,7 @@ def filter_by_whitelist(compose, module_builds, input_modules, expected_modules)
info.get("context"), info.get("context"),
) )
nvr_patterns.add((pattern, spec["name"])) nvr_patterns.add((pattern, spec["name"]))
modules_to_keep = [] modules_to_keep = []
for mb in sorted(module_builds, key=lambda i: i['name']): for mb in sorted(module_builds, key=lambda i: i['name']):
@ -575,7 +604,13 @@ def _filter_expected_modules(
def _get_modules_from_koji_tags( def _get_modules_from_koji_tags(
compose, koji_wrapper, event_id, variant, variant_tags, tag_to_mmd compose,
koji_wrapper,
event_id,
variant,
variant_tags,
tag_to_mmd,
exclude_module_ns,
): ):
""" """
Loads modules for given `variant` from Koji, adds them to Loads modules for given `variant` from Koji, adds them to
@ -587,10 +622,12 @@ def _get_modules_from_koji_tags(
:param Variant variant: Variant with modules to find. :param Variant variant: Variant with modules to find.
:param dict variant_tags: Dict populated by this method. Key is `variant` :param dict variant_tags: Dict populated by this method. Key is `variant`
and value is list of Koji tags to get the RPMs from. and value is list of Koji tags to get the RPMs from.
:param list exclude_module_ns: Module name:stream which will be excluded.
""" """
# Compose tags from configuration # Compose tags from configuration
compose_tags = [ compose_tags = [
{"name": tag} for tag in force_list(compose.conf["pkgset_koji_module_tag"]) {"name": tag} for tag in
force_list(compose.conf["pkgset_koji_module_tag"])
] ]
# Get set of configured module names for this variant. If nothing is # Get set of configured module names for this variant. If nothing is
# configured, the set is empty. # configured, the set is empty.
@ -617,7 +654,8 @@ def _get_modules_from_koji_tags(
) )
# Filter out builds inherited from non-top tag # Filter out builds inherited from non-top tag
module_builds = filter_inherited(koji_proxy, event_id, module_builds, tag) module_builds = filter_inherited(koji_proxy, event_id, module_builds,
tag)
# Apply whitelist of modules if specified. # Apply whitelist of modules if specified.
variant_modules = variant.get_modules() variant_modules = variant.get_modules()
@ -625,6 +663,7 @@ def _get_modules_from_koji_tags(
module_builds = filter_by_whitelist( module_builds = filter_by_whitelist(
compose, module_builds, variant_modules, expected_modules compose, module_builds, variant_modules, expected_modules
) )
# Find the latest builds of all modules. This does following: # Find the latest builds of all modules. This does following:
# - Sorts the module_builds descending by Koji NVR (which maps to NSV # - Sorts the module_builds descending by Koji NVR (which maps to NSV
# for modules). Split release into modular version and context, and # for modules). Split release into modular version and context, and
@ -662,6 +701,18 @@ def _get_modules_from_koji_tags(
for build in latest_builds: for build in latest_builds:
# Get the Build from Koji to get modulemd and module_tag. # Get the Build from Koji to get modulemd and module_tag.
build = koji_proxy.getBuild(build["build_id"]) build = koji_proxy.getBuild(build["build_id"])
nsvc = _add_module_to_variant(
koji_wrapper,
variant,
build,
True,
compose=compose,
exclude_module_ns=exclude_module_ns,
)
if not nsvc:
continue
module_tag = ( module_tag = (
build.get("extra", {}) build.get("extra", {})
.get("typeinfo", {}) .get("typeinfo", {})
@ -671,12 +722,6 @@ def _get_modules_from_koji_tags(
variant_tags[variant].append(module_tag) variant_tags[variant].append(module_tag)
nsvc = _add_module_to_variant(
koji_wrapper, variant, build, True, compose=compose
)
if not nsvc:
continue
tag_to_mmd.setdefault(module_tag, {}) tag_to_mmd.setdefault(module_tag, {})
for arch in variant.arch_mmds: for arch in variant.arch_mmds:
try: try:
@ -767,26 +812,48 @@ def populate_global_pkgset(compose, koji_wrapper, path_prefix, event):
"modules." "modules."
) )
extra_modules = get_variant_data(
compose.conf, "pkgset_koji_module_builds", variant
)
# When adding extra modules, other modules of the same name:stream available
# in brew tag should be excluded.
exclude_module_ns = []
if extra_modules:
exclude_module_ns = [
":".join(nsvc.split(":")[:2]) for nsvc in extra_modules
]
if modular_koji_tags or ( if modular_koji_tags or (
compose.conf["pkgset_koji_module_tag"] and variant.modules compose.conf["pkgset_koji_module_tag"] and variant.modules
): ):
# List modules tagged in particular tags. # List modules tagged in particular tags.
_get_modules_from_koji_tags( _get_modules_from_koji_tags(
compose, koji_wrapper, event, variant, variant_tags, tag_to_mmd compose,
koji_wrapper,
event,
variant,
variant_tags,
tag_to_mmd,
exclude_module_ns,
) )
elif variant.modules: elif variant.modules:
# Search each module in Koji separately. Tagging does not come into # Search each module in Koji separately. Tagging does not come into
# play here. # play here.
_get_modules_from_koji( _get_modules_from_koji(
compose, koji_wrapper, event, variant, variant_tags, tag_to_mmd compose,
koji_wrapper,
event,
variant,
variant_tags,
tag_to_mmd,
exclude_module_ns,
) )
extra_modules = get_variant_data(
compose.conf, "pkgset_koji_module_builds", variant
)
if extra_modules: if extra_modules:
_add_extra_modules_to_variant( _add_extra_modules_to_variant(
compose, koji_wrapper, variant, extra_modules, variant_tags, tag_to_mmd compose, koji_wrapper, variant, extra_modules, variant_tags,
tag_to_mmd
) )
variant_scratch_modules = get_variant_data( variant_scratch_modules = get_variant_data(
@ -794,7 +861,8 @@ def populate_global_pkgset(compose, koji_wrapper, path_prefix, event):
) )
if variant_scratch_modules: if variant_scratch_modules:
_add_scratch_modules_to_variant( _add_scratch_modules_to_variant(
compose, variant, variant_scratch_modules, variant_tags, tag_to_mmd compose, variant, variant_scratch_modules, variant_tags,
tag_to_mmd
) )
# Ensure that every tag added to `variant_tags` is added also to # Ensure that every tag added to `variant_tags` is added also to
@ -819,8 +887,10 @@ def populate_global_pkgset(compose, koji_wrapper, path_prefix, event):
for compose_tag in compose_tags: for compose_tag in compose_tags:
compose.log_info("Loading package set for tag %s", compose_tag) compose.log_info("Loading package set for tag %s", compose_tag)
if compose_tag in pkgset_koji_tags: if compose_tag in pkgset_koji_tags:
extra_builds = force_list(compose.conf.get("pkgset_koji_builds", [])) extra_builds = force_list(
extra_tasks = force_list(compose.conf.get("pkgset_koji_scratch_tasks", [])) compose.conf.get("pkgset_koji_builds", []))
extra_tasks = force_list(
compose.conf.get("pkgset_koji_scratch_tasks", []))
else: else:
extra_builds = [] extra_builds = []
extra_tasks = [] extra_tasks = []
@ -926,7 +996,8 @@ def populate_global_pkgset(compose, koji_wrapper, path_prefix, event):
def get_koji_event_info(compose, koji_wrapper): def get_koji_event_info(compose, koji_wrapper):
event_file = os.path.join(compose.paths.work.topdir(arch="global"), "koji-event") event_file = os.path.join(compose.paths.work.topdir(arch="global"),
"koji-event")
compose.log_info("Getting koji event") compose.log_info("Getting koji event")
result = get_koji_event_raw(koji_wrapper, compose.koji_event, event_file) result = get_koji_event_raw(koji_wrapper, compose.koji_event, event_file)

26
pungi/scripts/create_packages_json.py
View File

@ -14,7 +14,7 @@ import os
import re import re
import tempfile import tempfile
from collections import defaultdict from collections import defaultdict
from typing import AnyStr, Dict, List, Optional, Any, Iterator from typing import AnyStr, Dict, List, Any, Iterator
import binascii import binascii
import createrepo_c as cr import createrepo_c as cr
@ -65,7 +65,7 @@ class RepoInfo:
# Only layout of specific package (which don't exist # Only layout of specific package (which don't exist
# in a reference repository) will be taken as example # in a reference repository) will be taken as example
is_reference: bool = False is_reference: bool = False
strict_arch: bool = False repo_type: str = 'present'
class PackagesGenerator: class PackagesGenerator:
@ -262,7 +262,11 @@ class PackagesGenerator:
) )
) )
all_packages = defaultdict(lambda: {'variants': list()}) all_packages = defaultdict(lambda: {'variants': list()})
for repo_info in self.repos: for repo_info in sorted(
self.repos,
key=lambda i: i.repo_type,
reverse=True,
):
repomd_records = self._get_repomd_records( repomd_records = self._get_repomd_records(
repo_info=repo_info, repo_info=repo_info,
) )
@ -298,6 +302,8 @@ class PackagesGenerator:
all_packages[package_key]['arch'] = package_arch all_packages[package_key]['arch'] = package_arch
all_packages[package_key]['package'] = package all_packages[package_key]['package'] = package
all_packages[package_key]['type'] = repo_info.is_reference all_packages[package_key]['type'] = repo_info.is_reference
elif repo_info.repo_type == 'absent' and (repo_info.name, repo_info.arch) in all_packages[package_key]['variants']:
all_packages[package_key]['variants'].remove((repo_info.name, repo_info.arch))
# replace an older package if it's not reference or # replace an older package if it's not reference or
# a newer package is from reference repo # a newer package is from reference repo
elif (not all_packages[package_key]['type'] or elif (not all_packages[package_key]['type'] or
@ -404,6 +410,14 @@ def create_parser():
choices=['yes', 'no'], choices=['yes', 'no'],
required=True, required=True,
) )
parser.add_argument(
'--repo-type',
action='append',
type=str,
help='Packages from repository will be removed or added to variant',
choices=['present', 'absent'],
required=True,
)
parser.add_argument( parser.add_argument(
'--excluded-packages', '--excluded-packages',
nargs='+', nargs='+',
@ -436,13 +450,14 @@ def cli_main():
args = create_parser().parse_args() args = create_parser().parse_args()
repos = [] repos = []
for repo_path, repo_folder, repo_name, \ for repo_path, repo_folder, repo_name, \
repo_arch, is_remote, is_reference in zip( repo_arch, is_remote, is_reference, repo_type in zip(
args.repo_path, args.repo_path,
args.repo_folder, args.repo_folder,
args.repo_name, args.repo_name,
args.repo_arch, args.repo_arch,
args.is_remote, args.is_remote,
args.is_reference, args.is_reference,
args.repo_type,
): ):
repos.append(RepoInfo( repos.append(RepoInfo(
path=repo_path, path=repo_path,
@ -450,7 +465,8 @@ def cli_main():
name=repo_name, name=repo_name,
arch=repo_arch, arch=repo_arch,
is_remote=True if is_remote == 'yes' else False, is_remote=True if is_remote == 'yes' else False,
is_reference=True if is_reference == 'yes' else False is_reference=True if is_reference == 'yes' else False,
repo_type=repo_type,
)) ))
pg = PackagesGenerator( pg = PackagesGenerator(
repos=repos, repos=repos,