Merge pull request 'ALBS-1030: Generate Devel section in packages.json' (#14) from ALBS-1030 into al_master
Reviewed-on: #14
This commit is contained in:
commit
60a347a4a2
|
@ -889,6 +889,8 @@ class KojiMockPackageSet(KojiPackageSet):
|
||||||
sigkey.lower() for sigkey in self.sigkey_ordering
|
sigkey.lower() for sigkey in self.sigkey_ordering
|
||||||
if sigkey is not None
|
if sigkey is not None
|
||||||
]
|
]
|
||||||
|
if not sigkeys:
|
||||||
|
return True
|
||||||
with open(rpm_path, 'rb') as fd:
|
with open(rpm_path, 'rb') as fd:
|
||||||
header = ts.hdrFromFdno(fd)
|
header = ts.hdrFromFdno(fd)
|
||||||
signature = header[rpm.RPMTAG_SIGGPG] or header[rpm.RPMTAG_SIGPGP]
|
signature = header[rpm.RPMTAG_SIGGPG] or header[rpm.RPMTAG_SIGPGP]
|
||||||
|
|
|
@ -35,10 +35,12 @@ import pungi.wrappers.kojiwrapper
|
||||||
from pungi.wrappers.comps import CompsWrapper
|
from pungi.wrappers.comps import CompsWrapper
|
||||||
from pungi.wrappers.mbs import MBSWrapper
|
from pungi.wrappers.mbs import MBSWrapper
|
||||||
import pungi.phases.pkgset.pkgsets
|
import pungi.phases.pkgset.pkgsets
|
||||||
|
|
||||||
from pungi.util import (
|
from pungi.util import (
|
||||||
retry,
|
retry,
|
||||||
get_arch_variant_data,
|
get_arch_variant_data,
|
||||||
get_variant_data,
|
get_variant_data,
|
||||||
|
|
||||||
read_single_module_stream_from_string,
|
read_single_module_stream_from_string,
|
||||||
read_single_module_stream_from_file,
|
read_single_module_stream_from_file,
|
||||||
)
|
)
|
||||||
|
@ -101,12 +103,12 @@ def variant_dict_from_str(compose, module_str):
|
||||||
release_regex = re.compile(r"^(\d){14}$")
|
release_regex = re.compile(r"^(\d){14}$")
|
||||||
|
|
||||||
section_start = module_str.rfind("-")
|
section_start = module_str.rfind("-")
|
||||||
module_str_first_part = module_str[section_start + 1 :]
|
module_str_first_part = module_str[section_start + 1:]
|
||||||
if release_regex.match(module_str_first_part):
|
if release_regex.match(module_str_first_part):
|
||||||
module_info["version"] = module_str_first_part
|
module_info["version"] = module_str_first_part
|
||||||
module_str = module_str[:section_start]
|
module_str = module_str[:section_start]
|
||||||
section_start = module_str.rfind("-")
|
section_start = module_str.rfind("-")
|
||||||
module_info["stream"] = module_str[section_start + 1 :]
|
module_info["stream"] = module_str[section_start + 1:]
|
||||||
else:
|
else:
|
||||||
module_info["stream"] = module_str_first_part
|
module_info["stream"] = module_str_first_part
|
||||||
module_info["name"] = module_str[:section_start]
|
module_info["name"] = module_str[:section_start]
|
||||||
|
@ -160,14 +162,16 @@ def get_koji_modules(compose, koji_wrapper, event, module_info_str):
|
||||||
# Store module versioning information into the dict, but make sure
|
# Store module versioning information into the dict, but make sure
|
||||||
# not to overwrite any existing keys.
|
# not to overwrite any existing keys.
|
||||||
md["module_stream"] = md["extra"]["typeinfo"]["module"]["stream"]
|
md["module_stream"] = md["extra"]["typeinfo"]["module"]["stream"]
|
||||||
md["module_version"] = int(md["extra"]["typeinfo"]["module"]["version"])
|
md["module_version"] = int(
|
||||||
|
md["extra"]["typeinfo"]["module"]["version"])
|
||||||
md["module_context"] = md["extra"]["typeinfo"]["module"]["context"]
|
md["module_context"] = md["extra"]["typeinfo"]["module"]["context"]
|
||||||
except KeyError:
|
except KeyError:
|
||||||
continue
|
continue
|
||||||
|
|
||||||
if md["state"] == pungi.wrappers.kojiwrapper.KOJI_BUILD_DELETED:
|
if md["state"] == pungi.wrappers.kojiwrapper.KOJI_BUILD_DELETED:
|
||||||
compose.log_debug(
|
compose.log_debug(
|
||||||
"Module build %s has been deleted, ignoring it." % build["name"]
|
"Module build %s has been deleted, ignoring it." % build[
|
||||||
|
"name"]
|
||||||
)
|
)
|
||||||
continue
|
continue
|
||||||
|
|
||||||
|
@ -189,7 +193,8 @@ def get_koji_modules(compose, koji_wrapper, event, module_info_str):
|
||||||
)
|
)
|
||||||
latest_version = sorted_modules[0]["module_version"]
|
latest_version = sorted_modules[0]["module_version"]
|
||||||
modules = [
|
modules = [
|
||||||
module for module in modules if latest_version == module["module_version"]
|
module for module in modules
|
||||||
|
if latest_version == module["module_version"]
|
||||||
]
|
]
|
||||||
|
|
||||||
return modules
|
return modules
|
||||||
|
@ -205,7 +210,8 @@ class PkgsetSourceKojiMock(pungi.phases.pkgset.source.PkgsetSourceBase):
|
||||||
get_all_arches(compose),
|
get_all_arches(compose),
|
||||||
)
|
)
|
||||||
# path prefix must contain trailing '/'
|
# path prefix must contain trailing '/'
|
||||||
path_prefix = self.koji_wrapper.koji_module.config.topdir.rstrip("/") + "/"
|
path_prefix = self.koji_wrapper.koji_module.config.topdir.rstrip(
|
||||||
|
"/") + "/"
|
||||||
package_sets = get_pkgset_from_koji(
|
package_sets = get_pkgset_from_koji(
|
||||||
self.compose, self.koji_wrapper, path_prefix
|
self.compose, self.koji_wrapper, path_prefix
|
||||||
)
|
)
|
||||||
|
@ -214,7 +220,8 @@ class PkgsetSourceKojiMock(pungi.phases.pkgset.source.PkgsetSourceBase):
|
||||||
|
|
||||||
def get_pkgset_from_koji(compose, koji_wrapper, path_prefix):
|
def get_pkgset_from_koji(compose, koji_wrapper, path_prefix):
|
||||||
event_info = get_koji_event_info(compose, koji_wrapper)
|
event_info = get_koji_event_info(compose, koji_wrapper)
|
||||||
return populate_global_pkgset(compose, koji_wrapper, path_prefix, event_info)
|
return populate_global_pkgset(compose, koji_wrapper, path_prefix,
|
||||||
|
event_info)
|
||||||
|
|
||||||
|
|
||||||
def _add_module_to_variant(
|
def _add_module_to_variant(
|
||||||
|
@ -241,13 +248,16 @@ def _add_module_to_variant(
|
||||||
if archive["btype"] != "module":
|
if archive["btype"] != "module":
|
||||||
# Skip non module archives
|
# Skip non module archives
|
||||||
continue
|
continue
|
||||||
|
|
||||||
filename = archive["filename"]
|
filename = archive["filename"]
|
||||||
file_path = os.path.join(
|
file_path = os.path.join(
|
||||||
koji_wrapper.koji_module.pathinfo.topdir,
|
koji_wrapper.koji_module.pathinfo.topdir,
|
||||||
'modules',
|
'modules',
|
||||||
build['arch'],
|
build['arch'],
|
||||||
build['extra']['typeinfo']['module']['content_koji_tag']
|
build['extra']['typeinfo']['module']['content_koji_tag']
|
||||||
|
|
||||||
)
|
)
|
||||||
|
|
||||||
mmds[filename] = file_path
|
mmds[filename] = file_path
|
||||||
|
|
||||||
if len(mmds) <= 1:
|
if len(mmds) <= 1:
|
||||||
|
@ -266,17 +276,22 @@ def _add_module_to_variant(
|
||||||
added = False
|
added = False
|
||||||
|
|
||||||
for arch in variant.arches:
|
for arch in variant.arches:
|
||||||
if _is_filtered_out(compose, variant, arch, info["name"], info["stream"]):
|
if _is_filtered_out(compose, variant, arch, info["name"],
|
||||||
compose.log_debug("Module %s is filtered from %s.%s", nsvc, variant, arch)
|
info["stream"]):
|
||||||
|
compose.log_debug("Module %s is filtered from %s.%s", nsvc,
|
||||||
|
variant, arch)
|
||||||
continue
|
continue
|
||||||
|
|
||||||
filename = "modulemd.%s.txt" % arch
|
filename = "modulemd.%s.txt" % arch
|
||||||
try:
|
try:
|
||||||
mod_stream = read_single_module_stream_from_file(
|
mod_stream = read_single_module_stream_from_file(
|
||||||
mmds[filename], compose, arch, build
|
mmds[filename], compose, arch, build
|
||||||
|
|
||||||
)
|
)
|
||||||
if mod_stream:
|
if mod_stream:
|
||||||
added = True
|
added = True
|
||||||
variant.arch_mmds.setdefault(arch, {})[nsvc] = mod_stream
|
variant.arch_mmds.setdefault(arch, {})[nsvc] = mod_stream
|
||||||
|
|
||||||
added = True
|
added = True
|
||||||
except KeyError:
|
except KeyError:
|
||||||
# There is no modulemd for this arch. This could mean an arch was
|
# There is no modulemd for this arch. This could mean an arch was
|
||||||
|
@ -298,7 +313,8 @@ def _add_extra_modules_to_variant(
|
||||||
compose, koji_wrapper, variant, extra_modules, variant_tags, tag_to_mmd
|
compose, koji_wrapper, variant, extra_modules, variant_tags, tag_to_mmd
|
||||||
):
|
):
|
||||||
for nsvc in extra_modules:
|
for nsvc in extra_modules:
|
||||||
msg = "Adding extra module build '%s' to variant '%s'" % (nsvc, variant)
|
msg = "Adding extra module build '%s' to variant '%s'" % (
|
||||||
|
nsvc, variant)
|
||||||
compose.log_info(msg)
|
compose.log_info(msg)
|
||||||
|
|
||||||
nsvc_info = nsvc.split(":")
|
nsvc_info = nsvc.split(":")
|
||||||
|
@ -344,7 +360,8 @@ def _add_scratch_modules_to_variant(
|
||||||
compose, variant, scratch_modules, variant_tags, tag_to_mmd
|
compose, variant, scratch_modules, variant_tags, tag_to_mmd
|
||||||
):
|
):
|
||||||
if compose.compose_type != "test" and scratch_modules:
|
if compose.compose_type != "test" and scratch_modules:
|
||||||
compose.log_warning("Only test composes could include scratch module builds")
|
compose.log_warning(
|
||||||
|
"Only test composes could include scratch module builds")
|
||||||
return
|
return
|
||||||
|
|
||||||
mbs = MBSWrapper(compose.conf["mbs_api_url"])
|
mbs = MBSWrapper(compose.conf["mbs_api_url"])
|
||||||
|
@ -355,7 +372,8 @@ def _add_scratch_modules_to_variant(
|
||||||
try:
|
try:
|
||||||
final_modulemd = mbs.final_modulemd(module_build["id"])
|
final_modulemd = mbs.final_modulemd(module_build["id"])
|
||||||
except Exception:
|
except Exception:
|
||||||
compose.log_error("Unable to get modulemd for build %s" % module_build)
|
compose.log_error(
|
||||||
|
"Unable to get modulemd for build %s" % module_build)
|
||||||
raise
|
raise
|
||||||
tag = module_build["koji_tag"]
|
tag = module_build["koji_tag"]
|
||||||
variant_tags[variant].append(tag)
|
variant_tags[variant].append(tag)
|
||||||
|
@ -363,8 +381,7 @@ def _add_scratch_modules_to_variant(
|
||||||
for arch in variant.arches:
|
for arch in variant.arches:
|
||||||
try:
|
try:
|
||||||
mmd = read_single_module_stream_from_string(
|
mmd = read_single_module_stream_from_string(
|
||||||
final_modulemd[arch]
|
final_modulemd[arch])
|
||||||
)
|
|
||||||
variant.arch_mmds.setdefault(arch, {})[nsvc] = mmd
|
variant.arch_mmds.setdefault(arch, {})[nsvc] = mmd
|
||||||
except KeyError:
|
except KeyError:
|
||||||
continue
|
continue
|
||||||
|
@ -390,21 +407,24 @@ def _is_filtered_out(compose, variant, arch, module_name, module_stream):
|
||||||
if not compose:
|
if not compose:
|
||||||
return False
|
return False
|
||||||
|
|
||||||
for filter in get_arch_variant_data(compose.conf, "filter_modules", arch, variant):
|
for filter in get_arch_variant_data(compose.conf, "filter_modules", arch,
|
||||||
|
variant):
|
||||||
if ":" not in filter:
|
if ":" not in filter:
|
||||||
name_filter = filter
|
name_filter = filter
|
||||||
stream_filter = "*"
|
stream_filter = "*"
|
||||||
else:
|
else:
|
||||||
name_filter, stream_filter = filter.split(":", 1)
|
name_filter, stream_filter = filter.split(":", 1)
|
||||||
|
|
||||||
if fnmatch(module_name, name_filter) and fnmatch(module_stream, stream_filter):
|
if fnmatch(module_name, name_filter) and fnmatch(module_stream,
|
||||||
|
stream_filter):
|
||||||
return True
|
return True
|
||||||
|
|
||||||
return False
|
return False
|
||||||
|
|
||||||
|
|
||||||
def _get_modules_from_koji(
|
def _get_modules_from_koji(
|
||||||
compose, koji_wrapper, event, variant, variant_tags, tag_to_mmd
|
compose, koji_wrapper, event, variant, variant_tags, tag_to_mmd,
|
||||||
|
exclude_module_ns
|
||||||
):
|
):
|
||||||
"""
|
"""
|
||||||
Loads modules for given `variant` from koji `session`, adds them to
|
Loads modules for given `variant` from koji `session`, adds them to
|
||||||
|
@ -415,15 +435,21 @@ def _get_modules_from_koji(
|
||||||
:param Variant variant: Variant with modules to find.
|
:param Variant variant: Variant with modules to find.
|
||||||
:param dict variant_tags: Dict populated by this method. Key is `variant`
|
:param dict variant_tags: Dict populated by this method. Key is `variant`
|
||||||
and value is list of Koji tags to get the RPMs from.
|
and value is list of Koji tags to get the RPMs from.
|
||||||
|
:param list exclude_module_ns: Module name:stream which will be excluded.
|
||||||
"""
|
"""
|
||||||
|
|
||||||
# Find out all modules in every variant and add their Koji tags
|
# Find out all modules in every variant and add their Koji tags
|
||||||
# to variant and variant_tags list.
|
# to variant and variant_tags list.
|
||||||
for module in variant.get_modules():
|
for module in variant.get_modules():
|
||||||
koji_modules = get_koji_modules(compose, koji_wrapper, event, module["name"])
|
koji_modules = get_koji_modules(compose, koji_wrapper, event,
|
||||||
|
module["name"])
|
||||||
for koji_module in koji_modules:
|
for koji_module in koji_modules:
|
||||||
nsvc = _add_module_to_variant(
|
nsvc = _add_module_to_variant(
|
||||||
koji_wrapper, variant, koji_module, compose=compose
|
koji_wrapper,
|
||||||
|
variant,
|
||||||
|
koji_module,
|
||||||
|
compose=compose,
|
||||||
|
exclude_module_ns=exclude_module_ns,
|
||||||
)
|
)
|
||||||
if not nsvc:
|
if not nsvc:
|
||||||
continue
|
continue
|
||||||
|
@ -462,7 +488,8 @@ def filter_inherited(koji_proxy, event, module_builds, top_tag):
|
||||||
does not understand streams, so we have to reimplement it here.
|
does not understand streams, so we have to reimplement it here.
|
||||||
"""
|
"""
|
||||||
inheritance = [
|
inheritance = [
|
||||||
tag["name"] for tag in koji_proxy.getFullInheritance(top_tag, event=event["id"])
|
tag["name"] for tag in
|
||||||
|
koji_proxy.getFullInheritance(top_tag, event=event["id"])
|
||||||
]
|
]
|
||||||
|
|
||||||
def keyfunc(mb):
|
def keyfunc(mb):
|
||||||
|
@ -487,7 +514,8 @@ def filter_inherited(koji_proxy, event, module_builds, top_tag):
|
||||||
return result
|
return result
|
||||||
|
|
||||||
|
|
||||||
def filter_by_whitelist(compose, module_builds, input_modules, expected_modules):
|
def filter_by_whitelist(compose, module_builds, input_modules,
|
||||||
|
expected_modules):
|
||||||
"""
|
"""
|
||||||
Exclude modules from the list that do not match any pattern specified in
|
Exclude modules from the list that do not match any pattern specified in
|
||||||
input_modules. Order may not be preserved. The last argument is a set of
|
input_modules. Order may not be preserved. The last argument is a set of
|
||||||
|
@ -511,6 +539,7 @@ def filter_by_whitelist(compose, module_builds, input_modules, expected_modules)
|
||||||
info.get("context"),
|
info.get("context"),
|
||||||
)
|
)
|
||||||
nvr_patterns.add((pattern, spec["name"]))
|
nvr_patterns.add((pattern, spec["name"]))
|
||||||
|
|
||||||
modules_to_keep = []
|
modules_to_keep = []
|
||||||
|
|
||||||
for mb in sorted(module_builds, key=lambda i: i['name']):
|
for mb in sorted(module_builds, key=lambda i: i['name']):
|
||||||
|
@ -575,7 +604,13 @@ def _filter_expected_modules(
|
||||||
|
|
||||||
|
|
||||||
def _get_modules_from_koji_tags(
|
def _get_modules_from_koji_tags(
|
||||||
compose, koji_wrapper, event_id, variant, variant_tags, tag_to_mmd
|
compose,
|
||||||
|
koji_wrapper,
|
||||||
|
event_id,
|
||||||
|
variant,
|
||||||
|
variant_tags,
|
||||||
|
tag_to_mmd,
|
||||||
|
exclude_module_ns,
|
||||||
):
|
):
|
||||||
"""
|
"""
|
||||||
Loads modules for given `variant` from Koji, adds them to
|
Loads modules for given `variant` from Koji, adds them to
|
||||||
|
@ -587,10 +622,12 @@ def _get_modules_from_koji_tags(
|
||||||
:param Variant variant: Variant with modules to find.
|
:param Variant variant: Variant with modules to find.
|
||||||
:param dict variant_tags: Dict populated by this method. Key is `variant`
|
:param dict variant_tags: Dict populated by this method. Key is `variant`
|
||||||
and value is list of Koji tags to get the RPMs from.
|
and value is list of Koji tags to get the RPMs from.
|
||||||
|
:param list exclude_module_ns: Module name:stream which will be excluded.
|
||||||
"""
|
"""
|
||||||
# Compose tags from configuration
|
# Compose tags from configuration
|
||||||
compose_tags = [
|
compose_tags = [
|
||||||
{"name": tag} for tag in force_list(compose.conf["pkgset_koji_module_tag"])
|
{"name": tag} for tag in
|
||||||
|
force_list(compose.conf["pkgset_koji_module_tag"])
|
||||||
]
|
]
|
||||||
# Get set of configured module names for this variant. If nothing is
|
# Get set of configured module names for this variant. If nothing is
|
||||||
# configured, the set is empty.
|
# configured, the set is empty.
|
||||||
|
@ -617,7 +654,8 @@ def _get_modules_from_koji_tags(
|
||||||
)
|
)
|
||||||
|
|
||||||
# Filter out builds inherited from non-top tag
|
# Filter out builds inherited from non-top tag
|
||||||
module_builds = filter_inherited(koji_proxy, event_id, module_builds, tag)
|
module_builds = filter_inherited(koji_proxy, event_id, module_builds,
|
||||||
|
tag)
|
||||||
|
|
||||||
# Apply whitelist of modules if specified.
|
# Apply whitelist of modules if specified.
|
||||||
variant_modules = variant.get_modules()
|
variant_modules = variant.get_modules()
|
||||||
|
@ -625,6 +663,7 @@ def _get_modules_from_koji_tags(
|
||||||
module_builds = filter_by_whitelist(
|
module_builds = filter_by_whitelist(
|
||||||
compose, module_builds, variant_modules, expected_modules
|
compose, module_builds, variant_modules, expected_modules
|
||||||
)
|
)
|
||||||
|
|
||||||
# Find the latest builds of all modules. This does following:
|
# Find the latest builds of all modules. This does following:
|
||||||
# - Sorts the module_builds descending by Koji NVR (which maps to NSV
|
# - Sorts the module_builds descending by Koji NVR (which maps to NSV
|
||||||
# for modules). Split release into modular version and context, and
|
# for modules). Split release into modular version and context, and
|
||||||
|
@ -662,6 +701,18 @@ def _get_modules_from_koji_tags(
|
||||||
for build in latest_builds:
|
for build in latest_builds:
|
||||||
# Get the Build from Koji to get modulemd and module_tag.
|
# Get the Build from Koji to get modulemd and module_tag.
|
||||||
build = koji_proxy.getBuild(build["build_id"])
|
build = koji_proxy.getBuild(build["build_id"])
|
||||||
|
|
||||||
|
nsvc = _add_module_to_variant(
|
||||||
|
koji_wrapper,
|
||||||
|
variant,
|
||||||
|
build,
|
||||||
|
True,
|
||||||
|
compose=compose,
|
||||||
|
exclude_module_ns=exclude_module_ns,
|
||||||
|
)
|
||||||
|
if not nsvc:
|
||||||
|
continue
|
||||||
|
|
||||||
module_tag = (
|
module_tag = (
|
||||||
build.get("extra", {})
|
build.get("extra", {})
|
||||||
.get("typeinfo", {})
|
.get("typeinfo", {})
|
||||||
|
@ -671,12 +722,6 @@ def _get_modules_from_koji_tags(
|
||||||
|
|
||||||
variant_tags[variant].append(module_tag)
|
variant_tags[variant].append(module_tag)
|
||||||
|
|
||||||
nsvc = _add_module_to_variant(
|
|
||||||
koji_wrapper, variant, build, True, compose=compose
|
|
||||||
)
|
|
||||||
if not nsvc:
|
|
||||||
continue
|
|
||||||
|
|
||||||
tag_to_mmd.setdefault(module_tag, {})
|
tag_to_mmd.setdefault(module_tag, {})
|
||||||
for arch in variant.arch_mmds:
|
for arch in variant.arch_mmds:
|
||||||
try:
|
try:
|
||||||
|
@ -767,26 +812,48 @@ def populate_global_pkgset(compose, koji_wrapper, path_prefix, event):
|
||||||
"modules."
|
"modules."
|
||||||
)
|
)
|
||||||
|
|
||||||
|
extra_modules = get_variant_data(
|
||||||
|
compose.conf, "pkgset_koji_module_builds", variant
|
||||||
|
)
|
||||||
|
|
||||||
|
# When adding extra modules, other modules of the same name:stream available
|
||||||
|
# in brew tag should be excluded.
|
||||||
|
exclude_module_ns = []
|
||||||
|
if extra_modules:
|
||||||
|
exclude_module_ns = [
|
||||||
|
":".join(nsvc.split(":")[:2]) for nsvc in extra_modules
|
||||||
|
]
|
||||||
|
|
||||||
if modular_koji_tags or (
|
if modular_koji_tags or (
|
||||||
compose.conf["pkgset_koji_module_tag"] and variant.modules
|
compose.conf["pkgset_koji_module_tag"] and variant.modules
|
||||||
):
|
):
|
||||||
# List modules tagged in particular tags.
|
# List modules tagged in particular tags.
|
||||||
_get_modules_from_koji_tags(
|
_get_modules_from_koji_tags(
|
||||||
compose, koji_wrapper, event, variant, variant_tags, tag_to_mmd
|
compose,
|
||||||
|
koji_wrapper,
|
||||||
|
event,
|
||||||
|
variant,
|
||||||
|
variant_tags,
|
||||||
|
tag_to_mmd,
|
||||||
|
exclude_module_ns,
|
||||||
)
|
)
|
||||||
elif variant.modules:
|
elif variant.modules:
|
||||||
# Search each module in Koji separately. Tagging does not come into
|
# Search each module in Koji separately. Tagging does not come into
|
||||||
# play here.
|
# play here.
|
||||||
_get_modules_from_koji(
|
_get_modules_from_koji(
|
||||||
compose, koji_wrapper, event, variant, variant_tags, tag_to_mmd
|
compose,
|
||||||
|
koji_wrapper,
|
||||||
|
event,
|
||||||
|
variant,
|
||||||
|
variant_tags,
|
||||||
|
tag_to_mmd,
|
||||||
|
exclude_module_ns,
|
||||||
)
|
)
|
||||||
|
|
||||||
extra_modules = get_variant_data(
|
|
||||||
compose.conf, "pkgset_koji_module_builds", variant
|
|
||||||
)
|
|
||||||
if extra_modules:
|
if extra_modules:
|
||||||
_add_extra_modules_to_variant(
|
_add_extra_modules_to_variant(
|
||||||
compose, koji_wrapper, variant, extra_modules, variant_tags, tag_to_mmd
|
compose, koji_wrapper, variant, extra_modules, variant_tags,
|
||||||
|
tag_to_mmd
|
||||||
)
|
)
|
||||||
|
|
||||||
variant_scratch_modules = get_variant_data(
|
variant_scratch_modules = get_variant_data(
|
||||||
|
@ -794,7 +861,8 @@ def populate_global_pkgset(compose, koji_wrapper, path_prefix, event):
|
||||||
)
|
)
|
||||||
if variant_scratch_modules:
|
if variant_scratch_modules:
|
||||||
_add_scratch_modules_to_variant(
|
_add_scratch_modules_to_variant(
|
||||||
compose, variant, variant_scratch_modules, variant_tags, tag_to_mmd
|
compose, variant, variant_scratch_modules, variant_tags,
|
||||||
|
tag_to_mmd
|
||||||
)
|
)
|
||||||
|
|
||||||
# Ensure that every tag added to `variant_tags` is added also to
|
# Ensure that every tag added to `variant_tags` is added also to
|
||||||
|
@ -819,8 +887,10 @@ def populate_global_pkgset(compose, koji_wrapper, path_prefix, event):
|
||||||
for compose_tag in compose_tags:
|
for compose_tag in compose_tags:
|
||||||
compose.log_info("Loading package set for tag %s", compose_tag)
|
compose.log_info("Loading package set for tag %s", compose_tag)
|
||||||
if compose_tag in pkgset_koji_tags:
|
if compose_tag in pkgset_koji_tags:
|
||||||
extra_builds = force_list(compose.conf.get("pkgset_koji_builds", []))
|
extra_builds = force_list(
|
||||||
extra_tasks = force_list(compose.conf.get("pkgset_koji_scratch_tasks", []))
|
compose.conf.get("pkgset_koji_builds", []))
|
||||||
|
extra_tasks = force_list(
|
||||||
|
compose.conf.get("pkgset_koji_scratch_tasks", []))
|
||||||
else:
|
else:
|
||||||
extra_builds = []
|
extra_builds = []
|
||||||
extra_tasks = []
|
extra_tasks = []
|
||||||
|
@ -926,7 +996,8 @@ def populate_global_pkgset(compose, koji_wrapper, path_prefix, event):
|
||||||
|
|
||||||
|
|
||||||
def get_koji_event_info(compose, koji_wrapper):
|
def get_koji_event_info(compose, koji_wrapper):
|
||||||
event_file = os.path.join(compose.paths.work.topdir(arch="global"), "koji-event")
|
event_file = os.path.join(compose.paths.work.topdir(arch="global"),
|
||||||
|
"koji-event")
|
||||||
|
|
||||||
compose.log_info("Getting koji event")
|
compose.log_info("Getting koji event")
|
||||||
result = get_koji_event_raw(koji_wrapper, compose.koji_event, event_file)
|
result = get_koji_event_raw(koji_wrapper, compose.koji_event, event_file)
|
||||||
|
|
|
@ -14,7 +14,7 @@ import os
|
||||||
import re
|
import re
|
||||||
import tempfile
|
import tempfile
|
||||||
from collections import defaultdict
|
from collections import defaultdict
|
||||||
from typing import AnyStr, Dict, List, Optional, Any, Iterator
|
from typing import AnyStr, Dict, List, Any, Iterator
|
||||||
|
|
||||||
import binascii
|
import binascii
|
||||||
import createrepo_c as cr
|
import createrepo_c as cr
|
||||||
|
@ -65,7 +65,7 @@ class RepoInfo:
|
||||||
# Only layout of specific package (which don't exist
|
# Only layout of specific package (which don't exist
|
||||||
# in a reference repository) will be taken as example
|
# in a reference repository) will be taken as example
|
||||||
is_reference: bool = False
|
is_reference: bool = False
|
||||||
strict_arch: bool = False
|
repo_type: str = 'present'
|
||||||
|
|
||||||
|
|
||||||
class PackagesGenerator:
|
class PackagesGenerator:
|
||||||
|
@ -262,7 +262,11 @@ class PackagesGenerator:
|
||||||
)
|
)
|
||||||
)
|
)
|
||||||
all_packages = defaultdict(lambda: {'variants': list()})
|
all_packages = defaultdict(lambda: {'variants': list()})
|
||||||
for repo_info in self.repos:
|
for repo_info in sorted(
|
||||||
|
self.repos,
|
||||||
|
key=lambda i: i.repo_type,
|
||||||
|
reverse=True,
|
||||||
|
):
|
||||||
repomd_records = self._get_repomd_records(
|
repomd_records = self._get_repomd_records(
|
||||||
repo_info=repo_info,
|
repo_info=repo_info,
|
||||||
)
|
)
|
||||||
|
@ -298,6 +302,8 @@ class PackagesGenerator:
|
||||||
all_packages[package_key]['arch'] = package_arch
|
all_packages[package_key]['arch'] = package_arch
|
||||||
all_packages[package_key]['package'] = package
|
all_packages[package_key]['package'] = package
|
||||||
all_packages[package_key]['type'] = repo_info.is_reference
|
all_packages[package_key]['type'] = repo_info.is_reference
|
||||||
|
elif repo_info.repo_type == 'absent' and (repo_info.name, repo_info.arch) in all_packages[package_key]['variants']:
|
||||||
|
all_packages[package_key]['variants'].remove((repo_info.name, repo_info.arch))
|
||||||
# replace an older package if it's not reference or
|
# replace an older package if it's not reference or
|
||||||
# a newer package is from reference repo
|
# a newer package is from reference repo
|
||||||
elif (not all_packages[package_key]['type'] or
|
elif (not all_packages[package_key]['type'] or
|
||||||
|
@ -404,6 +410,14 @@ def create_parser():
|
||||||
choices=['yes', 'no'],
|
choices=['yes', 'no'],
|
||||||
required=True,
|
required=True,
|
||||||
)
|
)
|
||||||
|
parser.add_argument(
|
||||||
|
'--repo-type',
|
||||||
|
action='append',
|
||||||
|
type=str,
|
||||||
|
help='Packages from repository will be removed or added to variant',
|
||||||
|
choices=['present', 'absent'],
|
||||||
|
required=True,
|
||||||
|
)
|
||||||
parser.add_argument(
|
parser.add_argument(
|
||||||
'--excluded-packages',
|
'--excluded-packages',
|
||||||
nargs='+',
|
nargs='+',
|
||||||
|
@ -436,13 +450,14 @@ def cli_main():
|
||||||
args = create_parser().parse_args()
|
args = create_parser().parse_args()
|
||||||
repos = []
|
repos = []
|
||||||
for repo_path, repo_folder, repo_name, \
|
for repo_path, repo_folder, repo_name, \
|
||||||
repo_arch, is_remote, is_reference in zip(
|
repo_arch, is_remote, is_reference, repo_type in zip(
|
||||||
args.repo_path,
|
args.repo_path,
|
||||||
args.repo_folder,
|
args.repo_folder,
|
||||||
args.repo_name,
|
args.repo_name,
|
||||||
args.repo_arch,
|
args.repo_arch,
|
||||||
args.is_remote,
|
args.is_remote,
|
||||||
args.is_reference,
|
args.is_reference,
|
||||||
|
args.repo_type,
|
||||||
):
|
):
|
||||||
repos.append(RepoInfo(
|
repos.append(RepoInfo(
|
||||||
path=repo_path,
|
path=repo_path,
|
||||||
|
@ -450,7 +465,8 @@ def cli_main():
|
||||||
name=repo_name,
|
name=repo_name,
|
||||||
arch=repo_arch,
|
arch=repo_arch,
|
||||||
is_remote=True if is_remote == 'yes' else False,
|
is_remote=True if is_remote == 'yes' else False,
|
||||||
is_reference=True if is_reference == 'yes' else False
|
is_reference=True if is_reference == 'yes' else False,
|
||||||
|
repo_type=repo_type,
|
||||||
))
|
))
|
||||||
pg = PackagesGenerator(
|
pg = PackagesGenerator(
|
||||||
repos=repos,
|
repos=repos,
|
||||||
|
|
Loading…
Reference in New Issue