almalinux/pungi
14
1
Fork 0
Code Issues Pull Requests Releases Activity

ALBS-1040: Investigate why Pungi doesn't put modules packages into the final repos

Browse Source 
- Refactoring
- KojiMock extracts all modules which are suitable for the variant's arches
This commit is contained in:
soksanichenko 2023-03-14 18:25:21 +02:00
parent d66eb0dea8
commit fa4640f03e
7 changed files with 108 additions and 438 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
pungi/phases/gather/__init__.py
View File

@ -23,6 +23,7 @@ import threading
from kobo.rpmlib import parse_nvra
from kobo.shortcuts import run
from productmd.rpms import Rpms
from pungi.phases.pkgset.common import get_all_arches
from six.moves import cPickle as pickle
try:
@ -652,7 +653,8 @@ def _make_lookaside_repo(compose, variant, arch, pkg_map, package_sets=None):
).koji_module.config.topdir.rstrip("/")
+ "/",
"kojimock": lambda: pungi.wrappers.kojiwrapper.KojiMockWrapper(
compose
compose,
get_all_arches(compose),
).koji_module.config.topdir.rstrip("/")
+ "/",
}

426
pungi/phases/pkgset/pkgsets.py
View File

@ -480,7 +480,8 @@ class KojiPackageSet(PackageSetBase):
response = None
if self.cache_region:
cache_key = "KojiPackageSet.get_latest_rpms_%s_%s_%s" % (
cache_key = "%s.get_latest_rpms_%s_%s_%s" % (
str(self.__class__.__name__),
str(tag),
str(event),
str(inherit),
@ -816,22 +817,22 @@ class KojiPackageSet(PackageSetBase):
return False
class KojiMockPackageSet(PackageSetBase):
class KojiMockPackageSet(KojiPackageSet):
def __init__(
self,
name,
koji_wrapper,
sigkey_ordering,
arches=None,
logger=None,
packages=None,
allow_invalid_sigkeys=False,
populate_only_packages=False,
cache_region=None,
extra_builds=None,
extra_tasks=None,
signed_packages_retries=0,
signed_packages_wait=30,
self,
name,
koji_wrapper,
sigkey_ordering,
arches=None,
logger=None,
packages=None,
allow_invalid_sigkeys=False,
populate_only_packages=False,
cache_region=None,
extra_builds=None,
extra_tasks=None,
signed_packages_retries=0,
signed_packages_wait=30,
):
"""
Creates new KojiPackageSet.
@ -865,135 +866,21 @@ class KojiMockPackageSet(PackageSetBase):
and include in the package set. Useful when building testing compose
with RPM scratch builds.
"""
super(KojiMockPackageSet , self).__init__(
super(KojiMockPackageSet, self).__init__(
name,
koji_wrapper=koji_wrapper,
sigkey_ordering=sigkey_ordering,
arches=arches,
logger=logger,
packages=packages,
allow_invalid_sigkeys=allow_invalid_sigkeys,
populate_only_packages=populate_only_packages,
cache_region=cache_region,
extra_builds=extra_builds,
extra_tasks=extra_tasks,
signed_packages_retries=signed_packages_retries,
signed_packages_wait=signed_packages_wait,
)
self.koji_wrapper = koji_wrapper
# Names of packages to look for in the Koji tag.
self.packages = set(packages or [])
self.populate_only_packages = populate_only_packages
self.cache_region = cache_region
self.extra_builds = extra_builds or []
self.extra_tasks = extra_tasks or []
self.reuse = None
self.signed_packages_retries = signed_packages_retries
self.signed_packages_wait = signed_packages_wait
def __getstate__(self):
result = self.__dict__.copy()
del result["koji_wrapper"]
del result["_logger"]
if "cache_region" in result:
del result["cache_region"]
return result
def __setstate__(self, data):
self._logger = None
self.__dict__.update(data)
@property
def koji_proxy(self):
return self.koji_wrapper.koji_proxy
def get_extra_rpms(self):
if not self.extra_builds:
return [], []
rpms = []
builds = []
builds = self.koji_wrapper.retrying_multicall_map(
self.koji_proxy, self.koji_proxy.getBuild, list_of_args=self.extra_builds
)
rpms_in_builds = self.koji_wrapper.retrying_multicall_map(
self.koji_proxy,
self.koji_proxy.listBuildRPMs,
list_of_args=self.extra_builds,
)
rpms = []
for rpms_in_build in rpms_in_builds:
rpms += rpms_in_build
return rpms, builds
def get_extra_rpms_from_tasks(self):
"""
Returns manually constructed RPM infos from the Koji tasks defined
in `self.extra_tasks`.
:rtype: list
:return: List with RPM infos defined as dicts with following keys:
- name, version, release, arch, src - as returned by parse_nvra.
- path_from_task - Full path to RPM on /mnt/koji.
- build_id - Always set to None.
"""
if not self.extra_tasks:
return []
# Get the IDs of children tasks - these are the tasks containing
# the resulting RPMs.
children_tasks = self.koji_wrapper.retrying_multicall_map(
self.koji_proxy,
self.koji_proxy.getTaskChildren,
list_of_args=self.extra_tasks,
)
children_task_ids = []
for tasks in children_tasks:
children_task_ids += [t["id"] for t in tasks]
# Get the results of these children tasks.
results = self.koji_wrapper.retrying_multicall_map(
self.koji_proxy,
self.koji_proxy.getTaskResult,
list_of_args=children_task_ids,
)
rpms = []
for result in results:
rpms += result.get("rpms", [])
rpms += result.get("srpms", [])
rpm_infos = []
for rpm in rpms:
rpm_info = kobo.rpmlib.parse_nvra(os.path.basename(rpm))
rpm_info["path_from_task"] = os.path.join(
self.koji_wrapper.koji_module.pathinfo.work(), rpm
)
rpm_info["build_id"] = None
rpm_infos.append(rpm_info)
return rpm_infos
def get_latest_rpms(self, tag, event, inherit=True):
if not tag:
return [], []
response = None
if self.cache_region:
cache_key = "KojiPackageSet.get_latest_rpms_%s_%s_%s" % (
str(tag),
str(event),
str(inherit),
)
try:
response = self.cache_region.get(cache_key)
except Exception:
pass
if not response:
response = self.koji_proxy.listTaggedRPMS(
tag, event=event, inherit=inherit, latest=True
)
if self.cache_region:
try:
self.cache_region.set(cache_key, response)
except Exception:
pass
return response
def _is_rpm_signed(self, rpm_path) -> bool:
ts = rpm.TransactionSet()
@ -1016,8 +903,8 @@ class KojiMockPackageSet(PackageSetBase):
def get_package_path(self, queue_item):
rpm_info, build_info = queue_item
# Check if this RPM is coming from scratch task. In this case, we already
# know the path.
# Check if this RPM is coming from scratch task.
# In this case, we already know the path.
if "path_from_task" in rpm_info:
return rpm_info["path_from_task"]
@ -1043,261 +930,14 @@ class KojiMockPackageSet(PackageSetBase):
return None
def populate(self, tag, event=None, inherit=True, include_packages=None):
"""Populate the package set with packages from given tag.
:param event: the Koji event to query at (or latest if not given)
:param inherit: whether to enable tag inheritance
:param include_packages: an iterable of tuples (package name, arch) that should
be included, all others are skipped.
"""
result_rpms = []
result_srpms = []
include_packages = set(include_packages or [])
if type(event) is dict:
event = event["id"]
msg = "Getting latest RPMs (tag: %s, event: %s, inherit: %s)" % (
tag,
event,
inherit,
result = super().populate(
tag=tag,
event=event,
inherit=inherit,
include_packages=include_packages,
)
self.log_info("[BEGIN] %s" % msg)
rpms, builds = self.get_latest_rpms(tag, event, inherit=inherit)
extra_rpms, extra_builds = self.get_extra_rpms()
rpms += extra_rpms
builds += extra_builds
extra_builds_by_name = {}
for build_info in extra_builds:
extra_builds_by_name[build_info["name"]] = build_info["build_id"]
builds_by_id = {}
exclude_build_id = []
for build_info in builds:
build_id, build_name = build_info["build_id"], build_info["name"]
if (
build_name in extra_builds_by_name
and build_id != extra_builds_by_name[build_name]
):
exclude_build_id.append(build_id)
else:
builds_by_id.setdefault(build_id, build_info)
# Get extra RPMs from tasks.
rpms += self.get_extra_rpms_from_tasks()
skipped_arches = []
skipped_packages_count = 0
# We need to process binary packages first, and then source packages.
# If we have a list of packages to use, we need to put all source rpms
# names into it. Otherwise if the SRPM name does not occur on the list,
# it would be missing from the package set. Even if it ultimately does
# not end in the compose, we need it to extract ExcludeArch and
# ExclusiveArch for noarch packages.
for rpm_info in itertools.chain(
(rpm for rpm in rpms if not _is_src(rpm)),
(rpm for rpm in rpms if _is_src(rpm)),
):
if rpm_info["build_id"] in exclude_build_id:
continue
if self.arches and rpm_info["arch"] not in self.arches:
if rpm_info["arch"] not in skipped_arches:
self.log_debug("Skipping packages for arch: %s" % rpm_info["arch"])
skipped_arches.append(rpm_info["arch"])
continue
if (
include_packages
and (rpm_info["name"], rpm_info["arch"]) not in include_packages
and rpm_info["arch"] != "src"
):
self.log_debug(
"Skipping %(name)s-%(version)s-%(release)s.%(arch)s" % rpm_info
)
continue
if (
self.populate_only_packages
and self.packages
and rpm_info["name"] not in self.packages
):
skipped_packages_count += 1
continue
build_info = builds_by_id.get(rpm_info["build_id"], None)
if _is_src(rpm_info):
result_srpms.append((rpm_info, build_info))
else:
result_rpms.append((rpm_info, build_info))
if self.populate_only_packages and self.packages:
# Only add the package if we already have some whitelist.
if build_info:
self.packages.add(build_info["name"])
else:
# We have no build info and therefore no Koji package name,
# we can only guess that the Koji package name would be the same
# one as the RPM name.
self.packages.add(rpm_info["name"])
if skipped_packages_count:
self.log_debug(
"Skipped %d packages, not marked as to be "
"included in a compose." % skipped_packages_count
)
result = self.read_packages(result_rpms, result_srpms)
# Check that after reading the packages, every package that is
# included in a compose has the right sigkey.
if self._invalid_sigkey_rpms:
invalid_sigkey_rpms = [
rpm for rpm in self._invalid_sigkey_rpms if rpm["name"] in self.packages
]
if invalid_sigkey_rpms:
self.raise_invalid_sigkeys_exception(invalid_sigkey_rpms)
self.log_info("[DONE ] %s" % msg)
return result
def write_reuse_file(self, compose, include_packages):
"""Write data to files for reusing in future.
:param compose: compose object
:param include_packages: an iterable of tuples (package name, arch) that should
be included.
"""
reuse_file = compose.paths.work.pkgset_reuse_file(self.name)
self.log_info("Writing pkgset reuse file: %s" % reuse_file)
try:
with open(reuse_file, "wb") as f:
pickle.dump(
{
"name": self.name,
"allow_invalid_sigkeys": self._allow_invalid_sigkeys,
"arches": self.arches,
"sigkeys": self.sigkey_ordering,
"packages": self.packages,
"populate_only_packages": self.populate_only_packages,
"rpms_by_arch": self.rpms_by_arch,
"srpms_by_name": self.srpms_by_name,
"extra_builds": self.extra_builds,
"include_packages": include_packages,
},
f,
protocol=pickle.HIGHEST_PROTOCOL,
)
except Exception as e:
self.log_warning("Writing pkgset reuse file failed: %s" % str(e))
def _get_koji_event_from_file(self, event_file):
with open(event_file, "r") as f:
return json.load(f)["id"]
def try_to_reuse(self, compose, tag, inherit=True, include_packages=None):
"""Try to reuse pkgset data of old compose.
:param compose: compose object
:param str tag: koji tag name
:param inherit: whether to enable tag inheritance
:param include_packages: an iterable of tuples (package name, arch) that should
be included.
"""
if not compose.conf["pkgset_allow_reuse"]:
self.log_info("Reusing pkgset data from old compose is disabled.")
return False
self.log_info("Trying to reuse pkgset data of old compose")
if not compose.paths.get_old_compose_topdir():
self.log_debug("No old compose found. Nothing to reuse.")
return False
event_file = os.path.join(
compose.paths.work.topdir(arch="global", create_dir=False), "koji-event"
)
old_event_file = compose.paths.old_compose_path(event_file)
try:
koji_event = self._get_koji_event_from_file(event_file)
old_koji_event = self._get_koji_event_from_file(old_event_file)
except Exception as e:
self.log_debug("Can't read koji event from file: %s" % str(e))
return False
if koji_event != old_koji_event:
self.log_debug(
"Koji event doesn't match, querying changes between event %d and %d"
% (old_koji_event, koji_event)
)
changed = self.koji_proxy.queryHistory(
tables=["tag_listing", "tag_inheritance"],
tag=tag,
afterEvent=min(koji_event, old_koji_event),
beforeEvent=max(koji_event, old_koji_event) + 1,
)
if changed["tag_listing"]:
self.log_debug("Builds under tag %s changed. Can't reuse." % tag)
return False
if changed["tag_inheritance"]:
self.log_debug("Tag inheritance %s changed. Can't reuse." % tag)
return False
if inherit:
inherit_tags = self.koji_proxy.getFullInheritance(tag, koji_event)
for t in inherit_tags:
changed = self.koji_proxy.queryHistory(
tables=["tag_listing", "tag_inheritance"],
tag=t["name"],
afterEvent=min(koji_event, old_koji_event),
beforeEvent=max(koji_event, old_koji_event) + 1,
)
if changed["tag_listing"]:
self.log_debug(
"Builds under inherited tag %s changed. Can't reuse."
% t["name"]
)
return False
if changed["tag_inheritance"]:
self.log_debug("Tag inheritance %s changed. Can't reuse." % tag)
return False
repo_dir = compose.paths.work.pkgset_repo(tag, create_dir=False)
old_repo_dir = compose.paths.old_compose_path(repo_dir)
if not old_repo_dir:
self.log_debug("Can't find old repo dir to reuse.")
return False
old_reuse_file = compose.paths.old_compose_path(
compose.paths.work.pkgset_reuse_file(tag)
)
try:
self.log_debug("Loading reuse file: %s" % old_reuse_file)
reuse_data = self.load_old_file_cache(old_reuse_file)
except Exception as e:
self.log_debug("Failed to load reuse file: %s" % str(e))
return False
if (
reuse_data["allow_invalid_sigkeys"] == self._allow_invalid_sigkeys
and reuse_data["packages"] == self.packages
and reuse_data["populate_only_packages"] == self.populate_only_packages
and reuse_data["extra_builds"] == self.extra_builds
and reuse_data["sigkeys"] == self.sigkey_ordering
and reuse_data["include_packages"] == include_packages
):
self.log_info("Copying repo data for reuse: %s" % old_repo_dir)
copy_all(old_repo_dir, repo_dir)
self.reuse = old_repo_dir
self.rpms_by_arch = reuse_data["rpms_by_arch"]
self.srpms_by_name = reuse_data["srpms_by_name"]
if self.old_file_cache:
self.file_cache = self.old_file_cache
return True
else:
self.log_info("Criteria does not match. Nothing to reuse.")
return False
def _is_src(rpm_info):
"""Check if rpm info object returned by Koji refers to source packages."""

12
pungi/phases/pkgset/sources/source_kojimock.py
View File

@ -200,7 +200,10 @@ class PkgsetSourceKojiMock(pungi.phases.pkgset.source.PkgsetSourceBase):
def __call__(self):
compose = self.compose
self.koji_wrapper = pungi.wrappers.kojiwrapper.KojiMockWrapper(compose)
self.koji_wrapper = pungi.wrappers.kojiwrapper.KojiMockWrapper(
compose,
get_all_arches(compose),
)
# path prefix must contain trailing '/'
path_prefix = self.koji_wrapper.koji_module.config.topdir.rstrip("/") + "/"
package_sets = get_pkgset_from_koji(
@ -622,7 +625,6 @@ def _get_modules_from_koji_tags(
module_builds = filter_by_whitelist(
compose, module_builds, variant_modules, expected_modules
)
# Find the latest builds of all modules. This does following:
# - Sorts the module_builds descending by Koji NVR (which maps to NSV
# for modules). Split release into modular version and context, and
@ -643,7 +645,11 @@ def _get_modules_from_koji_tags(
latest_builds = []
module_builds = sorted(module_builds, key=_key, reverse=True)
for ns, ns_builds in groupby(
module_builds, key=lambda x: ":".join([x["name"], x["version"]])
module_builds, key=lambda x: ":".join([
x["name"],
x["version"],
x['arch'],
])
):
for nsv, nsv_builds in groupby(
ns_builds, key=lambda x: x["release"].split(".")[0]

12
pungi/scripts/create_packages_json.py
View File

@ -341,6 +341,18 @@ class PackagesGenerator:
)
else:
src_package_name = src_package_name[0].name
# TODO: for x86_64 + i686 in one packages.json
# don't remove!
# if package.arch in self.addon_repos[variant_arch]:
# arches = self.addon_repos[variant_arch] + [variant_arch]
# else:
# arches = [variant_arch]
# for arch in arches:
# pkgs_list = packages_json[variant_name][
# arch][src_package_name]
# added_pkg = f'{package_name}.{package_arch}'
# if added_pkg not in pkgs_list:
# pkgs_list.append(added_pkg)
pkgs_list = packages_json[variant_name][
variant_arch][src_package_name]
added_pkg = f'{package_name}.{package_arch}'

80
pungi/wrappers/kojimock.py
View File

@ -43,10 +43,11 @@ class KojiMock:
Class that acts like real koji (for some needed methods)
but uses local storage as data source
"""
def __init__(self, packages_dir, modules_dir):
def __init__(self, packages_dir, modules_dir, all_arches):
self._modules = self._gather_modules(modules_dir)
self._modules_dir = modules_dir
self._packages_dir = packages_dir
self._all_arches = all_arches
@staticmethod
def _gather_modules(modules_dir):
@ -93,6 +94,7 @@ class KojiMock:
'name': module.name,
'id': module.build_id,
'tag_name': tag_name,
'arch': module.arch,
# Following fields are currently not
# used but returned by real koji
# left them here just for reference
@ -246,15 +248,19 @@ class KojiMock:
"""
Get list of builds for module and given module tag name.
"""
module = self._get_module_by_name(tag_name)
path = os.path.join(
self._modules_dir,
module.arch,
tag_name,
)
builds = []
packages = []
modules = self._get_modules_by_name(tag_name)
for module in modules:
if module is None:
raise ValueError('Module %s is not found' % tag_name)
path = os.path.join(
self._modules_dir,
module.arch,
tag_name,
)
builds = [
{
builds.append({
"build_id": module.build_id,
"package_name": module.name,
"nvr": module.nvr,
@ -280,35 +286,33 @@ class KojiMock:
# "volume_id": 0,
# "package_id": 104,
# "owner_id": 6,
}
]
if module is None:
raise ValueError('Module %s is not found' % tag_name)
})
packages = []
if os.path.exists(path):
info = Modulemd.ModuleStream.read_string(open(path).read(), strict=True)
for art in info.get_rpm_artifacts():
data = parse_nvra(art)
packages.append({
"build_id": module.build_id,
"name": data['name'],
"extra": None,
"arch": data['arch'],
"epoch": data['epoch'] or None,
"version": data['version'],
"metadata_only": False,
"release": data['release'],
"id": 262555,
"size": 0
})
else:
raise RuntimeError('Unable to find module %s' % path)
if os.path.exists(path):
info = Modulemd.ModuleStream.read_string(open(path).read(), strict=True)
for art in info.get_rpm_artifacts():
data = parse_nvra(art)
packages.append({
"build_id": module.build_id,
"name": data['name'],
"extra": None,
"arch": data['arch'],
"epoch": data['epoch'] or None,
"version": data['version'],
"metadata_only": False,
"release": data['release'],
"id": 262555,
"size": 0
})
else:
raise RuntimeError('Unable to find module %s' % path)
return builds, packages
def _get_module_by_name(self, tag_name):
for module in self._modules.values():
if module.nvr != tag_name:
continue
return module
return None
def _get_modules_by_name(self, tag_name):
modules = []
for arch in self._all_arches:
for module in self._modules.values():
if module.nvr != tag_name or module.arch != arch:
continue
modules.append(module)
return modules

6
pungi/wrappers/kojiwrapper.py
View File

@ -868,7 +868,8 @@ class KojiWrapper(object):
class KojiMockWrapper(object):
lock = threading.Lock()
def __init__(self, compose):
def __init__(self, compose, all_arches):
self.all_arches = all_arches
self.compose = compose
try:
self.profile = self.compose.conf["koji_profile"]
@ -898,7 +899,8 @@ class KojiMockWrapper(object):
modules_dir=os.path.join(
self.koji_module.config.topdir,
'modules',
)
),
all_arches=self.all_arches,
)

6
tests/test_koji_local_source.py
View File

@ -117,7 +117,11 @@ version: '1'
os.makedirs(os.path.join(PATH_TO_REPOS, os.path.dirname(filepath)), exist_ok=True)
open(os.path.join(PATH_TO_REPOS, filepath), 'w').close()
self._koji = KojiMock(PATH_TO_REPOS, os.path.join(PATH_TO_REPOS, 'modules'))
self._koji = KojiMock(
PATH_TO_REPOS,
os.path.join(PATH_TO_REPOS, 'modules'),
['x86_64', 'noarch', 'i686'],
)
@ddt.data(
[0, {