almalinux
/
pungi
15
1
Fork 0
Code Issues Pull Requests Releases Activity

ALBS-1040: Investigate why Pungi doesn't put modules packages into the final repos #13

Merged
soksanichenko merged 2 commits from ALBS-1040 into al_master 2023-03-16 11:52:03 +00:00
Conversation 0 Commits 2 Files Changed 7 +110 -438
7 changed files with 110 additions and 438 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
pungi/phases/gather/__init__.py
View File

@ -23,6 +23,7 @@ import threading
from kobo.rpmlib import parse_nvra
from kobo.shortcuts import run
from productmd.rpms import Rpms
from pungi.phases.pkgset.common import get_all_arches
from six.moves import cPickle as pickle
try:
@ -652,7 +653,8 @@ def _make_lookaside_repo(compose, variant, arch, pkg_map, package_sets=None):
).koji_module.config.topdir.rstrip("/")
+ "/",
"kojimock": lambda: pungi.wrappers.kojiwrapper.KojiMockWrapper(
compose
compose,
get_all_arches(compose),
).koji_module.config.topdir.rstrip("/")
+ "/",
}

426
pungi/phases/pkgset/pkgsets.py
View File

@ -480,7 +480,8 @@ class KojiPackageSet(PackageSetBase):
response = None
if self.cache_region:
cache_key = "KojiPackageSet.get_latest_rpms_%s_%s_%s" % (
cache_key = "%s.get_latest_rpms_%s_%s_%s" % (
str(self.__class__.__name__),
str(tag),
str(event),
str(inherit),
@ -816,22 +817,22 @@ class KojiPackageSet(PackageSetBase):
return False
class KojiMockPackageSet(PackageSetBase):
class KojiMockPackageSet(KojiPackageSet):
def __init__(
self,
Korulag marked this conversation as resolved
Korulag commented 2023-03-15 09:57:08 +00:00
Review
Copy Link

Why double indentation for the arguments? Previous variant looks fine

Why double indentation for the arguments? Previous variant looks fine
soksanichenko commented 2023-03-15 11:41:50 +00:00
Review
Copy Link

It's pycharm and its rules) In some cases it does double indentation for which constructions.

It's pycharm and its rules) In some cases it does double indentation for which constructions.
name,
koji_wrapper,
sigkey_ordering,
arches=None,
logger=None,
packages=None,
allow_invalid_sigkeys=False,
populate_only_packages=False,
cache_region=None,
extra_builds=None,
extra_tasks=None,
signed_packages_retries=0,
signed_packages_wait=30,
self,
name,
koji_wrapper,
sigkey_ordering,
arches=None,
logger=None,
packages=None,
allow_invalid_sigkeys=False,
populate_only_packages=False,
cache_region=None,
extra_builds=None,
extra_tasks=None,
signed_packages_retries=0,
signed_packages_wait=30,
):
"""
Creates new KojiPackageSet.
@ -865,135 +866,21 @@ class KojiMockPackageSet(PackageSetBase):
and include in the package set. Useful when building testing compose
with RPM scratch builds.
"""
super(KojiMockPackageSet , self).__init__(
super(KojiMockPackageSet, self).__init__(
name,
koji_wrapper=koji_wrapper,
sigkey_ordering=sigkey_ordering,
arches=arches,
logger=logger,
packages=packages,
allow_invalid_sigkeys=allow_invalid_sigkeys,
populate_only_packages=populate_only_packages,
cache_region=cache_region,
extra_builds=extra_builds,
extra_tasks=extra_tasks,
signed_packages_retries=signed_packages_retries,
signed_packages_wait=signed_packages_wait,
)
self.koji_wrapper = koji_wrapper
# Names of packages to look for in the Koji tag.
self.packages = set(packages or [])
self.populate_only_packages = populate_only_packages
self.cache_region = cache_region
self.extra_builds = extra_builds or []
self.extra_tasks = extra_tasks or []
self.reuse = None
self.signed_packages_retries = signed_packages_retries
self.signed_packages_wait = signed_packages_wait
def __getstate__(self):
result = self.__dict__.copy()
del result["koji_wrapper"]
del result["_logger"]
if "cache_region" in result:
del result["cache_region"]
return result
def __setstate__(self, data):
self._logger = None
self.__dict__.update(data)
@property
def koji_proxy(self):
return self.koji_wrapper.koji_proxy
def get_extra_rpms(self):
if not self.extra_builds:
return [], []
rpms = []
builds = []
builds = self.koji_wrapper.retrying_multicall_map(
self.koji_proxy, self.koji_proxy.getBuild, list_of_args=self.extra_builds
)
rpms_in_builds = self.koji_wrapper.retrying_multicall_map(
self.koji_proxy,
self.koji_proxy.listBuildRPMs,
list_of_args=self.extra_builds,
)
rpms = []
for rpms_in_build in rpms_in_builds:
rpms += rpms_in_build
return rpms, builds
def get_extra_rpms_from_tasks(self):
"""
Returns manually constructed RPM infos from the Koji tasks defined
in `self.extra_tasks`.
:rtype: list
:return: List with RPM infos defined as dicts with following keys:
- name, version, release, arch, src - as returned by parse_nvra.
- path_from_task - Full path to RPM on /mnt/koji.
- build_id - Always set to None.
"""
if not self.extra_tasks:
return []
# Get the IDs of children tasks - these are the tasks containing
# the resulting RPMs.
children_tasks = self.koji_wrapper.retrying_multicall_map(
self.koji_proxy,
self.koji_proxy.getTaskChildren,
list_of_args=self.extra_tasks,
)
children_task_ids = []
for tasks in children_tasks:
children_task_ids += [t["id"] for t in tasks]
# Get the results of these children tasks.
results = self.koji_wrapper.retrying_multicall_map(
self.koji_proxy,
self.koji_proxy.getTaskResult,
list_of_args=children_task_ids,
)
rpms = []
for result in results:
rpms += result.get("rpms", [])
rpms += result.get("srpms", [])
rpm_infos = []
for rpm in rpms:
rpm_info = kobo.rpmlib.parse_nvra(os.path.basename(rpm))
rpm_info["path_from_task"] = os.path.join(
self.koji_wrapper.koji_module.pathinfo.work(), rpm
)
rpm_info["build_id"] = None
rpm_infos.append(rpm_info)
return rpm_infos
def get_latest_rpms(self, tag, event, inherit=True):
if not tag:
return [], []
response = None
if self.cache_region:
cache_key = "KojiPackageSet.get_latest_rpms_%s_%s_%s" % (
str(tag),
str(event),
str(inherit),
)
try:
response = self.cache_region.get(cache_key)
except Exception:
pass
if not response:
response = self.koji_proxy.listTaggedRPMS(
tag, event=event, inherit=inherit, latest=True
)
if self.cache_region:
try:
self.cache_region.set(cache_key, response)
except Exception:
pass
return response
def _is_rpm_signed(self, rpm_path) -> bool:
ts = rpm.TransactionSet()
@ -1016,8 +903,8 @@ class KojiMockPackageSet(PackageSetBase):
def get_package_path(self, queue_item):
rpm_info, build_info = queue_item
# Check if this RPM is coming from scratch task. In this case, we already
# know the path.
# Check if this RPM is coming from scratch task.
# In this case, we already know the path.
if "path_from_task" in rpm_info:
return rpm_info["path_from_task"]
@ -1043,261 +930,14 @@ class KojiMockPackageSet(PackageSetBase):
return None
def populate(self, tag, event=None, inherit=True, include_packages=None):
"""Populate the package set with packages from given tag.
:param event: the Koji event to query at (or latest if not given)
:param inherit: whether to enable tag inheritance
:param include_packages: an iterable of tuples (package name, arch) that should
be included, all others are skipped.
"""
result_rpms = []
result_srpms = []
include_packages = set(include_packages or [])
if type(event) is dict:
event = event["id"]
msg = "Getting latest RPMs (tag: %s, event: %s, inherit: %s)" % (
tag,
event,
inherit,
result = super().populate(
tag=tag,
event=event,
inherit=inherit,
include_packages=include_packages,
)
self.log_info("[BEGIN] %s" % msg)
rpms, builds = self.get_latest_rpms(tag, event, inherit=inherit)
extra_rpms, extra_builds = self.get_extra_rpms()
rpms += extra_rpms
builds += extra_builds
extra_builds_by_name = {}
for build_info in extra_builds:
extra_builds_by_name[build_info["name"]] = build_info["build_id"]
builds_by_id = {}
exclude_build_id = []
for build_info in builds:
build_id, build_name = build_info["build_id"], build_info["name"]
if (
build_name in extra_builds_by_name
and build_id != extra_builds_by_name[build_name]
):
exclude_build_id.append(build_id)
else:
builds_by_id.setdefault(build_id, build_info)
# Get extra RPMs from tasks.
rpms += self.get_extra_rpms_from_tasks()
skipped_arches = []
skipped_packages_count = 0
# We need to process binary packages first, and then source packages.
# If we have a list of packages to use, we need to put all source rpms
# names into it. Otherwise if the SRPM name does not occur on the list,
# it would be missing from the package set. Even if it ultimately does
# not end in the compose, we need it to extract ExcludeArch and
# ExclusiveArch for noarch packages.
for rpm_info in itertools.chain(
(rpm for rpm in rpms if not _is_src(rpm)),
(rpm for rpm in rpms if _is_src(rpm)),
):
if rpm_info["build_id"] in exclude_build_id:
continue
if self.arches and rpm_info["arch"] not in self.arches:
if rpm_info["arch"] not in skipped_arches:
self.log_debug("Skipping packages for arch: %s" % rpm_info["arch"])
skipped_arches.append(rpm_info["arch"])
continue
if (
include_packages
and (rpm_info["name"], rpm_info["arch"]) not in include_packages
and rpm_info["arch"] != "src"
):
self.log_debug(
"Skipping %(name)s-%(version)s-%(release)s.%(arch)s" % rpm_info
)
continue
if (
self.populate_only_packages
and self.packages
and rpm_info["name"] not in self.packages
):
skipped_packages_count += 1
continue
build_info = builds_by_id.get(rpm_info["build_id"], None)
if _is_src(rpm_info):
result_srpms.append((rpm_info, build_info))
else:
result_rpms.append((rpm_info, build_info))
if self.populate_only_packages and self.packages:
# Only add the package if we already have some whitelist.
if build_info:
self.packages.add(build_info["name"])
else:
# We have no build info and therefore no Koji package name,
# we can only guess that the Koji package name would be the same
# one as the RPM name.
self.packages.add(rpm_info["name"])
if skipped_packages_count:
self.log_debug(
"Skipped %d packages, not marked as to be "
"included in a compose." % skipped_packages_count
)
result = self.read_packages(result_rpms, result_srpms)
# Check that after reading the packages, every package that is
# included in a compose has the right sigkey.
if self._invalid_sigkey_rpms:
invalid_sigkey_rpms = [
rpm for rpm in self._invalid_sigkey_rpms if rpm["name"] in self.packages
]
if invalid_sigkey_rpms:
self.raise_invalid_sigkeys_exception(invalid_sigkey_rpms)
self.log_info("[DONE ] %s" % msg)
return result
def write_reuse_file(self, compose, include_packages):
"""Write data to files for reusing in future.
:param compose: compose object
:param include_packages: an iterable of tuples (package name, arch) that should
be included.
"""
reuse_file = compose.paths.work.pkgset_reuse_file(self.name)
self.log_info("Writing pkgset reuse file: %s" % reuse_file)
try:
with open(reuse_file, "wb") as f:
pickle.dump(
{
"name": self.name,
"allow_invalid_sigkeys": self._allow_invalid_sigkeys,
"arches": self.arches,
"sigkeys": self.sigkey_ordering,
"packages": self.packages,
"populate_only_packages": self.populate_only_packages,
"rpms_by_arch": self.rpms_by_arch,
"srpms_by_name": self.srpms_by_name,
"extra_builds": self.extra_builds,
"include_packages": include_packages,
},
f,
protocol=pickle.HIGHEST_PROTOCOL,
)
except Exception as e:
self.log_warning("Writing pkgset reuse file failed: %s" % str(e))
def _get_koji_event_from_file(self, event_file):
with open(event_file, "r") as f:
return json.load(f)["id"]
def try_to_reuse(self, compose, tag, inherit=True, include_packages=None):
"""Try to reuse pkgset data of old compose.
:param compose: compose object
:param str tag: koji tag name
:param inherit: whether to enable tag inheritance
:param include_packages: an iterable of tuples (package name, arch) that should
be included.
"""
if not compose.conf["pkgset_allow_reuse"]:
self.log_info("Reusing pkgset data from old compose is disabled.")
return False
self.log_info("Trying to reuse pkgset data of old compose")
if not compose.paths.get_old_compose_topdir():
self.log_debug("No old compose found. Nothing to reuse.")
return False
event_file = os.path.join(
compose.paths.work.topdir(arch="global", create_dir=False), "koji-event"
)
old_event_file = compose.paths.old_compose_path(event_file)
try:
koji_event = self._get_koji_event_from_file(event_file)
old_koji_event = self._get_koji_event_from_file(old_event_file)
except Exception as e:
self.log_debug("Can't read koji event from file: %s" % str(e))
return False
if koji_event != old_koji_event:
self.log_debug(
"Koji event doesn't match, querying changes between event %d and %d"
% (old_koji_event, koji_event)
)
changed = self.koji_proxy.queryHistory(
tables=["tag_listing", "tag_inheritance"],
tag=tag,
afterEvent=min(koji_event, old_koji_event),
beforeEvent=max(koji_event, old_koji_event) + 1,
)
if changed["tag_listing"]:
self.log_debug("Builds under tag %s changed. Can't reuse." % tag)
return False
if changed["tag_inheritance"]:
self.log_debug("Tag inheritance %s changed. Can't reuse." % tag)
return False
if inherit:
inherit_tags = self.koji_proxy.getFullInheritance(tag, koji_event)
for t in inherit_tags:
changed = self.koji_proxy.queryHistory(
tables=["tag_listing", "tag_inheritance"],
tag=t["name"],
afterEvent=min(koji_event, old_koji_event),
beforeEvent=max(koji_event, old_koji_event) + 1,
)
if changed["tag_listing"]:
self.log_debug(
"Builds under inherited tag %s changed. Can't reuse."
% t["name"]
)
return False
if changed["tag_inheritance"]:
self.log_debug("Tag inheritance %s changed. Can't reuse." % tag)
return False
repo_dir = compose.paths.work.pkgset_repo(tag, create_dir=False)
old_repo_dir = compose.paths.old_compose_path(repo_dir)
if not old_repo_dir:
self.log_debug("Can't find old repo dir to reuse.")
return False
old_reuse_file = compose.paths.old_compose_path(
compose.paths.work.pkgset_reuse_file(tag)
)
try:
self.log_debug("Loading reuse file: %s" % old_reuse_file)
reuse_data = self.load_old_file_cache(old_reuse_file)
except Exception as e:
self.log_debug("Failed to load reuse file: %s" % str(e))
return False
if (
reuse_data["allow_invalid_sigkeys"] == self._allow_invalid_sigkeys
and reuse_data["packages"] == self.packages
and reuse_data["populate_only_packages"] == self.populate_only_packages
and reuse_data["extra_builds"] == self.extra_builds
and reuse_data["sigkeys"] == self.sigkey_ordering
and reuse_data["include_packages"] == include_packages
):
self.log_info("Copying repo data for reuse: %s" % old_repo_dir)
copy_all(old_repo_dir, repo_dir)
self.reuse = old_repo_dir
self.rpms_by_arch = reuse_data["rpms_by_arch"]
self.srpms_by_name = reuse_data["srpms_by_name"]
if self.old_file_cache:
self.file_cache = self.old_file_cache
return True
else:
self.log_info("Criteria does not match. Nothing to reuse.")
return False
def _is_src(rpm_info):
"""Check if rpm info object returned by Koji refers to source packages."""

12
pungi/phases/pkgset/sources/source_kojimock.py
View File

@ -200,7 +200,10 @@ class PkgsetSourceKojiMock(pungi.phases.pkgset.source.PkgsetSourceBase):
def __call__(self):
compose = self.compose
self.koji_wrapper = pungi.wrappers.kojiwrapper.KojiMockWrapper(compose)
self.koji_wrapper = pungi.wrappers.kojiwrapper.KojiMockWrapper(
compose,
get_all_arches(compose),
)
# path prefix must contain trailing '/'
path_prefix = self.koji_wrapper.koji_module.config.topdir.rstrip("/") + "/"
package_sets = get_pkgset_from_koji(
@ -622,7 +625,6 @@ def _get_modules_from_koji_tags(
module_builds = filter_by_whitelist(
compose, module_builds, variant_modules, expected_modules
)
# Find the latest builds of all modules. This does following:
# - Sorts the module_builds descending by Koji NVR (which maps to NSV
# for modules). Split release into modular version and context, and
@ -643,7 +645,11 @@ def _get_modules_from_koji_tags(
latest_builds = []
module_builds = sorted(module_builds, key=_key, reverse=True)
for ns, ns_builds in groupby(
module_builds, key=lambda x: ":".join([x["name"], x["version"]])
module_builds, key=lambda x: ":".join([
x["name"],
x["version"],
x['arch'],
])
):
for nsv, nsv_builds in groupby(
ns_builds, key=lambda x: x["release"].split(".")[0]

12
pungi/scripts/create_packages_json.py
View File

@ -341,6 +341,18 @@ class PackagesGenerator:
)
else:
src_package_name = src_package_name[0].name
# TODO: for x86_64 + i686 in one packages.json
# don't remove!
# if package.arch in self.addon_repos[variant_arch]:
# arches = self.addon_repos[variant_arch] + [variant_arch]
# else:
# arches = [variant_arch]
# for arch in arches:
# pkgs_list = packages_json[variant_name][
# arch][src_package_name]
# added_pkg = f'{package_name}.{package_arch}'
# if added_pkg not in pkgs_list:
# pkgs_list.append(added_pkg)
pkgs_list = packages_json[variant_name][
variant_arch][src_package_name]
added_pkg = f'{package_name}.{package_arch}'

80
pungi/wrappers/kojimock.py
View File

@ -43,10 +43,11 @@ class KojiMock:
Class that acts like real koji (for some needed methods)
but uses local storage as data source
"""
def __init__(self, packages_dir, modules_dir):
def __init__(self, packages_dir, modules_dir, all_arches):
self._modules = self._gather_modules(modules_dir)
self._modules_dir = modules_dir
self._packages_dir = packages_dir
self._all_arches = all_arches
@staticmethod
def _gather_modules(modules_dir):
@ -93,6 +94,7 @@ class KojiMock:
'name': module.name,
'id': module.build_id,
'tag_name': tag_name,
'arch': module.arch,
# Following fields are currently not
# used but returned by real koji
# left them here just for reference
@ -246,15 +248,19 @@ class KojiMock:
"""
Get list of builds for module and given module tag name.
"""
module = self._get_module_by_name(tag_name)
path = os.path.join(
self._modules_dir,
module.arch,
tag_name,
)
builds = []
packages = []
modules = self._get_modules_by_name(tag_name)
for module in modules:
if module is None:
raise ValueError('Module %s is not found' % tag_name)
path = os.path.join(
self._modules_dir,
module.arch,
tag_name,
)
builds = [
{
builds.append({
"build_id": module.build_id,
"package_name": module.name,
"nvr": module.nvr,
@ -280,35 +286,33 @@ class KojiMock:
# "volume_id": 0,
# "package_id": 104,
# "owner_id": 6,
}
]
if module is None:
raise ValueError('Module %s is not found' % tag_name)
})
packages = []
if os.path.exists(path):
info = Modulemd.ModuleStream.read_string(open(path).read(), strict=True)
for art in info.get_rpm_artifacts():
data = parse_nvra(art)
packages.append({
"build_id": module.build_id,
"name": data['name'],
"extra": None,
"arch": data['arch'],
"epoch": data['epoch'] or None,
"version": data['version'],
"metadata_only": False,
"release": data['release'],
"id": 262555,
"size": 0
})
else:
raise RuntimeError('Unable to find module %s' % path)
if os.path.exists(path):
info = Modulemd.ModuleStream.read_string(open(path).read(), strict=True)
for art in info.get_rpm_artifacts():
data = parse_nvra(art)
packages.append({
"build_id": module.build_id,
"name": data['name'],
"extra": None,
"arch": data['arch'],
"epoch": data['epoch'] or None,
"version": data['version'],
"metadata_only": False,
"release": data['release'],
"id": 262555,
"size": 0
})
else:
raise RuntimeError('Unable to find module %s' % path)
return builds, packages
def _get_module_by_name(self, tag_name):
for module in self._modules.values():
if module.nvr != tag_name:
continue
return module
return None
def _get_modules_by_name(self, tag_name):
modules = []
for arch in self._all_arches:
for module in self._modules.values():
if module.nvr != tag_name or module.arch != arch:
continue
modules.append(module)
return modules

6
pungi/wrappers/kojiwrapper.py
View File

@ -868,7 +868,8 @@ class KojiWrapper(object):
class KojiMockWrapper(object):
lock = threading.Lock()
def __init__(self, compose):
def __init__(self, compose, all_arches):
self.all_arches = all_arches
self.compose = compose
try:
self.profile = self.compose.conf["koji_profile"]
@ -898,7 +899,8 @@ class KojiMockWrapper(object):
modules_dir=os.path.join(
self.koji_module.config.topdir,
'modules',
)
),
all_arches=self.all_arches,
)

8
tests/test_koji_local_source.py
View File

@ -117,7 +117,11 @@ version: '1'
os.makedirs(os.path.join(PATH_TO_REPOS, os.path.dirname(filepath)), exist_ok=True)
open(os.path.join(PATH_TO_REPOS, filepath), 'w').close()
self._koji = KojiMock(PATH_TO_REPOS, os.path.join(PATH_TO_REPOS, 'modules'))
self._koji = KojiMock(
PATH_TO_REPOS,
os.path.join(PATH_TO_REPOS, 'modules'),
['x86_64', 'noarch', 'i686'],
)
@ddt.data(
[0, {
@ -331,6 +335,7 @@ version: '1'
result = self._koji.listTagged('dist-c8-module-compose')
self.assertEqual([
{
'arch': 'x86_64',
'build_id': 0,
'id': 0,
'name': 'javapackages-tools',
@ -342,6 +347,7 @@ version: '1'
'version': '201801'
},
{
'arch': 'x86_64',
'build_id': 1,
'id': 1,
'name': 'mariadb-devel',