ALBS-1040: Investigate why Pungi doesn't put modules packages into the final repos #13
@ -23,6 +23,7 @@ import threading
|
|||||||
from kobo.rpmlib import parse_nvra
|
from kobo.rpmlib import parse_nvra
|
||||||
from kobo.shortcuts import run
|
from kobo.shortcuts import run
|
||||||
from productmd.rpms import Rpms
|
from productmd.rpms import Rpms
|
||||||
|
from pungi.phases.pkgset.common import get_all_arches
|
||||||
from six.moves import cPickle as pickle
|
from six.moves import cPickle as pickle
|
||||||
|
|
||||||
try:
|
try:
|
||||||
@ -652,7 +653,8 @@ def _make_lookaside_repo(compose, variant, arch, pkg_map, package_sets=None):
|
|||||||
).koji_module.config.topdir.rstrip("/")
|
).koji_module.config.topdir.rstrip("/")
|
||||||
+ "/",
|
+ "/",
|
||||||
"kojimock": lambda: pungi.wrappers.kojiwrapper.KojiMockWrapper(
|
"kojimock": lambda: pungi.wrappers.kojiwrapper.KojiMockWrapper(
|
||||||
compose
|
compose,
|
||||||
|
get_all_arches(compose),
|
||||||
).koji_module.config.topdir.rstrip("/")
|
).koji_module.config.topdir.rstrip("/")
|
||||||
+ "/",
|
+ "/",
|
||||||
}
|
}
|
||||||
|
@ -480,7 +480,8 @@ class KojiPackageSet(PackageSetBase):
|
|||||||
|
|
||||||
response = None
|
response = None
|
||||||
if self.cache_region:
|
if self.cache_region:
|
||||||
cache_key = "KojiPackageSet.get_latest_rpms_%s_%s_%s" % (
|
cache_key = "%s.get_latest_rpms_%s_%s_%s" % (
|
||||||
|
str(self.__class__.__name__),
|
||||||
str(tag),
|
str(tag),
|
||||||
str(event),
|
str(event),
|
||||||
str(inherit),
|
str(inherit),
|
||||||
@ -816,22 +817,22 @@ class KojiPackageSet(PackageSetBase):
|
|||||||
return False
|
return False
|
||||||
|
|
||||||
|
|
||||||
class KojiMockPackageSet(PackageSetBase):
|
class KojiMockPackageSet(KojiPackageSet):
|
||||||
def __init__(
|
def __init__(
|
||||||
self,
|
self,
|
||||||
Korulag marked this conversation as resolved
|
|||||||
name,
|
name,
|
||||||
koji_wrapper,
|
koji_wrapper,
|
||||||
sigkey_ordering,
|
sigkey_ordering,
|
||||||
arches=None,
|
arches=None,
|
||||||
logger=None,
|
logger=None,
|
||||||
packages=None,
|
packages=None,
|
||||||
allow_invalid_sigkeys=False,
|
allow_invalid_sigkeys=False,
|
||||||
populate_only_packages=False,
|
populate_only_packages=False,
|
||||||
cache_region=None,
|
cache_region=None,
|
||||||
extra_builds=None,
|
extra_builds=None,
|
||||||
extra_tasks=None,
|
extra_tasks=None,
|
||||||
signed_packages_retries=0,
|
signed_packages_retries=0,
|
||||||
signed_packages_wait=30,
|
signed_packages_wait=30,
|
||||||
):
|
):
|
||||||
"""
|
"""
|
||||||
Creates new KojiPackageSet.
|
Creates new KojiPackageSet.
|
||||||
@ -865,135 +866,21 @@ class KojiMockPackageSet(PackageSetBase):
|
|||||||
and include in the package set. Useful when building testing compose
|
and include in the package set. Useful when building testing compose
|
||||||
with RPM scratch builds.
|
with RPM scratch builds.
|
||||||
"""
|
"""
|
||||||
super(KojiMockPackageSet , self).__init__(
|
super(KojiMockPackageSet, self).__init__(
|
||||||
name,
|
name,
|
||||||
|
koji_wrapper=koji_wrapper,
|
||||||
sigkey_ordering=sigkey_ordering,
|
sigkey_ordering=sigkey_ordering,
|
||||||
arches=arches,
|
arches=arches,
|
||||||
logger=logger,
|
logger=logger,
|
||||||
|
packages=packages,
|
||||||
allow_invalid_sigkeys=allow_invalid_sigkeys,
|
allow_invalid_sigkeys=allow_invalid_sigkeys,
|
||||||
|
populate_only_packages=populate_only_packages,
|
||||||
|
cache_region=cache_region,
|
||||||
|
extra_builds=extra_builds,
|
||||||
|
extra_tasks=extra_tasks,
|
||||||
|
signed_packages_retries=signed_packages_retries,
|
||||||
|
signed_packages_wait=signed_packages_wait,
|
||||||
)
|
)
|
||||||
self.koji_wrapper = koji_wrapper
|
|
||||||
# Names of packages to look for in the Koji tag.
|
|
||||||
self.packages = set(packages or [])
|
|
||||||
self.populate_only_packages = populate_only_packages
|
|
||||||
self.cache_region = cache_region
|
|
||||||
self.extra_builds = extra_builds or []
|
|
||||||
self.extra_tasks = extra_tasks or []
|
|
||||||
self.reuse = None
|
|
||||||
self.signed_packages_retries = signed_packages_retries
|
|
||||||
self.signed_packages_wait = signed_packages_wait
|
|
||||||
|
|
||||||
def __getstate__(self):
|
|
||||||
result = self.__dict__.copy()
|
|
||||||
del result["koji_wrapper"]
|
|
||||||
del result["_logger"]
|
|
||||||
if "cache_region" in result:
|
|
||||||
del result["cache_region"]
|
|
||||||
return result
|
|
||||||
|
|
||||||
def __setstate__(self, data):
|
|
||||||
self._logger = None
|
|
||||||
self.__dict__.update(data)
|
|
||||||
|
|
||||||
@property
|
|
||||||
def koji_proxy(self):
|
|
||||||
return self.koji_wrapper.koji_proxy
|
|
||||||
|
|
||||||
def get_extra_rpms(self):
|
|
||||||
if not self.extra_builds:
|
|
||||||
return [], []
|
|
||||||
|
|
||||||
rpms = []
|
|
||||||
builds = []
|
|
||||||
|
|
||||||
builds = self.koji_wrapper.retrying_multicall_map(
|
|
||||||
self.koji_proxy, self.koji_proxy.getBuild, list_of_args=self.extra_builds
|
|
||||||
)
|
|
||||||
rpms_in_builds = self.koji_wrapper.retrying_multicall_map(
|
|
||||||
self.koji_proxy,
|
|
||||||
self.koji_proxy.listBuildRPMs,
|
|
||||||
list_of_args=self.extra_builds,
|
|
||||||
)
|
|
||||||
|
|
||||||
rpms = []
|
|
||||||
for rpms_in_build in rpms_in_builds:
|
|
||||||
rpms += rpms_in_build
|
|
||||||
return rpms, builds
|
|
||||||
|
|
||||||
def get_extra_rpms_from_tasks(self):
|
|
||||||
"""
|
|
||||||
Returns manually constructed RPM infos from the Koji tasks defined
|
|
||||||
in `self.extra_tasks`.
|
|
||||||
|
|
||||||
:rtype: list
|
|
||||||
:return: List with RPM infos defined as dicts with following keys:
|
|
||||||
- name, version, release, arch, src - as returned by parse_nvra.
|
|
||||||
- path_from_task - Full path to RPM on /mnt/koji.
|
|
||||||
- build_id - Always set to None.
|
|
||||||
"""
|
|
||||||
if not self.extra_tasks:
|
|
||||||
return []
|
|
||||||
|
|
||||||
# Get the IDs of children tasks - these are the tasks containing
|
|
||||||
# the resulting RPMs.
|
|
||||||
children_tasks = self.koji_wrapper.retrying_multicall_map(
|
|
||||||
self.koji_proxy,
|
|
||||||
self.koji_proxy.getTaskChildren,
|
|
||||||
list_of_args=self.extra_tasks,
|
|
||||||
)
|
|
||||||
children_task_ids = []
|
|
||||||
for tasks in children_tasks:
|
|
||||||
children_task_ids += [t["id"] for t in tasks]
|
|
||||||
|
|
||||||
# Get the results of these children tasks.
|
|
||||||
results = self.koji_wrapper.retrying_multicall_map(
|
|
||||||
self.koji_proxy,
|
|
||||||
self.koji_proxy.getTaskResult,
|
|
||||||
list_of_args=children_task_ids,
|
|
||||||
)
|
|
||||||
rpms = []
|
|
||||||
for result in results:
|
|
||||||
rpms += result.get("rpms", [])
|
|
||||||
rpms += result.get("srpms", [])
|
|
||||||
|
|
||||||
rpm_infos = []
|
|
||||||
for rpm in rpms:
|
|
||||||
rpm_info = kobo.rpmlib.parse_nvra(os.path.basename(rpm))
|
|
||||||
rpm_info["path_from_task"] = os.path.join(
|
|
||||||
self.koji_wrapper.koji_module.pathinfo.work(), rpm
|
|
||||||
)
|
|
||||||
rpm_info["build_id"] = None
|
|
||||||
rpm_infos.append(rpm_info)
|
|
||||||
|
|
||||||
return rpm_infos
|
|
||||||
|
|
||||||
def get_latest_rpms(self, tag, event, inherit=True):
|
|
||||||
if not tag:
|
|
||||||
return [], []
|
|
||||||
|
|
||||||
response = None
|
|
||||||
if self.cache_region:
|
|
||||||
cache_key = "KojiPackageSet.get_latest_rpms_%s_%s_%s" % (
|
|
||||||
str(tag),
|
|
||||||
str(event),
|
|
||||||
str(inherit),
|
|
||||||
)
|
|
||||||
try:
|
|
||||||
response = self.cache_region.get(cache_key)
|
|
||||||
except Exception:
|
|
||||||
pass
|
|
||||||
|
|
||||||
if not response:
|
|
||||||
response = self.koji_proxy.listTaggedRPMS(
|
|
||||||
tag, event=event, inherit=inherit, latest=True
|
|
||||||
)
|
|
||||||
if self.cache_region:
|
|
||||||
try:
|
|
||||||
self.cache_region.set(cache_key, response)
|
|
||||||
except Exception:
|
|
||||||
pass
|
|
||||||
|
|
||||||
return response
|
|
||||||
|
|
||||||
def _is_rpm_signed(self, rpm_path) -> bool:
|
def _is_rpm_signed(self, rpm_path) -> bool:
|
||||||
ts = rpm.TransactionSet()
|
ts = rpm.TransactionSet()
|
||||||
@ -1016,8 +903,8 @@ class KojiMockPackageSet(PackageSetBase):
|
|||||||
def get_package_path(self, queue_item):
|
def get_package_path(self, queue_item):
|
||||||
rpm_info, build_info = queue_item
|
rpm_info, build_info = queue_item
|
||||||
|
|
||||||
# Check if this RPM is coming from scratch task. In this case, we already
|
# Check if this RPM is coming from scratch task.
|
||||||
# know the path.
|
# In this case, we already know the path.
|
||||||
if "path_from_task" in rpm_info:
|
if "path_from_task" in rpm_info:
|
||||||
return rpm_info["path_from_task"]
|
return rpm_info["path_from_task"]
|
||||||
|
|
||||||
@ -1043,261 +930,14 @@ class KojiMockPackageSet(PackageSetBase):
|
|||||||
return None
|
return None
|
||||||
|
|
||||||
def populate(self, tag, event=None, inherit=True, include_packages=None):
|
def populate(self, tag, event=None, inherit=True, include_packages=None):
|
||||||
"""Populate the package set with packages from given tag.
|
result = super().populate(
|
||||||
|
tag=tag,
|
||||||
:param event: the Koji event to query at (or latest if not given)
|
event=event,
|
||||||
:param inherit: whether to enable tag inheritance
|
inherit=inherit,
|
||||||
:param include_packages: an iterable of tuples (package name, arch) that should
|
include_packages=include_packages,
|
||||||
be included, all others are skipped.
|
|
||||||
"""
|
|
||||||
result_rpms = []
|
|
||||||
result_srpms = []
|
|
||||||
include_packages = set(include_packages or [])
|
|
||||||
|
|
||||||
if type(event) is dict:
|
|
||||||
event = event["id"]
|
|
||||||
|
|
||||||
msg = "Getting latest RPMs (tag: %s, event: %s, inherit: %s)" % (
|
|
||||||
tag,
|
|
||||||
event,
|
|
||||||
inherit,
|
|
||||||
)
|
)
|
||||||
self.log_info("[BEGIN] %s" % msg)
|
|
||||||
rpms, builds = self.get_latest_rpms(tag, event, inherit=inherit)
|
|
||||||
extra_rpms, extra_builds = self.get_extra_rpms()
|
|
||||||
rpms += extra_rpms
|
|
||||||
builds += extra_builds
|
|
||||||
|
|
||||||
extra_builds_by_name = {}
|
|
||||||
for build_info in extra_builds:
|
|
||||||
extra_builds_by_name[build_info["name"]] = build_info["build_id"]
|
|
||||||
|
|
||||||
builds_by_id = {}
|
|
||||||
exclude_build_id = []
|
|
||||||
for build_info in builds:
|
|
||||||
build_id, build_name = build_info["build_id"], build_info["name"]
|
|
||||||
if (
|
|
||||||
build_name in extra_builds_by_name
|
|
||||||
and build_id != extra_builds_by_name[build_name]
|
|
||||||
):
|
|
||||||
exclude_build_id.append(build_id)
|
|
||||||
else:
|
|
||||||
builds_by_id.setdefault(build_id, build_info)
|
|
||||||
|
|
||||||
# Get extra RPMs from tasks.
|
|
||||||
rpms += self.get_extra_rpms_from_tasks()
|
|
||||||
|
|
||||||
skipped_arches = []
|
|
||||||
skipped_packages_count = 0
|
|
||||||
# We need to process binary packages first, and then source packages.
|
|
||||||
# If we have a list of packages to use, we need to put all source rpms
|
|
||||||
# names into it. Otherwise if the SRPM name does not occur on the list,
|
|
||||||
# it would be missing from the package set. Even if it ultimately does
|
|
||||||
# not end in the compose, we need it to extract ExcludeArch and
|
|
||||||
# ExclusiveArch for noarch packages.
|
|
||||||
for rpm_info in itertools.chain(
|
|
||||||
(rpm for rpm in rpms if not _is_src(rpm)),
|
|
||||||
(rpm for rpm in rpms if _is_src(rpm)),
|
|
||||||
):
|
|
||||||
if rpm_info["build_id"] in exclude_build_id:
|
|
||||||
continue
|
|
||||||
|
|
||||||
if self.arches and rpm_info["arch"] not in self.arches:
|
|
||||||
if rpm_info["arch"] not in skipped_arches:
|
|
||||||
self.log_debug("Skipping packages for arch: %s" % rpm_info["arch"])
|
|
||||||
skipped_arches.append(rpm_info["arch"])
|
|
||||||
continue
|
|
||||||
|
|
||||||
if (
|
|
||||||
include_packages
|
|
||||||
and (rpm_info["name"], rpm_info["arch"]) not in include_packages
|
|
||||||
and rpm_info["arch"] != "src"
|
|
||||||
):
|
|
||||||
self.log_debug(
|
|
||||||
"Skipping %(name)s-%(version)s-%(release)s.%(arch)s" % rpm_info
|
|
||||||
)
|
|
||||||
continue
|
|
||||||
|
|
||||||
if (
|
|
||||||
self.populate_only_packages
|
|
||||||
and self.packages
|
|
||||||
and rpm_info["name"] not in self.packages
|
|
||||||
):
|
|
||||||
skipped_packages_count += 1
|
|
||||||
continue
|
|
||||||
|
|
||||||
build_info = builds_by_id.get(rpm_info["build_id"], None)
|
|
||||||
if _is_src(rpm_info):
|
|
||||||
result_srpms.append((rpm_info, build_info))
|
|
||||||
else:
|
|
||||||
result_rpms.append((rpm_info, build_info))
|
|
||||||
if self.populate_only_packages and self.packages:
|
|
||||||
# Only add the package if we already have some whitelist.
|
|
||||||
if build_info:
|
|
||||||
self.packages.add(build_info["name"])
|
|
||||||
else:
|
|
||||||
# We have no build info and therefore no Koji package name,
|
|
||||||
# we can only guess that the Koji package name would be the same
|
|
||||||
# one as the RPM name.
|
|
||||||
self.packages.add(rpm_info["name"])
|
|
||||||
|
|
||||||
if skipped_packages_count:
|
|
||||||
self.log_debug(
|
|
||||||
"Skipped %d packages, not marked as to be "
|
|
||||||
"included in a compose." % skipped_packages_count
|
|
||||||
)
|
|
||||||
|
|
||||||
result = self.read_packages(result_rpms, result_srpms)
|
|
||||||
|
|
||||||
# Check that after reading the packages, every package that is
|
|
||||||
# included in a compose has the right sigkey.
|
|
||||||
if self._invalid_sigkey_rpms:
|
|
||||||
invalid_sigkey_rpms = [
|
|
||||||
rpm for rpm in self._invalid_sigkey_rpms if rpm["name"] in self.packages
|
|
||||||
]
|
|
||||||
if invalid_sigkey_rpms:
|
|
||||||
self.raise_invalid_sigkeys_exception(invalid_sigkey_rpms)
|
|
||||||
|
|
||||||
self.log_info("[DONE ] %s" % msg)
|
|
||||||
return result
|
return result
|
||||||
|
|
||||||
def write_reuse_file(self, compose, include_packages):
|
|
||||||
"""Write data to files for reusing in future.
|
|
||||||
|
|
||||||
:param compose: compose object
|
|
||||||
:param include_packages: an iterable of tuples (package name, arch) that should
|
|
||||||
be included.
|
|
||||||
"""
|
|
||||||
reuse_file = compose.paths.work.pkgset_reuse_file(self.name)
|
|
||||||
self.log_info("Writing pkgset reuse file: %s" % reuse_file)
|
|
||||||
try:
|
|
||||||
with open(reuse_file, "wb") as f:
|
|
||||||
pickle.dump(
|
|
||||||
{
|
|
||||||
"name": self.name,
|
|
||||||
"allow_invalid_sigkeys": self._allow_invalid_sigkeys,
|
|
||||||
"arches": self.arches,
|
|
||||||
"sigkeys": self.sigkey_ordering,
|
|
||||||
"packages": self.packages,
|
|
||||||
"populate_only_packages": self.populate_only_packages,
|
|
||||||
"rpms_by_arch": self.rpms_by_arch,
|
|
||||||
"srpms_by_name": self.srpms_by_name,
|
|
||||||
"extra_builds": self.extra_builds,
|
|
||||||
"include_packages": include_packages,
|
|
||||||
},
|
|
||||||
f,
|
|
||||||
protocol=pickle.HIGHEST_PROTOCOL,
|
|
||||||
)
|
|
||||||
except Exception as e:
|
|
||||||
self.log_warning("Writing pkgset reuse file failed: %s" % str(e))
|
|
||||||
|
|
||||||
def _get_koji_event_from_file(self, event_file):
|
|
||||||
with open(event_file, "r") as f:
|
|
||||||
return json.load(f)["id"]
|
|
||||||
|
|
||||||
def try_to_reuse(self, compose, tag, inherit=True, include_packages=None):
|
|
||||||
"""Try to reuse pkgset data of old compose.
|
|
||||||
:param compose: compose object
|
|
||||||
:param str tag: koji tag name
|
|
||||||
:param inherit: whether to enable tag inheritance
|
|
||||||
:param include_packages: an iterable of tuples (package name, arch) that should
|
|
||||||
be included.
|
|
||||||
"""
|
|
||||||
if not compose.conf["pkgset_allow_reuse"]:
|
|
||||||
self.log_info("Reusing pkgset data from old compose is disabled.")
|
|
||||||
return False
|
|
||||||
|
|
||||||
self.log_info("Trying to reuse pkgset data of old compose")
|
|
||||||
if not compose.paths.get_old_compose_topdir():
|
|
||||||
self.log_debug("No old compose found. Nothing to reuse.")
|
|
||||||
return False
|
|
||||||
|
|
||||||
event_file = os.path.join(
|
|
||||||
compose.paths.work.topdir(arch="global", create_dir=False), "koji-event"
|
|
||||||
)
|
|
||||||
old_event_file = compose.paths.old_compose_path(event_file)
|
|
||||||
|
|
||||||
try:
|
|
||||||
koji_event = self._get_koji_event_from_file(event_file)
|
|
||||||
old_koji_event = self._get_koji_event_from_file(old_event_file)
|
|
||||||
except Exception as e:
|
|
||||||
self.log_debug("Can't read koji event from file: %s" % str(e))
|
|
||||||
return False
|
|
||||||
|
|
||||||
if koji_event != old_koji_event:
|
|
||||||
self.log_debug(
|
|
||||||
"Koji event doesn't match, querying changes between event %d and %d"
|
|
||||||
% (old_koji_event, koji_event)
|
|
||||||
)
|
|
||||||
changed = self.koji_proxy.queryHistory(
|
|
||||||
tables=["tag_listing", "tag_inheritance"],
|
|
||||||
tag=tag,
|
|
||||||
afterEvent=min(koji_event, old_koji_event),
|
|
||||||
beforeEvent=max(koji_event, old_koji_event) + 1,
|
|
||||||
)
|
|
||||||
if changed["tag_listing"]:
|
|
||||||
self.log_debug("Builds under tag %s changed. Can't reuse." % tag)
|
|
||||||
return False
|
|
||||||
if changed["tag_inheritance"]:
|
|
||||||
self.log_debug("Tag inheritance %s changed. Can't reuse." % tag)
|
|
||||||
return False
|
|
||||||
|
|
||||||
if inherit:
|
|
||||||
inherit_tags = self.koji_proxy.getFullInheritance(tag, koji_event)
|
|
||||||
for t in inherit_tags:
|
|
||||||
changed = self.koji_proxy.queryHistory(
|
|
||||||
tables=["tag_listing", "tag_inheritance"],
|
|
||||||
tag=t["name"],
|
|
||||||
afterEvent=min(koji_event, old_koji_event),
|
|
||||||
beforeEvent=max(koji_event, old_koji_event) + 1,
|
|
||||||
)
|
|
||||||
if changed["tag_listing"]:
|
|
||||||
self.log_debug(
|
|
||||||
"Builds under inherited tag %s changed. Can't reuse."
|
|
||||||
% t["name"]
|
|
||||||
)
|
|
||||||
return False
|
|
||||||
if changed["tag_inheritance"]:
|
|
||||||
self.log_debug("Tag inheritance %s changed. Can't reuse." % tag)
|
|
||||||
return False
|
|
||||||
|
|
||||||
repo_dir = compose.paths.work.pkgset_repo(tag, create_dir=False)
|
|
||||||
old_repo_dir = compose.paths.old_compose_path(repo_dir)
|
|
||||||
if not old_repo_dir:
|
|
||||||
self.log_debug("Can't find old repo dir to reuse.")
|
|
||||||
return False
|
|
||||||
|
|
||||||
old_reuse_file = compose.paths.old_compose_path(
|
|
||||||
compose.paths.work.pkgset_reuse_file(tag)
|
|
||||||
)
|
|
||||||
|
|
||||||
try:
|
|
||||||
self.log_debug("Loading reuse file: %s" % old_reuse_file)
|
|
||||||
reuse_data = self.load_old_file_cache(old_reuse_file)
|
|
||||||
except Exception as e:
|
|
||||||
self.log_debug("Failed to load reuse file: %s" % str(e))
|
|
||||||
return False
|
|
||||||
|
|
||||||
if (
|
|
||||||
reuse_data["allow_invalid_sigkeys"] == self._allow_invalid_sigkeys
|
|
||||||
and reuse_data["packages"] == self.packages
|
|
||||||
and reuse_data["populate_only_packages"] == self.populate_only_packages
|
|
||||||
and reuse_data["extra_builds"] == self.extra_builds
|
|
||||||
and reuse_data["sigkeys"] == self.sigkey_ordering
|
|
||||||
and reuse_data["include_packages"] == include_packages
|
|
||||||
):
|
|
||||||
self.log_info("Copying repo data for reuse: %s" % old_repo_dir)
|
|
||||||
copy_all(old_repo_dir, repo_dir)
|
|
||||||
self.reuse = old_repo_dir
|
|
||||||
self.rpms_by_arch = reuse_data["rpms_by_arch"]
|
|
||||||
self.srpms_by_name = reuse_data["srpms_by_name"]
|
|
||||||
if self.old_file_cache:
|
|
||||||
self.file_cache = self.old_file_cache
|
|
||||||
return True
|
|
||||||
else:
|
|
||||||
self.log_info("Criteria does not match. Nothing to reuse.")
|
|
||||||
return False
|
|
||||||
|
|
||||||
|
|
||||||
def _is_src(rpm_info):
|
def _is_src(rpm_info):
|
||||||
"""Check if rpm info object returned by Koji refers to source packages."""
|
"""Check if rpm info object returned by Koji refers to source packages."""
|
||||||
|
@ -200,7 +200,10 @@ class PkgsetSourceKojiMock(pungi.phases.pkgset.source.PkgsetSourceBase):
|
|||||||
|
|
||||||
def __call__(self):
|
def __call__(self):
|
||||||
compose = self.compose
|
compose = self.compose
|
||||||
self.koji_wrapper = pungi.wrappers.kojiwrapper.KojiMockWrapper(compose)
|
self.koji_wrapper = pungi.wrappers.kojiwrapper.KojiMockWrapper(
|
||||||
|
compose,
|
||||||
|
get_all_arches(compose),
|
||||||
|
)
|
||||||
# path prefix must contain trailing '/'
|
# path prefix must contain trailing '/'
|
||||||
path_prefix = self.koji_wrapper.koji_module.config.topdir.rstrip("/") + "/"
|
path_prefix = self.koji_wrapper.koji_module.config.topdir.rstrip("/") + "/"
|
||||||
package_sets = get_pkgset_from_koji(
|
package_sets = get_pkgset_from_koji(
|
||||||
@ -622,7 +625,6 @@ def _get_modules_from_koji_tags(
|
|||||||
module_builds = filter_by_whitelist(
|
module_builds = filter_by_whitelist(
|
||||||
compose, module_builds, variant_modules, expected_modules
|
compose, module_builds, variant_modules, expected_modules
|
||||||
)
|
)
|
||||||
|
|
||||||
# Find the latest builds of all modules. This does following:
|
# Find the latest builds of all modules. This does following:
|
||||||
# - Sorts the module_builds descending by Koji NVR (which maps to NSV
|
# - Sorts the module_builds descending by Koji NVR (which maps to NSV
|
||||||
# for modules). Split release into modular version and context, and
|
# for modules). Split release into modular version and context, and
|
||||||
@ -643,7 +645,11 @@ def _get_modules_from_koji_tags(
|
|||||||
latest_builds = []
|
latest_builds = []
|
||||||
module_builds = sorted(module_builds, key=_key, reverse=True)
|
module_builds = sorted(module_builds, key=_key, reverse=True)
|
||||||
for ns, ns_builds in groupby(
|
for ns, ns_builds in groupby(
|
||||||
module_builds, key=lambda x: ":".join([x["name"], x["version"]])
|
module_builds, key=lambda x: ":".join([
|
||||||
|
x["name"],
|
||||||
|
x["version"],
|
||||||
|
x['arch'],
|
||||||
|
])
|
||||||
):
|
):
|
||||||
for nsv, nsv_builds in groupby(
|
for nsv, nsv_builds in groupby(
|
||||||
ns_builds, key=lambda x: x["release"].split(".")[0]
|
ns_builds, key=lambda x: x["release"].split(".")[0]
|
||||||
|
@ -341,6 +341,18 @@ class PackagesGenerator:
|
|||||||
)
|
)
|
||||||
else:
|
else:
|
||||||
src_package_name = src_package_name[0].name
|
src_package_name = src_package_name[0].name
|
||||||
|
# TODO: for x86_64 + i686 in one packages.json
|
||||||
|
# don't remove!
|
||||||
|
# if package.arch in self.addon_repos[variant_arch]:
|
||||||
|
# arches = self.addon_repos[variant_arch] + [variant_arch]
|
||||||
|
# else:
|
||||||
|
# arches = [variant_arch]
|
||||||
|
# for arch in arches:
|
||||||
|
# pkgs_list = packages_json[variant_name][
|
||||||
|
# arch][src_package_name]
|
||||||
|
# added_pkg = f'{package_name}.{package_arch}'
|
||||||
|
# if added_pkg not in pkgs_list:
|
||||||
|
# pkgs_list.append(added_pkg)
|
||||||
pkgs_list = packages_json[variant_name][
|
pkgs_list = packages_json[variant_name][
|
||||||
variant_arch][src_package_name]
|
variant_arch][src_package_name]
|
||||||
added_pkg = f'{package_name}.{package_arch}'
|
added_pkg = f'{package_name}.{package_arch}'
|
||||||
|
@ -43,10 +43,11 @@ class KojiMock:
|
|||||||
Class that acts like real koji (for some needed methods)
|
Class that acts like real koji (for some needed methods)
|
||||||
but uses local storage as data source
|
but uses local storage as data source
|
||||||
"""
|
"""
|
||||||
def __init__(self, packages_dir, modules_dir):
|
def __init__(self, packages_dir, modules_dir, all_arches):
|
||||||
self._modules = self._gather_modules(modules_dir)
|
self._modules = self._gather_modules(modules_dir)
|
||||||
self._modules_dir = modules_dir
|
self._modules_dir = modules_dir
|
||||||
self._packages_dir = packages_dir
|
self._packages_dir = packages_dir
|
||||||
|
self._all_arches = all_arches
|
||||||
|
|
||||||
@staticmethod
|
@staticmethod
|
||||||
def _gather_modules(modules_dir):
|
def _gather_modules(modules_dir):
|
||||||
@ -93,6 +94,7 @@ class KojiMock:
|
|||||||
'name': module.name,
|
'name': module.name,
|
||||||
'id': module.build_id,
|
'id': module.build_id,
|
||||||
'tag_name': tag_name,
|
'tag_name': tag_name,
|
||||||
|
'arch': module.arch,
|
||||||
# Following fields are currently not
|
# Following fields are currently not
|
||||||
# used but returned by real koji
|
# used but returned by real koji
|
||||||
# left them here just for reference
|
# left them here just for reference
|
||||||
@ -246,15 +248,19 @@ class KojiMock:
|
|||||||
"""
|
"""
|
||||||
Get list of builds for module and given module tag name.
|
Get list of builds for module and given module tag name.
|
||||||
"""
|
"""
|
||||||
module = self._get_module_by_name(tag_name)
|
builds = []
|
||||||
path = os.path.join(
|
packages = []
|
||||||
self._modules_dir,
|
modules = self._get_modules_by_name(tag_name)
|
||||||
module.arch,
|
for module in modules:
|
||||||
tag_name,
|
if module is None:
|
||||||
)
|
raise ValueError('Module %s is not found' % tag_name)
|
||||||
|
path = os.path.join(
|
||||||
|
self._modules_dir,
|
||||||
|
module.arch,
|
||||||
|
tag_name,
|
||||||
|
)
|
||||||
|
|
||||||
builds = [
|
builds.append({
|
||||||
{
|
|
||||||
"build_id": module.build_id,
|
"build_id": module.build_id,
|
||||||
"package_name": module.name,
|
"package_name": module.name,
|
||||||
"nvr": module.nvr,
|
"nvr": module.nvr,
|
||||||
@ -280,35 +286,33 @@ class KojiMock:
|
|||||||
# "volume_id": 0,
|
# "volume_id": 0,
|
||||||
# "package_id": 104,
|
# "package_id": 104,
|
||||||
# "owner_id": 6,
|
# "owner_id": 6,
|
||||||
}
|
})
|
||||||
]
|
|
||||||
if module is None:
|
|
||||||
raise ValueError('Module %s is not found' % tag_name)
|
|
||||||
|
|
||||||
packages = []
|
if os.path.exists(path):
|
||||||
if os.path.exists(path):
|
info = Modulemd.ModuleStream.read_string(open(path).read(), strict=True)
|
||||||
info = Modulemd.ModuleStream.read_string(open(path).read(), strict=True)
|
for art in info.get_rpm_artifacts():
|
||||||
for art in info.get_rpm_artifacts():
|
data = parse_nvra(art)
|
||||||
data = parse_nvra(art)
|
packages.append({
|
||||||
packages.append({
|
"build_id": module.build_id,
|
||||||
"build_id": module.build_id,
|
"name": data['name'],
|
||||||
"name": data['name'],
|
"extra": None,
|
||||||
"extra": None,
|
"arch": data['arch'],
|
||||||
"arch": data['arch'],
|
"epoch": data['epoch'] or None,
|
||||||
"epoch": data['epoch'] or None,
|
"version": data['version'],
|
||||||
"version": data['version'],
|
"metadata_only": False,
|
||||||
"metadata_only": False,
|
"release": data['release'],
|
||||||
"release": data['release'],
|
"id": 262555,
|
||||||
"id": 262555,
|
"size": 0
|
||||||
"size": 0
|
})
|
||||||
})
|
else:
|
||||||
else:
|
raise RuntimeError('Unable to find module %s' % path)
|
||||||
raise RuntimeError('Unable to find module %s' % path)
|
|
||||||
return builds, packages
|
return builds, packages
|
||||||
|
|
||||||
def _get_module_by_name(self, tag_name):
|
def _get_modules_by_name(self, tag_name):
|
||||||
for module in self._modules.values():
|
modules = []
|
||||||
if module.nvr != tag_name:
|
for arch in self._all_arches:
|
||||||
continue
|
for module in self._modules.values():
|
||||||
return module
|
if module.nvr != tag_name or module.arch != arch:
|
||||||
return None
|
continue
|
||||||
|
modules.append(module)
|
||||||
|
return modules
|
||||||
|
@ -868,7 +868,8 @@ class KojiWrapper(object):
|
|||||||
class KojiMockWrapper(object):
|
class KojiMockWrapper(object):
|
||||||
lock = threading.Lock()
|
lock = threading.Lock()
|
||||||
|
|
||||||
def __init__(self, compose):
|
def __init__(self, compose, all_arches):
|
||||||
|
self.all_arches = all_arches
|
||||||
self.compose = compose
|
self.compose = compose
|
||||||
try:
|
try:
|
||||||
self.profile = self.compose.conf["koji_profile"]
|
self.profile = self.compose.conf["koji_profile"]
|
||||||
@ -898,7 +899,8 @@ class KojiMockWrapper(object):
|
|||||||
modules_dir=os.path.join(
|
modules_dir=os.path.join(
|
||||||
self.koji_module.config.topdir,
|
self.koji_module.config.topdir,
|
||||||
'modules',
|
'modules',
|
||||||
)
|
),
|
||||||
|
all_arches=self.all_arches,
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
|
@ -117,7 +117,11 @@ version: '1'
|
|||||||
os.makedirs(os.path.join(PATH_TO_REPOS, os.path.dirname(filepath)), exist_ok=True)
|
os.makedirs(os.path.join(PATH_TO_REPOS, os.path.dirname(filepath)), exist_ok=True)
|
||||||
open(os.path.join(PATH_TO_REPOS, filepath), 'w').close()
|
open(os.path.join(PATH_TO_REPOS, filepath), 'w').close()
|
||||||
|
|
||||||
self._koji = KojiMock(PATH_TO_REPOS, os.path.join(PATH_TO_REPOS, 'modules'))
|
self._koji = KojiMock(
|
||||||
|
PATH_TO_REPOS,
|
||||||
|
os.path.join(PATH_TO_REPOS, 'modules'),
|
||||||
|
['x86_64', 'noarch', 'i686'],
|
||||||
|
)
|
||||||
|
|
||||||
@ddt.data(
|
@ddt.data(
|
||||||
[0, {
|
[0, {
|
||||||
@ -331,6 +335,7 @@ version: '1'
|
|||||||
result = self._koji.listTagged('dist-c8-module-compose')
|
result = self._koji.listTagged('dist-c8-module-compose')
|
||||||
self.assertEqual([
|
self.assertEqual([
|
||||||
{
|
{
|
||||||
|
'arch': 'x86_64',
|
||||||
'build_id': 0,
|
'build_id': 0,
|
||||||
'id': 0,
|
'id': 0,
|
||||||
'name': 'javapackages-tools',
|
'name': 'javapackages-tools',
|
||||||
@ -342,6 +347,7 @@ version: '1'
|
|||||||
'version': '201801'
|
'version': '201801'
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
|
'arch': 'x86_64',
|
||||||
'build_id': 1,
|
'build_id': 1,
|
||||||
'id': 1,
|
'id': 1,
|
||||||
'name': 'mariadb-devel',
|
'name': 'mariadb-devel',
|
||||||
|
Loading…
Reference in New Issue
Block a user
Why double indentation for the arguments? Previous variant looks fine
It's pycharm and its rules) In some cases it does double indentation for which constructions.