almalinux/pungi
14
1
Fork 0
Code Issues Pull Requests Releases Activity

Allow building compose with scratch builds defined by pkgset_koji_scratch_tasks.

Browse Source 
Signed-off-by: Jan Kaluza <jkaluza@redhat.com>
This commit is contained in:
Jan Kaluza 2020-06-08 08:00:41 +02:00 committed by lsedlar
parent b59bdcea92
commit 4a15d1351a
6 changed files with 162 additions and 2 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

7
doc/configuration.rst
View File

@ -522,6 +522,13 @@ Options
**pkgset_koji_builds**
(*str|[str]*) -- extra build(s) to include in a package set defined as NVRs.
**pkgset_koji_scratch_tasks**
(*str|[str]*) -- RPM scratch build task(s) to include in a package set,
defined as task IDs. This option can be used only when ``compose_type``
is set to ``test``. The RPM still needs to have higher NVR than any
other RPM with the same name comming from other sources in order to
appear in the resulting compose.
**pkgset_koji_module_tag**
(*str|[str]*) -- tags to read module from. This option works similarly to
listing tags in variants XML. If tags are specified and variants XML

1
pungi/checks.py
View File

@ -778,6 +778,7 @@ def make_schema():
"koji_event": {"type": "number"},
"pkgset_koji_tag": {"$ref": "#/definitions/strings"},
"pkgset_koji_builds": {"$ref": "#/definitions/strings"},
"pkgset_koji_scratch_tasks": {"$ref": "#/definitions/strings"},
"pkgset_koji_module_tag": {"$ref": "#/definitions/strings", "default": []},
"pkgset_koji_inherit": {"type": "boolean", "default": True},
"pkgset_koji_inherit_modules": {"type": "boolean", "default": False},

75
pungi/phases/pkgset/pkgsets.py
View File

@ -328,6 +328,7 @@ class KojiPackageSet(PackageSetBase):
populate_only_packages=False,
cache_region=None,
extra_builds=None,
extra_tasks=None,
):
"""
Creates new KojiPackageSet.
@ -357,6 +358,9 @@ class KojiPackageSet(PackageSetBase):
again.
:param list extra_builds: Extra builds NVRs to get from Koji and include
in the package set.
:param list extra_tasks: Extra RPMs defined as Koji task IDs to get from Koji
and include in the package set. Useful when building testing compose
with RPM scratch builds.
"""
super(KojiPackageSet, self).__init__(
name,
@ -371,6 +375,7 @@ class KojiPackageSet(PackageSetBase):
self.populate_only_packages = populate_only_packages
self.cache_region = cache_region
self.extra_builds = extra_builds or []
self.extra_tasks = extra_tasks or []
self.reuse = None
def __getstate__(self):
@ -413,6 +418,53 @@ class KojiPackageSet(PackageSetBase):
rpms += rpms_in_build
return rpms, builds
def get_extra_rpms_from_tasks(self):
"""
Returns manually constructed RPM infos from the Koji tasks defined
in `self.extra_tasks`.
:rtype: list
:return: List with RPM infos defined as dicts with following keys:
- name, version, release, arch, src - as returned by parse_nvra.
- path_from_task - Full path to RPM on /mnt/koji.
- build_id - Always set to None.
"""
if not self.extra_tasks:
return []
# Get the IDs of children tasks - these are the tasks containing
# the resulting RPMs.
children_tasks = self.koji_wrapper.retrying_multicall_map(
self.koji_proxy,
self.koji_proxy.getTaskChildren,
list_of_args=self.extra_tasks,
)
children_task_ids = []
for tasks in children_tasks:
children_task_ids += [t["id"] for t in tasks]
# Get the results of these children tasks.
results = self.koji_wrapper.retrying_multicall_map(
self.koji_proxy,
self.koji_proxy.getTaskResult,
list_of_args=children_task_ids,
)
rpms = []
for result in results:
rpms += result.get("rpms", [])
rpms += result.get("srpms", [])
rpm_infos = []
for rpm in rpms:
rpm_info = kobo.rpmlib.parse_nvra(os.path.basename(rpm))
rpm_info["path_from_task"] = os.path.join(
self.koji_wrapper.koji_module.pathinfo.work(), rpm
)
rpm_info["build_id"] = None
rpm_infos.append(rpm_info)
return rpm_infos
def get_latest_rpms(self, tag, event, inherit=True):
if not tag:
return [], []
@ -443,6 +495,18 @@ class KojiPackageSet(PackageSetBase):
def get_package_path(self, queue_item):
rpm_info, build_info = queue_item
# Check if this RPM is comming from scratch task. In this case, we already
# know the path and we must ensure unsigned packages are allowed.
if "path_from_task" in rpm_info:
if None in self.sigkey_ordering or "" in self.sigkey_ordering:
return rpm_info["path_from_task"]
else:
self.log_error(
"Scratch RPM %s cannot be used in signed compose." % (rpm_info)
)
return None
pathinfo = self.koji_wrapper.koji_module.pathinfo
paths = []
for sigkey in self.sigkey_ordering:
@ -521,6 +585,9 @@ class KojiPackageSet(PackageSetBase):
else:
builds_by_id.setdefault(build_id, build_info)
# Get extra RPMs from tasks.
rpms += self.get_extra_rpms_from_tasks()
skipped_arches = []
skipped_packages_count = 0
# We need to process binary packages first, and then source packages.
@ -560,14 +627,20 @@ class KojiPackageSet(PackageSetBase):
skipped_packages_count += 1
continue
build_info = builds_by_id[rpm_info["build_id"]]
build_info = builds_by_id.get(rpm_info["build_id"], None)
if _is_src(rpm_info):
result_srpms.append((rpm_info, build_info))
else:
result_rpms.append((rpm_info, build_info))
if self.populate_only_packages and self.packages:
# Only add the package if we already have some whitelist.
if build_info:
self.packages.add(build_info["name"])
else:
# We have no build info and therefore no Koji package name,
# we can only guess that the Koji package name would be the same
# one as the RPM name.
self.packages.add(rpm_info["name"])
if skipped_packages_count:
self.log_debug(

3
pungi/phases/pkgset/sources/source_koji.py
View File

@ -641,8 +641,10 @@ def populate_global_pkgset(compose, koji_wrapper, path_prefix, event):
compose.log_info("Loading package set for tag %s", compose_tag)
if compose_tag in pkgset_koji_tags:
extra_builds = force_list(compose.conf.get("pkgset_koji_builds", []))
extra_tasks = force_list(compose.conf.get("pkgset_koji_scratch_tasks", []))
else:
extra_builds = []
extra_tasks = []
pkgset = pungi.phases.pkgset.pkgsets.KojiPackageSet(
compose_tag,
@ -655,6 +657,7 @@ def populate_global_pkgset(compose, koji_wrapper, path_prefix, event):
populate_only_packages=populate_only_packages_to_gather,
cache_region=compose.cache_region,
extra_builds=extra_builds,
extra_tasks=extra_tasks,
)
# Check if we have cache for this tag from previous compose. If so, use

6
pungi/scripts/pungi_koji.py
View File

@ -232,6 +232,12 @@ def main():
if compose_type == "production" and not opts.label and not opts.no_label:
abort("must specify label for a production compose")
if (
compose_type != "test"
and conf.get("pkgset_koji_scratch_tasks", None) is not None
):
abort('pkgset_koji_scratch_tasks can be used only for "test" compose type')
# check if all requirements are met
import pungi.checks

70
tests/test_pkgset_pkgsets.py
View File

@ -33,6 +33,9 @@ class MockPathInfo(object):
def rpm(self, rpm_info):
return os.path.join("rpms", self.get_filename(rpm_info))
def work(self):
return "work"
class MockFile(object):
def __init__(self, path):
@ -380,6 +383,73 @@ class TestKojiPkgset(PkgsetCompareMixin, helpers.PungiTestCase):
},
)
def test_get_extra_rpms_from_tasks(self):
pkgset = pkgsets.KojiPackageSet(
"pkgset",
self.koji_wrapper,
[None],
arches=["x86_64"],
extra_tasks=["123", "456"],
)
children_tasks = [[{"id": 1}, {"id": 2}], [{"id": 3}, {"id": 4}]]
task_results = [
{
"logs": [
"tasks/root.log",
"tasks/hw_info.log",
"tasks/state.log",
"tasks/build.log",
"tasks/mock_output.log",
"tasks/noarch_rpmdiff.json",
],
"rpms": ["tasks/pungi-4.1.39-5.f30.noarch.rpm"],
"srpms": ["tasks/pungi-4.1.39-5.f30.src.rpm"],
},
{
"logs": [
"tasks/5478/29155478/root.log",
"tasks/5478/29155478/hw_info.log",
"tasks/5478/29155478/state.log",
"tasks/5478/29155478/build.log",
],
"source": {
"source": "pungi-4.1.39-5.f30.src.rpm",
"url": "pungi-4.1.39-5.f30.src.rpm",
},
"srpm": "tasks/5478/29155478/pungi-4.1.39-5.f30.src.rpm",
},
]
self.koji_wrapper.retrying_multicall_map.side_effect = [
children_tasks,
task_results,
]
expected_rpms = [
{
"arch": "noarch",
"build_id": None,
"epoch": "",
"name": "pungi",
"path_from_task": "work/tasks/pungi-4.1.39-5.f30.noarch.rpm",
"release": "5.f30",
"src": False,
"version": "4.1.39",
},
{
"arch": "src",
"build_id": None,
"epoch": "",
"name": "pungi",
"path_from_task": "work/tasks/pungi-4.1.39-5.f30.src.rpm",
"release": "5.f30",
"src": True,
"version": "4.1.39",
},
]
rpms = pkgset.get_extra_rpms_from_tasks()
self.assertEqual(rpms, expected_rpms)
def test_get_latest_rpms_cache(self):
self._touch_files(
[