almalinux/pungi
15
2
Fork 0
Code Issues Pull Requests Releases Activity

New upstream release 4.8.0

Browse Source 
(cherry picked from commit 85b8b74f54cc6e1afd13c9608ad791d6ad103b61)
This commit is contained in:
Lubomír Sedlář 2024-11-29 15:08:29 +02:00 committed by Stepan Oksanichenko
parent 1fb0c8aa16
commit 80e22467e7
9 changed files with 18 additions and 1039 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

214
1780.patch
View File

@ -1,214 +0,0 @@
From d351773dab7b3aa8e6de82bbe23058b6b3448dd4 Mon Sep 17 00:00:00 2001
From: Lubomír Sedlář <lsedlar@redhat.com>
Date: Aug 26 2024 06:53:10 +0000
Subject: kiwibuild: Add options for version and repo_releasever
The version follows the same rules as versioning for live media etc.
That means it's always going to be set. The precedence goes like this:
* image specific option
* `kiwibuild_version`
* `global_version`
* `release_version` or `<release_version>_<label_milestone>`.
Signed-off-by: Lubomír Sedlář <lsedlar@redhat.com>
---
diff --git a/doc/configuration.rst b/doc/configuration.rst
index b0a6d5c..c6f936a 100644
--- a/doc/configuration.rst
+++ b/doc/configuration.rst
@@ -1570,6 +1570,13 @@ KiwiBuild Settings
* ``type_attr`` -- (*[str]*) override default attributes for the build type
from description.
* ``bundle_name_format`` -- (*str*) override default bundle format name.
+ * ``version`` -- (*str*) override version. Follows the same rules as
+ described in :ref:`automatic versioning <auto-version>`.
+ * ``repo_releasever`` -- (*str*) Override default releasever of the output
+ image.
+
+ The options can be set either for the specific image, or at the phase level
+ (see below). Version also falls back to ``global_version``.
**kiwibuild_description_scm**
(*str*) -- URL for scm containing the description files
@@ -1586,6 +1593,12 @@ KiwiBuild Settings
**kiwibuild_bundle_name_format**
(*str*) -- override default bundle format name.
+**kiwibuild_version**
+ (*str*) -- overide version for all kiwibuild tasks.
+
+**kiwibuild_repo_releasever**
+ (*str*) -- override releasever for all kiwibuild tasks.
+
OSBuild Composer for building images
====================================
diff --git a/pungi/checks.py b/pungi/checks.py
index dae31f5..bcf23e5 100644
--- a/pungi/checks.py
+++ b/pungi/checks.py
@@ -1212,6 +1212,8 @@ def make_schema():
"type": {"type": "string"},
"type_attr": {"$ref": "#/definitions/list_of_strings"},
"bundle_name_format": {"type": "string"},
+ "version": {"type": "string"},
+ "repo_releasever": {"type": "string"},
},
"required": [
# description_scm and description_path
@@ -1233,6 +1235,8 @@ def make_schema():
"kiwibuild_type": {"type": "string"},
"kiwibuild_type_attr": {"$ref": "#/definitions/list_of_strings"},
"kiwibuild_bundle_name_format": {"type": "string"},
+ "kiwibuild_version": {"type": "string"},
+ "kiwibuild_repo_releasever": {"type": "string"},
"osbuild_target": {"type": "string"},
"osbuild_release": {"$ref": "#/definitions/optional_string"},
"osbuild_version": {"type": "string"},
diff --git a/pungi/phases/kiwibuild.py b/pungi/phases/kiwibuild.py
index fabcce5..7ce3a54 100644
--- a/pungi/phases/kiwibuild.py
+++ b/pungi/phases/kiwibuild.py
@@ -84,6 +84,8 @@ class KiwiBuildPhase(
"bundle_name_format": self.get_config(
image_conf, "bundle_name_format"
),
+ "version": self.get_version(image_conf),
+ "repo_releasever": self.get_config(image_conf, "repo_releasever"),
}
repo = self._get_repo(image_conf, variant)
@@ -144,6 +146,8 @@ class RunKiwiBuildThread(WorkerThread):
result_bundle_name_format=generics["bundle_name_format"],
# this ensures the task won't fail if only failable arches fail
optional_arches=self.failable_arches,
+ version=generics["version"],
+ repo_releasever=generics["repo_releasever"],
)
koji.save_task_id(task_id)
diff --git a/tests/test_kiwibuildphase.py b/tests/test_kiwibuildphase.py
index 237dcaf..50116a2 100644
--- a/tests/test_kiwibuildphase.py
+++ b/tests/test_kiwibuildphase.py
@@ -50,6 +50,8 @@ class TestKiwiBuildPhase(PungiTestCase):
"type": None,
"type_attr": None,
"bundle_name_format": None,
+ "version": compose.image_version,
+ "repo_releasever": None,
},
[self.topdir + "/compose/Server/$arch/os"],
[], # failable arches
@@ -69,6 +71,8 @@ class TestKiwiBuildPhase(PungiTestCase):
"type": "custom",
"type_attr": ["foo", "bar"],
"bundle_name_format": "fmt",
+ "version": "Rawhide",
+ "repo_releasever": "41",
},
MINIMAL_CONF,
)
@@ -95,6 +99,8 @@ class TestKiwiBuildPhase(PungiTestCase):
"type": "custom",
"type_attr": ["foo", "bar"],
"bundle_name_format": "fmt",
+ "version": "Rawhide",
+ "repo_releasever": "41",
},
[
"https://example.com/repo/",
@@ -131,6 +137,8 @@ class TestKiwiBuildPhase(PungiTestCase):
"type": None,
"type_attr": None,
"bundle_name_format": None,
+ "version": compose.image_version,
+ "repo_releasever": None,
},
[self.topdir + "/compose/Server/$arch/os"],
["x86_64"], # failable arches
@@ -151,6 +159,8 @@ class TestKiwiBuildPhase(PungiTestCase):
"kiwibuild_type": "custom",
"kiwibuild_type_attr": ["foo", "bar"],
"kiwibuild_bundle_name_format": "fmt",
+ "kiwibuild_version": "Rawhide",
+ "kiwibuild_repo_releasever": "41",
},
)
@@ -175,6 +185,8 @@ class TestKiwiBuildPhase(PungiTestCase):
"type": "custom",
"type_attr": ["foo", "bar"],
"bundle_name_format": "fmt",
+ "version": "Rawhide",
+ "repo_releasever": "41",
},
[self.topdir + "/compose/Server/$arch/os"],
[], # failable arches
@@ -190,6 +202,7 @@ class TestKiwiBuildPhase(PungiTestCase):
"kiwibuild": {"^Server$": [cfg]},
"global_target": "f40",
"global_release": "1234",
+ "global_version": "41",
},
)
@@ -214,6 +227,8 @@ class TestKiwiBuildPhase(PungiTestCase):
"type": None,
"type_attr": None,
"bundle_name_format": None,
+ "version": "41",
+ "repo_releasever": None,
},
[self.topdir + "/compose/Server/$arch/os"],
[], # failable arches
@@ -278,6 +293,8 @@ class TestKiwiBuildThread(PungiTestCase):
"type": "t",
"type_attr": ["ta"],
"bundle_name_format": "fmt",
+ "version": "v",
+ "repo_releasever": "r",
},
[self.repo],
[],
@@ -298,6 +315,8 @@ class TestKiwiBuildThread(PungiTestCase):
type_attr=["ta"],
result_bundle_name_format="fmt",
optional_arches=[],
+ version="v",
+ repo_releasever="r",
)
]
@@ -362,6 +381,8 @@ class TestKiwiBuildThread(PungiTestCase):
"type": None,
"type_attr": None,
"bundle_name_format": None,
+ "version": None,
+ "repo_releasever": None,
},
[self.repo],
[],
@@ -413,6 +434,8 @@ class TestKiwiBuildThread(PungiTestCase):
"type": None,
"type_attr": None,
"bundle_name_format": None,
+ "version": None,
+ "repo_releasever": None,
},
[self.repo],
["amd64"],
@@ -433,6 +456,8 @@ class TestKiwiBuildThread(PungiTestCase):
type_attr=None,
result_bundle_name_format=None,
optional_arches=["amd64"],
+ version=None,
+ repo_releasever=None,
)
]
assert get_image_paths.mock_calls == [mock.call(1234)]

286
1782.patch
View File

@ -1,286 +0,0 @@
From d9d21d3cf4eaad5cc7f2959a4abdafed781bb9cf Mon Sep 17 00:00:00 2001
From: Lubomír Sedlář <lsedlar@redhat.com>
Date: Aug 28 2024 11:07:12 +0000
Subject: [PATCH 1/2] kiwibuild: Fix location and metadata for ISOs
When Kiwi builds an ISO, it is always supposed to be bootable and should
be located in the iso/ subdirectory.
Any other kind of image should still land in images/ and be listed as
not bootable in the metadata.
Relates: https://pagure.io/pungi-fedora/issue/1342
Signed-off-by: Lubomír Sedlář <lsedlar@redhat.com>
---
diff --git a/pungi/phases/kiwibuild.py b/pungi/phases/kiwibuild.py
index 7ce3a54..12ec26d 100644
--- a/pungi/phases/kiwibuild.py
+++ b/pungi/phases/kiwibuild.py
@@ -184,10 +184,20 @@ class RunKiwiBuildThread(WorkerThread):
# image_dir is absolute path to which the image should be copied.
# We also need the same path as relative to compose directory for
# including in the metadata.
- image_dir = compose.paths.compose.image_dir(variant) % {"arch": arch}
- rel_image_dir = compose.paths.compose.image_dir(
- variant, relative=True
- ) % {"arch": arch}
+ if format_ == "iso":
+ # If the produced image is actually an ISO, it should go to
+ # iso/ subdirectory.
+ image_dir = compose.paths.compose.iso_dir(arch, variant)
+ rel_image_dir = compose.paths.compose.iso_dir(
+ arch, variant, relative=True
+ )
+ else:
+ image_dir = compose.paths.compose.image_dir(variant) % {
+ "arch": arch
+ }
+ rel_image_dir = compose.paths.compose.image_dir(
+ variant, relative=True
+ ) % {"arch": arch}
util.makedirs(image_dir)
filename = os.path.basename(path)
@@ -211,7 +221,8 @@ class RunKiwiBuildThread(WorkerThread):
img.arch = arch
img.disc_number = 1 # We don't expect multiple disks
img.disc_count = 1
- img.bootable = False
+ # Kiwi produces only bootable ISOs. Other kinds of images are
+ img.bootable = format_ == "iso"
img.subvariant = config.get("subvariant", variant.uid)
setattr(img, "can_fail", arch in self.failable_arches)
setattr(img, "deliverable", "kiwibuild")
diff --git a/tests/test_kiwibuildphase.py b/tests/test_kiwibuildphase.py
index 50116a2..491dbf3 100644
--- a/tests/test_kiwibuildphase.py
+++ b/tests/test_kiwibuildphase.py
@@ -242,13 +242,14 @@ class TestKiwiBuildPhase(PungiTestCase):
@mock.patch("pungi.util.get_file_size")
@mock.patch("pungi.wrappers.kojiwrapper.KojiWrapper")
class TestKiwiBuildThread(PungiTestCase):
- def _img_path(self, arch, filename=None):
- path = self.topdir + "/compose/Server/%s/images" % arch
+ def _img_path(self, arch, filename=None, dir=None):
+ dir = dir or "images"
+ path = self.topdir + "/compose/Server/%s/%s" % (arch, dir)
if filename:
path += "/" + filename
return path
- def test_process(self, KojiWrapper, get_file_size, get_mtime, Linker):
+ def test_process_vagrant_box(self, KojiWrapper, get_file_size, get_mtime, Linker):
img_name = "FCBG.{arch}-Rawhide-1.6.vagrant.libvirt.box"
self.repo = self.topdir + "/compose/Server/$arch/os"
compose = DummyCompose(
@@ -353,6 +354,114 @@ class TestKiwiBuildThread(PungiTestCase):
assert "vagrant-libvirt.box" == image.format
assert "vagrant-libvirt" == image.type
assert "Test" == image.subvariant
+ assert not image.bootable
+
+ def test_process_iso(self, KojiWrapper, get_file_size, get_mtime, Linker):
+ img_name = "FCBG.{arch}-Rawhide-1.6.iso"
+ self.repo = self.topdir + "/compose/Server/$arch/os"
+ compose = DummyCompose(
+ self.topdir,
+ {
+ "koji_profile": "koji",
+ "kiwibuild_bundle_format": "%N-%P-40_Beta-%I.%A.%T",
+ },
+ )
+ config = _merge({"subvariant": "Test"}, MINIMAL_CONF)
+ pool = mock.Mock()
+
+ get_image_paths = KojiWrapper.return_value.get_image_paths
+ get_image_paths.return_value = {
+ "x86_64": [
+ "/koji/task/1234/FCBG.x86_64-Rawhide-1.6.packages",
+ "/koji/task/1234/%s" % img_name.format(arch="x86_64"),
+ ],
+ "amd64": [
+ "/koji/task/1234/FCBG.amd64-Rawhide-1.6.packages",
+ "/koji/task/1234/%s" % img_name.format(arch="amd64"),
+ ],
+ }
+
+ KojiWrapper.return_value.koji_proxy.kiwiBuild.return_value = 1234
+ KojiWrapper.return_value.watch_task.return_value = 0
+
+ t = RunKiwiBuildThread(pool)
+ get_file_size.return_value = 1024
+ get_mtime.return_value = 13579
+ t.process(
+ (
+ compose,
+ compose.variants["Server"],
+ config,
+ ["amd64", "x86_64"],
+ {
+ "release": "1.6",
+ "target": "f40",
+ "descscm": MINIMAL_CONF["description_scm"],
+ "descpath": MINIMAL_CONF["description_path"],
+ "type": "t",
+ "type_attr": ["ta"],
+ "bundle_name_format": "fmt",
+ "version": "v",
+ "repo_releasever": "r",
+ },
+ [self.repo],
+ [],
+ ),
+ 1,
+ )
+
+ assert KojiWrapper.return_value.koji_proxy.kiwiBuild.mock_calls == [
+ mock.call(
+ "f40",
+ ["amd64", "x86_64"],
+ MINIMAL_CONF["description_scm"],
+ MINIMAL_CONF["description_path"],
+ profile=MINIMAL_CONF["kiwi_profile"],
+ release="1.6",
+ repos=[self.repo],
+ type="t",
+ type_attr=["ta"],
+ result_bundle_name_format="fmt",
+ optional_arches=[],
+ version="v",
+ repo_releasever="r",
+ )
+ ]
+
+ assert get_image_paths.mock_calls == [mock.call(1234)]
+ assert os.path.isdir(self._img_path("x86_64", dir="iso"))
+ assert os.path.isdir(self._img_path("amd64", dir="iso"))
+ Linker.return_value.link.assert_has_calls(
+ [
+ mock.call(
+ "/koji/task/1234/FCBG.amd64-Rawhide-1.6.iso",
+ self._img_path("amd64", img_name.format(arch="amd64"), dir="iso"),
+ link_type="hardlink-or-copy",
+ ),
+ mock.call(
+ "/koji/task/1234/FCBG.x86_64-Rawhide-1.6.iso",
+ self._img_path("x86_64", img_name.format(arch="x86_64"), dir="iso"),
+ link_type="hardlink-or-copy",
+ ),
+ ],
+ any_order=True,
+ )
+
+ assert len(compose.im.add.call_args_list) == 2
+ for call in compose.im.add.call_args_list:
+ _, kwargs = call
+ image = kwargs["image"]
+ expected_path = "Server/{0.arch}/iso/{1}".format(
+ image, img_name.format(arch=image.arch)
+ )
+ assert kwargs["variant"] == "Server"
+ assert kwargs["arch"] in ("amd64", "x86_64")
+ assert kwargs["arch"] == image.arch
+ assert image.path == expected_path
+ assert "iso" == image.format
+ assert "iso" == image.type
+ assert image.bootable
+ assert "Test" == image.subvariant
def test_handle_koji_fail(self, KojiWrapper, get_file_size, get_mtime, Linker):
self.repo = self.topdir + "/compose/Server/$arch/os"
From cd2ae81e3c63316997b9617ff2e30e3148af14f2 Mon Sep 17 00:00:00 2001
From: Lubomír Sedlář <lsedlar@redhat.com>
Date: Aug 29 2024 06:46:10 +0000
Subject: [PATCH 2/2] kiwibuild: Allow setting metadata type explicitly
It is not possible to reliably detect what the type for an image should
be in the metadata. This commit adds an option for user to explicitly
provide it.
It can only be configured on the specific image, not globally.
Signed-off-by: Lubomír Sedlář <lsedlar@redhat.com>
---
diff --git a/doc/configuration.rst b/doc/configuration.rst
index c6f936a..ded479e 100644
--- a/doc/configuration.rst
+++ b/doc/configuration.rst
@@ -1574,6 +1574,9 @@ KiwiBuild Settings
described in :ref:`automatic versioning <auto-version>`.
* ``repo_releasever`` -- (*str*) Override default releasever of the output
image.
+ * ``manifest_type`` -- the image type that is put into the manifest by
+ pungi. If not supplied, an autodetected value will be provided. It may or
+ may not make sense.
The options can be set either for the specific image, or at the phase level
(see below). Version also falls back to ``global_version``.
diff --git a/pungi/checks.py b/pungi/checks.py
index bcf23e5..cc16d12 100644
--- a/pungi/checks.py
+++ b/pungi/checks.py
@@ -1214,6 +1214,7 @@ def make_schema():
"bundle_name_format": {"type": "string"},
"version": {"type": "string"},
"repo_releasever": {"type": "string"},
+ "manifest_type": {"type": "string"},
},
"required": [
# description_scm and description_path
diff --git a/pungi/phases/kiwibuild.py b/pungi/phases/kiwibuild.py
index 12ec26d..bb26812 100644
--- a/pungi/phases/kiwibuild.py
+++ b/pungi/phases/kiwibuild.py
@@ -211,9 +211,9 @@ class RunKiwiBuildThread(WorkerThread):
# Update image manifest
img = Image(compose.im)
- # Get the manifest type from the config if supplied, otherwise we
- # determine the manifest type based on the koji output
- img.type = type_
+ # If user configured exact type, use it, otherwise try to
+ # figure it out based on the koji output.
+ img.type = config.get("manifest_type", type_)
img.format = format_
img.path = os.path.join(rel_image_dir, filename)
img.mtime = util.get_mtime(image_dest)
diff --git a/tests/test_kiwibuildphase.py b/tests/test_kiwibuildphase.py
index 491dbf3..e3ccdb7 100644
--- a/tests/test_kiwibuildphase.py
+++ b/tests/test_kiwibuildphase.py
@@ -73,6 +73,7 @@ class TestKiwiBuildPhase(PungiTestCase):
"bundle_name_format": "fmt",
"version": "Rawhide",
"repo_releasever": "41",
+ "manifest_type": "live-kiwi",
},
MINIMAL_CONF,
)
@@ -366,7 +367,9 @@ class TestKiwiBuildThread(PungiTestCase):
"kiwibuild_bundle_format": "%N-%P-40_Beta-%I.%A.%T",
},
)
- config = _merge({"subvariant": "Test"}, MINIMAL_CONF)
+ config = _merge(
+ {"subvariant": "Test", "manifest_type": "live-kiwi"}, MINIMAL_CONF
+ )
pool = mock.Mock()
get_image_paths = KojiWrapper.return_value.get_image_paths
@@ -459,7 +462,7 @@ class TestKiwiBuildThread(PungiTestCase):
assert kwargs["arch"] == image.arch
assert image.path == expected_path
assert "iso" == image.format
- assert "iso" == image.type
+ assert "live-kiwi" == image.type
assert image.bootable
assert "Test" == image.subvariant

211
1788.patch
View File

@ -1,211 +0,0 @@
From 5338d3098ccd614a8fd32f837a393aed78b471bd Mon Sep 17 00:00:00 2001
From: Adam Williamson <awilliam@redhat.com>
Date: Sep 20 2024 06:03:15 +0000
Subject: [PATCH 1/2] move osbuild/kiwi-specific EXTENSIONS to each phase
The image-build phase's EXTENSIONS dict is meant to exactly
mirror the 'formats' that exist in the context of the command
`koji image-build`, which is driven by this phase. That nice
association was lost, however, by adding a couple of items to it
which exist for the purposes of the osbuild phase (and in the
case of .iso, also the kiwibuild phase), which import this dict
and uses it for image identification.
To make the association 1:1 again and more clearly show what's
going on here, let's move those entries out into the osbuild and
kiwi phases. osbuild now has its own dict which starts out as a
copy of the image-build one before being extended. And let's
update the relevant comments.
Signed-off-by: Adam Williamson <awilliam@redhat.com>
---
diff --git a/pungi/phases/image_build.py b/pungi/phases/image_build.py
index cb76c1a..cd034cd 100644
--- a/pungi/phases/image_build.py
+++ b/pungi/phases/image_build.py
@@ -22,10 +22,13 @@ from productmd.rpms import Rpms
# This is a mapping from formats to file extensions. The format is what koji
# image-build command expects as argument, and the extension is what the file
# name will be ending with. The extensions are used to filter out which task
-# results will be pulled into the compose.
+# results will be pulled into the compose. This dict is also used later in
+# the process to set the image 'type' in productmd metadata terms - the type
+# is set as the first key in this dict which has the file's extension in its
+# values. This dict is imported and extended for similar purposes by other
+# phases (at least osbuild and kiwibuild).
EXTENSIONS = {
"docker": ["tar.gz", "tar.xz"],
- "iso": ["iso"],
"liveimg-squashfs": ["liveimg.squashfs"],
"qcow": ["qcow"],
"qcow2": ["qcow2"],
@@ -40,7 +43,6 @@ EXTENSIONS = {
"vdi": ["vdi"],
"vmdk": ["vmdk"],
"vpc": ["vhd"],
- "vhd-compressed": ["vhd.gz", "vhd.xz"],
"vsphere-ova": ["vsphere.ova"],
}
diff --git a/pungi/phases/kiwibuild.py b/pungi/phases/kiwibuild.py
index bb26812..15732ce 100644
--- a/pungi/phases/kiwibuild.py
+++ b/pungi/phases/kiwibuild.py
@@ -15,6 +15,10 @@ KIWIEXTENSIONS = [
("vhd-compressed", ["vhdfixed.xz"], "vhd.xz"),
("vagrant-libvirt", ["vagrant.libvirt.box"], "vagrant-libvirt.box"),
("vagrant-virtualbox", ["vagrant.virtualbox.box"], "vagrant-virtualbox.box"),
+ # .iso images can be of many types - boot, cd, dvd, live... -
+ # so 'boot' is just a default guess. 'iso' is not a valid
+ # productmd image type
+ ("boot", [".iso"], "iso"),
]
diff --git a/pungi/phases/osbuild.py b/pungi/phases/osbuild.py
index 6c5b7e5..a3b5c9b 100644
--- a/pungi/phases/osbuild.py
+++ b/pungi/phases/osbuild.py
@@ -11,6 +11,16 @@ from ..linker import Linker
from ..wrappers import kojiwrapper
from .image_build import EXTENSIONS
+# copy and modify EXTENSIONS with some that osbuild produces but which
+# do not exist as `koji image-build` formats
+OSBUILDEXTENSIONS = EXTENSIONS.copy()
+OSBUILDEXTENSIONS.update(
+ {
+ "iso": ["iso"],
+ "vhd-compressed": ["vhd.gz", "vhd.xz"],
+ }
+)
+
class OSBuildPhase(
base.PhaseLoggerMixin, base.ImageConfigMixin, base.ConfigGuardedPhase
@@ -203,7 +213,7 @@ class RunOSBuildThread(WorkerThread):
# architecture, but we don't verify that.
build_info = koji.koji_proxy.getBuild(build_id)
for archive in koji.koji_proxy.listArchives(buildID=build_id):
- if archive["type_name"] not in EXTENSIONS:
+ if archive["type_name"] not in OSBUILDEXTENSIONS:
# Ignore values that are not of required types.
continue
@@ -241,7 +251,7 @@ class RunOSBuildThread(WorkerThread):
linker.link(src_file, image_dest, link_type=compose.conf["link_type"])
- for suffix in EXTENSIONS[archive["type_name"]]:
+ for suffix in OSBUILDEXTENSIONS[archive["type_name"]]:
if archive["filename"].endswith(suffix):
break
else:
From 739062ed3c471e74ba9c5144c4047f67f9fbe8c8 Mon Sep 17 00:00:00 2001
From: Adam Williamson <awilliam@redhat.com>
Date: Sep 20 2024 06:03:58 +0000
Subject: [PATCH 2/2] image_build: drop .tar.gz as an expected extension for docker
Koji's image-build command has not been capable of producing a
docker image with .tar.gz as its extension since 2015:
https://pagure.io/koji/c/b489f282bee7a008108534404dd2e78efb2256e7?branch=master
as that commit message implies, the files have not actually been
gzip-compressed for even longer:
https://pagure.io/koji/c/82a405c7943192e3bba3340efe7a8d07a0e26b70?branch=master
so there's no point to having this any more. It is causing the
wrong productmd 'type' to be set for GCE cloud images, which *do*
have the .tar.gz extension - because docker appears in this dict
before tar-gz, their type is being set as 'docker' not 'tar-gz'.
Signed-off-by: Adam Williamson <awilliam@redhat.com>
---
diff --git a/pungi/phases/image_build.py b/pungi/phases/image_build.py
index cd034cd..c28ccc2 100644
--- a/pungi/phases/image_build.py
+++ b/pungi/phases/image_build.py
@@ -28,7 +28,7 @@ from productmd.rpms import Rpms
# values. This dict is imported and extended for similar purposes by other
# phases (at least osbuild and kiwibuild).
EXTENSIONS = {
- "docker": ["tar.gz", "tar.xz"],
+ "docker": ["tar.xz"],
"liveimg-squashfs": ["liveimg.squashfs"],
"qcow": ["qcow"],
"qcow2": ["qcow2"],
diff --git a/tests/test_imagebuildphase.py b/tests/test_imagebuildphase.py
index 90f10ef..895f040 100644
--- a/tests/test_imagebuildphase.py
+++ b/tests/test_imagebuildphase.py
@@ -846,12 +846,12 @@ class TestCreateImageBuildThread(PungiTestCase):
"amd64": [
"/koji/task/1235/tdl-amd64.xml",
"/koji/task/1235/Fedora-Docker-Base-20160103.amd64.qcow2",
- "/koji/task/1235/Fedora-Docker-Base-20160103.amd64.tar.gz",
+ "/koji/task/1235/Fedora-Docker-Base-20160103.amd64.tar.xz",
],
"x86_64": [
"/koji/task/1235/tdl-x86_64.xml",
"/koji/task/1235/Fedora-Docker-Base-20160103.x86_64.qcow2",
- "/koji/task/1235/Fedora-Docker-Base-20160103.x86_64.tar.gz",
+ "/koji/task/1235/Fedora-Docker-Base-20160103.x86_64.tar.xz",
],
}
@@ -896,9 +896,9 @@ class TestCreateImageBuildThread(PungiTestCase):
link_type="hardlink-or-copy",
),
mock.call.link(
- "/koji/task/1235/Fedora-Docker-Base-20160103.amd64.tar.gz",
+ "/koji/task/1235/Fedora-Docker-Base-20160103.amd64.tar.xz",
self.topdir
- + "/compose/Client/amd64/images/Fedora-Docker-Base-20160103.amd64.tar.gz", # noqa: E501
+ + "/compose/Client/amd64/images/Fedora-Docker-Base-20160103.amd64.tar.xz", # noqa: E501
link_type="hardlink-or-copy",
),
mock.call.link(
@@ -908,9 +908,9 @@ class TestCreateImageBuildThread(PungiTestCase):
link_type="hardlink-or-copy",
),
mock.call.link(
- "/koji/task/1235/Fedora-Docker-Base-20160103.x86_64.tar.gz",
+ "/koji/task/1235/Fedora-Docker-Base-20160103.x86_64.tar.xz",
self.topdir
- + "/compose/Client/x86_64/images/Fedora-Docker-Base-20160103.x86_64.tar.gz", # noqa: E501
+ + "/compose/Client/x86_64/images/Fedora-Docker-Base-20160103.x86_64.tar.xz", # noqa: E501
link_type="hardlink-or-copy",
),
],
@@ -922,8 +922,8 @@ class TestCreateImageBuildThread(PungiTestCase):
"type": "qcow2",
"arch": "amd64",
},
- "image_dir/Client/amd64/Fedora-Docker-Base-20160103.amd64.tar.gz": {
- "format": "tar.gz",
+ "image_dir/Client/amd64/Fedora-Docker-Base-20160103.amd64.tar.xz": {
+ "format": "tar.xz",
"type": "docker",
"arch": "amd64",
},
@@ -932,8 +932,8 @@ class TestCreateImageBuildThread(PungiTestCase):
"type": "qcow2",
"arch": "x86_64",
},
- "image_dir/Client/x86_64/Fedora-Docker-Base-20160103.x86_64.tar.gz": {
- "format": "tar.gz",
+ "image_dir/Client/x86_64/Fedora-Docker-Base-20160103.x86_64.tar.xz": {
+ "format": "tar.xz",
"type": "docker",
"arch": "x86_64",
},

146
1789.patch
View File

@ -1,146 +0,0 @@
From 391a5eaed5198e5ee2941dac4ae43a2fe057eedd Mon Sep 17 00:00:00 2001
From: Adam Williamson <awilliam@redhat.com>
Date: Oct 09 2024 23:28:54 +0000
Subject: [PATCH 1/2] Correct subvariant handling for ostree_container phase
The image metadata construction code allows for subvariant to be
set in the image config dict, but checks.py doesn't expect it;
fix that. Also, when a subvariant is set, use it in the image
name template rather than the variant; otherwise you can't
build more than one subvariant in any variant (they will have
identical names, which isn't alllowed).
Signed-off-by: Adam Williamson <awilliam@redhat.com>
---
diff --git a/doc/configuration.rst b/doc/configuration.rst
index ded479e..90621bb 100644
--- a/doc/configuration.rst
+++ b/doc/configuration.rst
@@ -1886,6 +1886,10 @@ will thus create a new OCI archive image *from scratch*.
reference will not be created.
* ``runroot_packages`` -- (*list*) A list of additional package names to be
installed in the runroot environment in Koji.
+ * ``subvariant`` -- (*str*) The subvariant value to be used in the metadata
+ for the image. Also used in the image's filename. Defaults to being the
+ same as the variant. If building more than one ostree container in a
+ variant, each must have a unique subvariant.
Example config
--------------
diff --git a/pungi/checks.py b/pungi/checks.py
index cc16d12..e23babc 100644
--- a/pungi/checks.py
+++ b/pungi/checks.py
@@ -1101,6 +1101,7 @@ def make_schema():
"runroot_packages": {
"$ref": "#/definitions/list_of_strings",
},
+ "subvariant": {"type": "string"},
},
"required": [
"treefile",
diff --git a/pungi/phases/ostree_container.py b/pungi/phases/ostree_container.py
index 38a647e..e111998 100644
--- a/pungi/phases/ostree_container.py
+++ b/pungi/phases/ostree_container.py
@@ -119,12 +119,13 @@ class OSTreeContainerThread(WorkerThread):
def _run_ostree_container_cmd(
self, compose, variant, arch, config, config_repo, extra_config_file=None
):
+ subvariant = config.get("subvariant", variant.uid)
target_dir = compose.paths.compose.image_dir(variant) % {"arch": arch}
util.makedirs(target_dir)
version = util.version_generator(compose, config.get("version"))
archive_name = "%s-%s-%s" % (
compose.conf["release_short"],
- variant.uid,
+ subvariant,
version,
)
@@ -177,7 +178,7 @@ class OSTreeContainerThread(WorkerThread):
img.disc_number = 1
img.disc_count = 1
img.bootable = False
- img.subvariant = config.get("subvariant", variant.uid)
+ img.subvariant = subvariant
setattr(img, "can_fail", self.can_fail)
setattr(img, "deliverable", "ostree-container")
compose.im.add(variant=variant.uid, arch=arch, image=img)
From aea8da5225aeb31b4c5dd413f0a31b6ab395a9ac Mon Sep 17 00:00:00 2001
From: Adam Williamson <awilliam@redhat.com>
Date: Oct 09 2024 23:33:48 +0000
Subject: [PATCH 2/2] ostree_container: make filename configurable, include arch
The default base name is probably fine in most cases, but there
are some where we might want to tweak it. We already allow this
for other phases (e.g. the livemedia phase).
Also, we should include the arch in the image filename. Not doing
this doesn't blow up the compose as, while they have identical
filenames, the images for different arches are in different paths,
but it's confusing for people who actually download and use the
images.
Signed-off-by: Adam Williamson <awilliam@redhat.com>
---
diff --git a/doc/configuration.rst b/doc/configuration.rst
index 90621bb..14947fe 100644
--- a/doc/configuration.rst
+++ b/doc/configuration.rst
@@ -1887,9 +1887,14 @@ will thus create a new OCI archive image *from scratch*.
* ``runroot_packages`` -- (*list*) A list of additional package names to be
installed in the runroot environment in Koji.
* ``subvariant`` -- (*str*) The subvariant value to be used in the metadata
- for the image. Also used in the image's filename. Defaults to being the
- same as the variant. If building more than one ostree container in a
- variant, each must have a unique subvariant.
+ for the image. Also used in the image's filename, unless overridden by
+ ``name``. Defaults to being the same as the variant. If building more
+ than one ostree container in a variant, each must have a unique
+ subvariant.
+ * ``name`` -- (*str*) The base for the image's filename. To produce the
+ complete filename, the image's architecture, the version string, and the
+ format suffix are appended to this. Defaults to the value of
+ ``release_short`` and the subvariant, joined by a dash.
Example config
--------------
diff --git a/pungi/checks.py b/pungi/checks.py
index e23babc..0cdb619 100644
--- a/pungi/checks.py
+++ b/pungi/checks.py
@@ -1102,6 +1102,7 @@ def make_schema():
"$ref": "#/definitions/list_of_strings",
},
"subvariant": {"type": "string"},
+ "name": {"type": "string"},
},
"required": [
"treefile",
diff --git a/pungi/phases/ostree_container.py b/pungi/phases/ostree_container.py
index e111998..c50633d 100644
--- a/pungi/phases/ostree_container.py
+++ b/pungi/phases/ostree_container.py
@@ -123,11 +123,8 @@ class OSTreeContainerThread(WorkerThread):
target_dir = compose.paths.compose.image_dir(variant) % {"arch": arch}
util.makedirs(target_dir)
version = util.version_generator(compose, config.get("version"))
- archive_name = "%s-%s-%s" % (
- compose.conf["release_short"],
- subvariant,
- version,
- )
+ anb = config.get("name", "%s-%s" % (compose.conf["release_short"], subvariant))
+ archive_name = "%s-%s-%s" % (anb, arch, version)
# Run the pungi-make-ostree command locally to create a script to
# execute in runroot environment.

82
1790.patch
View File

@ -1,82 +0,0 @@
From b83ac496b5a20bd0493328999fc619711c6a512a Mon Sep 17 00:00:00 2001
From: Adam Williamson <awilliam@redhat.com>
Date: Oct 11 2024 18:18:48 +0000
Subject: Don't block main image build phase on ostree_install phase
I did a time map of a Fedora compose today, and noticed that we
spend about an hour waiting for the ostree_install phase to
complete before we start up the compose_images_phase which does
all the other image builds.
This is unnecessary. Nothing else depends on ostree_install; it
should be fine to start up the extra_phase (which contains
compose_images_phase) while the ostree stuff is still running.
This implements that by splitting the ostree phases out of the
essentials_phase which contains the real precursors to the
extra_phase. We start the essentials and ostree phases together,
but only wait for the essentials phase to complete before
kicking off extra_phase, so it can start while the ostree
phase is still running.
One tweak we have to make to accommodate this is to move
image_checksum_phase out of extra_phase, to avoid it potentially
running before all ostree installer images are built. The
checksum phase is quite fast - it takes about five minutes -
and any time benefit of running it in parallel with the osbs and
repoclosure phases seems like it must be smaller than the time
loss of waiting for ostree_install before kicking off extra.
Signed-off-by: Adam Williamson <awilliam@redhat.com>
---
diff --git a/pungi/scripts/pungi_koji.py b/pungi/scripts/pungi_koji.py
index c643d53..b2382b0 100644
--- a/pungi/scripts/pungi_koji.py
+++ b/pungi/scripts/pungi_koji.py
@@ -476,11 +476,15 @@ def run_compose(
buildinstall_phase,
(gather_phase, createrepo_phase),
extrafiles_phase,
+ )
+ ostree_schema = (
(ostree_phase, ostree_installer_phase),
ostree_container_phase,
)
essentials_phase = pungi.phases.WeaverPhase(compose, essentials_schema)
+ ostree_phase = pungi.phases.WeaverPhase(compose, ostree_schema)
essentials_phase.start()
+ ostree_phase.start()
essentials_phase.stop()
# write treeinfo before ISOs are created
@@ -507,12 +511,9 @@ def run_compose(
osbuild_phase,
kiwibuild_phase,
)
- post_image_phase = pungi.phases.WeaverPhase(
- compose, (image_checksum_phase, image_container_phase)
- )
compose_images_phase = pungi.phases.WeaverPhase(compose, compose_images_schema)
extra_phase_schema = (
- (compose_images_phase, post_image_phase),
+ (compose_images_phase, image_container_phase),
osbs_phase,
repoclosure_phase,
)
@@ -520,6 +521,12 @@ def run_compose(
extra_phase.start()
extra_phase.stop()
+ # wait for ostree phase here too - it can happily run in parallel
+ # with all of the other stuff
+ ostree_phase.stop()
+ # now we do checksums as all images are done
+ image_checksum_phase.start()
+ image_checksum_phase.stop()
pungi.metadata.write_compose_info(compose)
if not (

60
1796.patch
View File

@ -1,60 +0,0 @@
From c8fe99b1aa5a9a9b941b7515cda367d24829dedf Mon Sep 17 00:00:00 2001
From: Adam Williamson <awilliam@redhat.com>
Date: Nov 19 2024 08:22:09 +0000
Subject: pkgset: optimize cache check (saves 20 minutes)
The pkgset phase takes around 35 minutes in current composes.
Around 20 minutes of that is spent creating these per-arch
subsets of the global package set. In a rather roundabout way
(see #1794 ), I figured out that almost all of this time is
spent in this cache check, which is broken for a subtle reason.
Python's `in` keyword works by first attempting to call the
container's magic `__contains__` method. If the container does
not implement `__contains__`, it falls back to iteration - it
tries to iterate over the container until it either hits what
it's looking for, or runs out. (If the container implements
neither, you get an error).
The FileCache instance's `file_cache` is a plain Python dict.
dicts have a very efficient `__contains__` implementation, so
doing `foo in (somedict)` is basically always very fast no matter
how huge the dict is. FileCache itself, though, implements
`__iter__` by returning an iterator over the `file_cache` dict's
keys, but it does *not* implement `__contains__`. So when we do
`foo in self.file_cache`, Python has to iterate over every key
in the dict until it hits foo or runs out. This is massively
slower than `foo in self.file_cache.file_cache`, which uses the
efficient `__contains__` method.
Because these package sets are so huge, and we're looping over
*one* huge set and checking each package from it against the cache
of another, increasingly huge, set, this effect becomes massive.
To make it even worse, I ran a few tests where I added a debug log
if we ever hit the cache, and it looks like we never actually do -
so every check has to iterate through the entire dict.
We could probably remove this entirely, but changing it to check
the dict instead of the FileCache instance makes it just about as
fast as taking it out, so I figured let's go with that in case
there's some unusual scenario in which the cache does work here.
Signed-off-by: Adam Williamson <awilliam@redhat.com>
---
diff --git a/pungi/phases/pkgset/pkgsets.py b/pungi/phases/pkgset/pkgsets.py
index 81c090c..ee12c3c 100644
--- a/pungi/phases/pkgset/pkgsets.py
+++ b/pungi/phases/pkgset/pkgsets.py
@@ -265,7 +265,7 @@ class PackageSetBase(kobo.log.LoggingBase):
for arch in arch_list:
self.rpms_by_arch.setdefault(arch, [])
for i in other.rpms_by_arch.get(arch, []):
- if i.file_path in self.file_cache:
+ if i.file_path in self.file_cache.file_cache:
# TODO: test if it really works
continue
if inherit_to_noarch and exclusivearch_list and arch == "noarch":

28
1798.patch
View File

@ -1,28 +0,0 @@
From eb4ba5f637153f0037f05981adea8b35fc0f6b25 Mon Sep 17 00:00:00 2001
From: Adam Williamson <awilliam@redhat.com>
Date: Nov 20 2024 00:49:30 +0000
Subject: kiwibuild: extend productmd type/format detection for FEX images
Signed-off-by: Adam Williamson <awilliam@redhat.com>
---
diff --git a/pungi/phases/kiwibuild.py b/pungi/phases/kiwibuild.py
index 15732ce..400ea6a 100644
--- a/pungi/phases/kiwibuild.py
+++ b/pungi/phases/kiwibuild.py
@@ -19,6 +19,12 @@ KIWIEXTENSIONS = [
# so 'boot' is just a default guess. 'iso' is not a valid
# productmd image type
("boot", [".iso"], "iso"),
+ ("fex", ["erofs.xz"], "erofs.xz"),
+ ("fex", ["erofs.gz"], "erofs.gz"),
+ ("fex", ["erofs"], "erofs"),
+ ("fex", ["squashfs.xz"], "squashfs.xz"),
+ ("fex", ["squashfs.gz"], "squashfs.gz"),
+ ("fex", ["squashfs"], "squashfs"),
]

28
pungi.spec
View File

@ -1,20 +1,13 @@
%{?python_enable_dependency_generator} %{?python_enable_dependency_generator}
Name: pungi Name: pungi
Version: 4.7.0 Version: 4.8.0
Release: 8%{?dist}.alma.1 Release: 1%{?dist}.alma.1
Summary: Distribution compose tool Summary: Distribution compose tool
License: GPL-2.0-only License: GPL-2.0-only
URL: https://pagure.io/pungi URL: https://pagure.io/pungi
Source0: %{name}-%{version}.tar.bz2 Source0: %{name}-%{version}.tar.bz2
Patch: https://pagure.io/pungi/pull-request/1780.patch
Patch: https://pagure.io/pungi/pull-request/1782.patch
Patch: https://pagure.io/pungi/pull-request/1788.patch
Patch: https://pagure.io/pungi/pull-request/1789.patch
Patch: https://pagure.io/pungi/pull-request/1790.patch
Patch: https://pagure.io/pungi/pull-request/1796.patch
Patch: https://pagure.io/pungi/pull-request/1798.patch
BuildRequires: make BuildRequires: make
BuildRequires: python3-pytest BuildRequires: python3-pytest
@ -141,8 +134,6 @@ gzip _build/man/pungi.1
%{__install} -d %{buildroot}%{_mandir}/man1 %{__install} -d %{buildroot}%{_mandir}/man1
%{__install} -m 0644 doc/_build/man/pungi.1.gz %{buildroot}%{_mandir}/man1 %{__install} -m 0644 doc/_build/man/pungi.1.gz %{buildroot}%{_mandir}/man1
rm %{buildroot}%{_bindir}/pungi
%check %check
%pytest %pytest
@ -181,6 +172,21 @@ rm %{buildroot}%{_bindir}/pungi
%{_bindir}/%{name}-cache-cleanup %{_bindir}/%{name}-cache-cleanup
%changelog %changelog
* Fri Nov 29 2024 Lubomír Sedlář <lsedlar@redhat.com> - 4.8.0-1
- Drop spec file (lsedlar)
- Remove python 2.7 from tox configuration (lsedlar)
- Remove forgotten multilib module for yum (lsedlar)
- Drop usage of six (lsedlar)
- Ensure ostree phase threads are stopped (lsedlar)
- scm: Clone git submodules (lsedlar)
- Drop unittest2 (lsedlar)
- Remove pungi/gather.py and associated code (lsedlar)
- Reduce legacy pungi script to gather phase only (#1792) (awilliam)
- Install dnf4 into test image (lsedlar)
- ostree_container: make filename configurable, include arch (awilliam)
- Correct subvariant handling for ostree_container phase (awilliam)
- Drop compatibility helper for dnf.Package.source_name (lsedlar)
* Tue Nov 19 2024 Adam Williamson <awilliam@redhat.com> - 4.7.0-8 * Tue Nov 19 2024 Adam Williamson <awilliam@redhat.com> - 4.7.0-8
- Backport #1798 to infer types/formats for new FEX backing images - Backport #1798 to infer types/formats for new FEX backing images

2
sources
View File

@ -1 +1 @@
SHA512 (pungi-4.7.0.tar.bz2) = 55c7527a0dff6efa8ed13b1ccdfd3628686fadb55b78fb456e552f4972b831aa96f3ff37ac54837462d91df834157f38426e6b66b52216e1e5861628df724eca SHA512 (pungi-4.8.0.tar.bz2) = 8ba996b2aeed47ef3b155fc3523519a5570fc191b0a4d779afa3ec3a0fb028ea6a21804ab195ef11eb97010042007f4aa483de51c36568653c8a2fe178b7b4ef