diff --git a/1780.patch b/1780.patch deleted file mode 100644 index fd000442..00000000 --- a/1780.patch +++ /dev/null @@ -1,214 +0,0 @@ -From d351773dab7b3aa8e6de82bbe23058b6b3448dd4 Mon Sep 17 00:00:00 2001 -From: Lubomír Sedlář -Date: Aug 26 2024 06:53:10 +0000 -Subject: kiwibuild: Add options for version and repo_releasever - - -The version follows the same rules as versioning for live media etc. -That means it's always going to be set. The precedence goes like this: - - * image specific option - * `kiwibuild_version` - * `global_version` - * `release_version` or `_`. - -Signed-off-by: Lubomír Sedlář - ---- - -diff --git a/doc/configuration.rst b/doc/configuration.rst -index b0a6d5c..c6f936a 100644 ---- a/doc/configuration.rst -+++ b/doc/configuration.rst -@@ -1570,6 +1570,13 @@ KiwiBuild Settings - * ``type_attr`` -- (*[str]*) override default attributes for the build type - from description. - * ``bundle_name_format`` -- (*str*) override default bundle format name. -+ * ``version`` -- (*str*) override version. Follows the same rules as -+ described in :ref:`automatic versioning `. -+ * ``repo_releasever`` -- (*str*) Override default releasever of the output -+ image. -+ -+ The options can be set either for the specific image, or at the phase level -+ (see below). Version also falls back to ``global_version``. - - **kiwibuild_description_scm** - (*str*) -- URL for scm containing the description files -@@ -1586,6 +1593,12 @@ KiwiBuild Settings - **kiwibuild_bundle_name_format** - (*str*) -- override default bundle format name. - -+**kiwibuild_version** -+ (*str*) -- overide version for all kiwibuild tasks. -+ -+**kiwibuild_repo_releasever** -+ (*str*) -- override releasever for all kiwibuild tasks. -+ - - OSBuild Composer for building images - ==================================== -diff --git a/pungi/checks.py b/pungi/checks.py -index dae31f5..bcf23e5 100644 ---- a/pungi/checks.py -+++ b/pungi/checks.py -@@ -1212,6 +1212,8 @@ def make_schema(): - "type": {"type": "string"}, - "type_attr": {"$ref": "#/definitions/list_of_strings"}, - "bundle_name_format": {"type": "string"}, -+ "version": {"type": "string"}, -+ "repo_releasever": {"type": "string"}, - }, - "required": [ - # description_scm and description_path -@@ -1233,6 +1235,8 @@ def make_schema(): - "kiwibuild_type": {"type": "string"}, - "kiwibuild_type_attr": {"$ref": "#/definitions/list_of_strings"}, - "kiwibuild_bundle_name_format": {"type": "string"}, -+ "kiwibuild_version": {"type": "string"}, -+ "kiwibuild_repo_releasever": {"type": "string"}, - "osbuild_target": {"type": "string"}, - "osbuild_release": {"$ref": "#/definitions/optional_string"}, - "osbuild_version": {"type": "string"}, -diff --git a/pungi/phases/kiwibuild.py b/pungi/phases/kiwibuild.py -index fabcce5..7ce3a54 100644 ---- a/pungi/phases/kiwibuild.py -+++ b/pungi/phases/kiwibuild.py -@@ -84,6 +84,8 @@ class KiwiBuildPhase( - "bundle_name_format": self.get_config( - image_conf, "bundle_name_format" - ), -+ "version": self.get_version(image_conf), -+ "repo_releasever": self.get_config(image_conf, "repo_releasever"), - } - - repo = self._get_repo(image_conf, variant) -@@ -144,6 +146,8 @@ class RunKiwiBuildThread(WorkerThread): - result_bundle_name_format=generics["bundle_name_format"], - # this ensures the task won't fail if only failable arches fail - optional_arches=self.failable_arches, -+ version=generics["version"], -+ repo_releasever=generics["repo_releasever"], - ) - - koji.save_task_id(task_id) -diff --git a/tests/test_kiwibuildphase.py b/tests/test_kiwibuildphase.py -index 237dcaf..50116a2 100644 ---- a/tests/test_kiwibuildphase.py -+++ b/tests/test_kiwibuildphase.py -@@ -50,6 +50,8 @@ class TestKiwiBuildPhase(PungiTestCase): - "type": None, - "type_attr": None, - "bundle_name_format": None, -+ "version": compose.image_version, -+ "repo_releasever": None, - }, - [self.topdir + "/compose/Server/$arch/os"], - [], # failable arches -@@ -69,6 +71,8 @@ class TestKiwiBuildPhase(PungiTestCase): - "type": "custom", - "type_attr": ["foo", "bar"], - "bundle_name_format": "fmt", -+ "version": "Rawhide", -+ "repo_releasever": "41", - }, - MINIMAL_CONF, - ) -@@ -95,6 +99,8 @@ class TestKiwiBuildPhase(PungiTestCase): - "type": "custom", - "type_attr": ["foo", "bar"], - "bundle_name_format": "fmt", -+ "version": "Rawhide", -+ "repo_releasever": "41", - }, - [ - "https://example.com/repo/", -@@ -131,6 +137,8 @@ class TestKiwiBuildPhase(PungiTestCase): - "type": None, - "type_attr": None, - "bundle_name_format": None, -+ "version": compose.image_version, -+ "repo_releasever": None, - }, - [self.topdir + "/compose/Server/$arch/os"], - ["x86_64"], # failable arches -@@ -151,6 +159,8 @@ class TestKiwiBuildPhase(PungiTestCase): - "kiwibuild_type": "custom", - "kiwibuild_type_attr": ["foo", "bar"], - "kiwibuild_bundle_name_format": "fmt", -+ "kiwibuild_version": "Rawhide", -+ "kiwibuild_repo_releasever": "41", - }, - ) - -@@ -175,6 +185,8 @@ class TestKiwiBuildPhase(PungiTestCase): - "type": "custom", - "type_attr": ["foo", "bar"], - "bundle_name_format": "fmt", -+ "version": "Rawhide", -+ "repo_releasever": "41", - }, - [self.topdir + "/compose/Server/$arch/os"], - [], # failable arches -@@ -190,6 +202,7 @@ class TestKiwiBuildPhase(PungiTestCase): - "kiwibuild": {"^Server$": [cfg]}, - "global_target": "f40", - "global_release": "1234", -+ "global_version": "41", - }, - ) - -@@ -214,6 +227,8 @@ class TestKiwiBuildPhase(PungiTestCase): - "type": None, - "type_attr": None, - "bundle_name_format": None, -+ "version": "41", -+ "repo_releasever": None, - }, - [self.topdir + "/compose/Server/$arch/os"], - [], # failable arches -@@ -278,6 +293,8 @@ class TestKiwiBuildThread(PungiTestCase): - "type": "t", - "type_attr": ["ta"], - "bundle_name_format": "fmt", -+ "version": "v", -+ "repo_releasever": "r", - }, - [self.repo], - [], -@@ -298,6 +315,8 @@ class TestKiwiBuildThread(PungiTestCase): - type_attr=["ta"], - result_bundle_name_format="fmt", - optional_arches=[], -+ version="v", -+ repo_releasever="r", - ) - ] - -@@ -362,6 +381,8 @@ class TestKiwiBuildThread(PungiTestCase): - "type": None, - "type_attr": None, - "bundle_name_format": None, -+ "version": None, -+ "repo_releasever": None, - }, - [self.repo], - [], -@@ -413,6 +434,8 @@ class TestKiwiBuildThread(PungiTestCase): - "type": None, - "type_attr": None, - "bundle_name_format": None, -+ "version": None, -+ "repo_releasever": None, - }, - [self.repo], - ["amd64"], -@@ -433,6 +456,8 @@ class TestKiwiBuildThread(PungiTestCase): - type_attr=None, - result_bundle_name_format=None, - optional_arches=["amd64"], -+ version=None, -+ repo_releasever=None, - ) - ] - assert get_image_paths.mock_calls == [mock.call(1234)] - diff --git a/1782.patch b/1782.patch deleted file mode 100644 index ce03c1f4..00000000 --- a/1782.patch +++ /dev/null @@ -1,286 +0,0 @@ -From d9d21d3cf4eaad5cc7f2959a4abdafed781bb9cf Mon Sep 17 00:00:00 2001 -From: Lubomír Sedlář -Date: Aug 28 2024 11:07:12 +0000 -Subject: [PATCH 1/2] kiwibuild: Fix location and metadata for ISOs - - -When Kiwi builds an ISO, it is always supposed to be bootable and should -be located in the iso/ subdirectory. - -Any other kind of image should still land in images/ and be listed as -not bootable in the metadata. - -Relates: https://pagure.io/pungi-fedora/issue/1342 -Signed-off-by: Lubomír Sedlář - ---- - -diff --git a/pungi/phases/kiwibuild.py b/pungi/phases/kiwibuild.py -index 7ce3a54..12ec26d 100644 ---- a/pungi/phases/kiwibuild.py -+++ b/pungi/phases/kiwibuild.py -@@ -184,10 +184,20 @@ class RunKiwiBuildThread(WorkerThread): - # image_dir is absolute path to which the image should be copied. - # We also need the same path as relative to compose directory for - # including in the metadata. -- image_dir = compose.paths.compose.image_dir(variant) % {"arch": arch} -- rel_image_dir = compose.paths.compose.image_dir( -- variant, relative=True -- ) % {"arch": arch} -+ if format_ == "iso": -+ # If the produced image is actually an ISO, it should go to -+ # iso/ subdirectory. -+ image_dir = compose.paths.compose.iso_dir(arch, variant) -+ rel_image_dir = compose.paths.compose.iso_dir( -+ arch, variant, relative=True -+ ) -+ else: -+ image_dir = compose.paths.compose.image_dir(variant) % { -+ "arch": arch -+ } -+ rel_image_dir = compose.paths.compose.image_dir( -+ variant, relative=True -+ ) % {"arch": arch} - util.makedirs(image_dir) - - filename = os.path.basename(path) -@@ -211,7 +221,8 @@ class RunKiwiBuildThread(WorkerThread): - img.arch = arch - img.disc_number = 1 # We don't expect multiple disks - img.disc_count = 1 -- img.bootable = False -+ # Kiwi produces only bootable ISOs. Other kinds of images are -+ img.bootable = format_ == "iso" - img.subvariant = config.get("subvariant", variant.uid) - setattr(img, "can_fail", arch in self.failable_arches) - setattr(img, "deliverable", "kiwibuild") -diff --git a/tests/test_kiwibuildphase.py b/tests/test_kiwibuildphase.py -index 50116a2..491dbf3 100644 ---- a/tests/test_kiwibuildphase.py -+++ b/tests/test_kiwibuildphase.py -@@ -242,13 +242,14 @@ class TestKiwiBuildPhase(PungiTestCase): - @mock.patch("pungi.util.get_file_size") - @mock.patch("pungi.wrappers.kojiwrapper.KojiWrapper") - class TestKiwiBuildThread(PungiTestCase): -- def _img_path(self, arch, filename=None): -- path = self.topdir + "/compose/Server/%s/images" % arch -+ def _img_path(self, arch, filename=None, dir=None): -+ dir = dir or "images" -+ path = self.topdir + "/compose/Server/%s/%s" % (arch, dir) - if filename: - path += "/" + filename - return path - -- def test_process(self, KojiWrapper, get_file_size, get_mtime, Linker): -+ def test_process_vagrant_box(self, KojiWrapper, get_file_size, get_mtime, Linker): - img_name = "FCBG.{arch}-Rawhide-1.6.vagrant.libvirt.box" - self.repo = self.topdir + "/compose/Server/$arch/os" - compose = DummyCompose( -@@ -353,6 +354,114 @@ class TestKiwiBuildThread(PungiTestCase): - assert "vagrant-libvirt.box" == image.format - assert "vagrant-libvirt" == image.type - assert "Test" == image.subvariant -+ assert not image.bootable -+ -+ def test_process_iso(self, KojiWrapper, get_file_size, get_mtime, Linker): -+ img_name = "FCBG.{arch}-Rawhide-1.6.iso" -+ self.repo = self.topdir + "/compose/Server/$arch/os" -+ compose = DummyCompose( -+ self.topdir, -+ { -+ "koji_profile": "koji", -+ "kiwibuild_bundle_format": "%N-%P-40_Beta-%I.%A.%T", -+ }, -+ ) -+ config = _merge({"subvariant": "Test"}, MINIMAL_CONF) -+ pool = mock.Mock() -+ -+ get_image_paths = KojiWrapper.return_value.get_image_paths -+ get_image_paths.return_value = { -+ "x86_64": [ -+ "/koji/task/1234/FCBG.x86_64-Rawhide-1.6.packages", -+ "/koji/task/1234/%s" % img_name.format(arch="x86_64"), -+ ], -+ "amd64": [ -+ "/koji/task/1234/FCBG.amd64-Rawhide-1.6.packages", -+ "/koji/task/1234/%s" % img_name.format(arch="amd64"), -+ ], -+ } -+ -+ KojiWrapper.return_value.koji_proxy.kiwiBuild.return_value = 1234 -+ KojiWrapper.return_value.watch_task.return_value = 0 -+ -+ t = RunKiwiBuildThread(pool) -+ get_file_size.return_value = 1024 -+ get_mtime.return_value = 13579 -+ t.process( -+ ( -+ compose, -+ compose.variants["Server"], -+ config, -+ ["amd64", "x86_64"], -+ { -+ "release": "1.6", -+ "target": "f40", -+ "descscm": MINIMAL_CONF["description_scm"], -+ "descpath": MINIMAL_CONF["description_path"], -+ "type": "t", -+ "type_attr": ["ta"], -+ "bundle_name_format": "fmt", -+ "version": "v", -+ "repo_releasever": "r", -+ }, -+ [self.repo], -+ [], -+ ), -+ 1, -+ ) -+ -+ assert KojiWrapper.return_value.koji_proxy.kiwiBuild.mock_calls == [ -+ mock.call( -+ "f40", -+ ["amd64", "x86_64"], -+ MINIMAL_CONF["description_scm"], -+ MINIMAL_CONF["description_path"], -+ profile=MINIMAL_CONF["kiwi_profile"], -+ release="1.6", -+ repos=[self.repo], -+ type="t", -+ type_attr=["ta"], -+ result_bundle_name_format="fmt", -+ optional_arches=[], -+ version="v", -+ repo_releasever="r", -+ ) -+ ] -+ -+ assert get_image_paths.mock_calls == [mock.call(1234)] -+ assert os.path.isdir(self._img_path("x86_64", dir="iso")) -+ assert os.path.isdir(self._img_path("amd64", dir="iso")) -+ Linker.return_value.link.assert_has_calls( -+ [ -+ mock.call( -+ "/koji/task/1234/FCBG.amd64-Rawhide-1.6.iso", -+ self._img_path("amd64", img_name.format(arch="amd64"), dir="iso"), -+ link_type="hardlink-or-copy", -+ ), -+ mock.call( -+ "/koji/task/1234/FCBG.x86_64-Rawhide-1.6.iso", -+ self._img_path("x86_64", img_name.format(arch="x86_64"), dir="iso"), -+ link_type="hardlink-or-copy", -+ ), -+ ], -+ any_order=True, -+ ) -+ -+ assert len(compose.im.add.call_args_list) == 2 -+ for call in compose.im.add.call_args_list: -+ _, kwargs = call -+ image = kwargs["image"] -+ expected_path = "Server/{0.arch}/iso/{1}".format( -+ image, img_name.format(arch=image.arch) -+ ) -+ assert kwargs["variant"] == "Server" -+ assert kwargs["arch"] in ("amd64", "x86_64") -+ assert kwargs["arch"] == image.arch -+ assert image.path == expected_path -+ assert "iso" == image.format -+ assert "iso" == image.type -+ assert image.bootable -+ assert "Test" == image.subvariant - - def test_handle_koji_fail(self, KojiWrapper, get_file_size, get_mtime, Linker): - self.repo = self.topdir + "/compose/Server/$arch/os" - -From cd2ae81e3c63316997b9617ff2e30e3148af14f2 Mon Sep 17 00:00:00 2001 -From: Lubomír Sedlář -Date: Aug 29 2024 06:46:10 +0000 -Subject: [PATCH 2/2] kiwibuild: Allow setting metadata type explicitly - - -It is not possible to reliably detect what the type for an image should -be in the metadata. This commit adds an option for user to explicitly -provide it. - -It can only be configured on the specific image, not globally. - -Signed-off-by: Lubomír Sedlář - ---- - -diff --git a/doc/configuration.rst b/doc/configuration.rst -index c6f936a..ded479e 100644 ---- a/doc/configuration.rst -+++ b/doc/configuration.rst -@@ -1574,6 +1574,9 @@ KiwiBuild Settings - described in :ref:`automatic versioning `. - * ``repo_releasever`` -- (*str*) Override default releasever of the output - image. -+ * ``manifest_type`` -- the image type that is put into the manifest by -+ pungi. If not supplied, an autodetected value will be provided. It may or -+ may not make sense. - - The options can be set either for the specific image, or at the phase level - (see below). Version also falls back to ``global_version``. -diff --git a/pungi/checks.py b/pungi/checks.py -index bcf23e5..cc16d12 100644 ---- a/pungi/checks.py -+++ b/pungi/checks.py -@@ -1214,6 +1214,7 @@ def make_schema(): - "bundle_name_format": {"type": "string"}, - "version": {"type": "string"}, - "repo_releasever": {"type": "string"}, -+ "manifest_type": {"type": "string"}, - }, - "required": [ - # description_scm and description_path -diff --git a/pungi/phases/kiwibuild.py b/pungi/phases/kiwibuild.py -index 12ec26d..bb26812 100644 ---- a/pungi/phases/kiwibuild.py -+++ b/pungi/phases/kiwibuild.py -@@ -211,9 +211,9 @@ class RunKiwiBuildThread(WorkerThread): - # Update image manifest - img = Image(compose.im) - -- # Get the manifest type from the config if supplied, otherwise we -- # determine the manifest type based on the koji output -- img.type = type_ -+ # If user configured exact type, use it, otherwise try to -+ # figure it out based on the koji output. -+ img.type = config.get("manifest_type", type_) - img.format = format_ - img.path = os.path.join(rel_image_dir, filename) - img.mtime = util.get_mtime(image_dest) -diff --git a/tests/test_kiwibuildphase.py b/tests/test_kiwibuildphase.py -index 491dbf3..e3ccdb7 100644 ---- a/tests/test_kiwibuildphase.py -+++ b/tests/test_kiwibuildphase.py -@@ -73,6 +73,7 @@ class TestKiwiBuildPhase(PungiTestCase): - "bundle_name_format": "fmt", - "version": "Rawhide", - "repo_releasever": "41", -+ "manifest_type": "live-kiwi", - }, - MINIMAL_CONF, - ) -@@ -366,7 +367,9 @@ class TestKiwiBuildThread(PungiTestCase): - "kiwibuild_bundle_format": "%N-%P-40_Beta-%I.%A.%T", - }, - ) -- config = _merge({"subvariant": "Test"}, MINIMAL_CONF) -+ config = _merge( -+ {"subvariant": "Test", "manifest_type": "live-kiwi"}, MINIMAL_CONF -+ ) - pool = mock.Mock() - - get_image_paths = KojiWrapper.return_value.get_image_paths -@@ -459,7 +462,7 @@ class TestKiwiBuildThread(PungiTestCase): - assert kwargs["arch"] == image.arch - assert image.path == expected_path - assert "iso" == image.format -- assert "iso" == image.type -+ assert "live-kiwi" == image.type - assert image.bootable - assert "Test" == image.subvariant - - diff --git a/1788.patch b/1788.patch deleted file mode 100644 index 3f081c0c..00000000 --- a/1788.patch +++ /dev/null @@ -1,211 +0,0 @@ -From 5338d3098ccd614a8fd32f837a393aed78b471bd Mon Sep 17 00:00:00 2001 -From: Adam Williamson -Date: Sep 20 2024 06:03:15 +0000 -Subject: [PATCH 1/2] move osbuild/kiwi-specific EXTENSIONS to each phase - - -The image-build phase's EXTENSIONS dict is meant to exactly -mirror the 'formats' that exist in the context of the command -`koji image-build`, which is driven by this phase. That nice -association was lost, however, by adding a couple of items to it -which exist for the purposes of the osbuild phase (and in the -case of .iso, also the kiwibuild phase), which import this dict -and uses it for image identification. - -To make the association 1:1 again and more clearly show what's -going on here, let's move those entries out into the osbuild and -kiwi phases. osbuild now has its own dict which starts out as a -copy of the image-build one before being extended. And let's -update the relevant comments. - -Signed-off-by: Adam Williamson - ---- - -diff --git a/pungi/phases/image_build.py b/pungi/phases/image_build.py -index cb76c1a..cd034cd 100644 ---- a/pungi/phases/image_build.py -+++ b/pungi/phases/image_build.py -@@ -22,10 +22,13 @@ from productmd.rpms import Rpms - # This is a mapping from formats to file extensions. The format is what koji - # image-build command expects as argument, and the extension is what the file - # name will be ending with. The extensions are used to filter out which task --# results will be pulled into the compose. -+# results will be pulled into the compose. This dict is also used later in -+# the process to set the image 'type' in productmd metadata terms - the type -+# is set as the first key in this dict which has the file's extension in its -+# values. This dict is imported and extended for similar purposes by other -+# phases (at least osbuild and kiwibuild). - EXTENSIONS = { - "docker": ["tar.gz", "tar.xz"], -- "iso": ["iso"], - "liveimg-squashfs": ["liveimg.squashfs"], - "qcow": ["qcow"], - "qcow2": ["qcow2"], -@@ -40,7 +43,6 @@ EXTENSIONS = { - "vdi": ["vdi"], - "vmdk": ["vmdk"], - "vpc": ["vhd"], -- "vhd-compressed": ["vhd.gz", "vhd.xz"], - "vsphere-ova": ["vsphere.ova"], - } - -diff --git a/pungi/phases/kiwibuild.py b/pungi/phases/kiwibuild.py -index bb26812..15732ce 100644 ---- a/pungi/phases/kiwibuild.py -+++ b/pungi/phases/kiwibuild.py -@@ -15,6 +15,10 @@ KIWIEXTENSIONS = [ - ("vhd-compressed", ["vhdfixed.xz"], "vhd.xz"), - ("vagrant-libvirt", ["vagrant.libvirt.box"], "vagrant-libvirt.box"), - ("vagrant-virtualbox", ["vagrant.virtualbox.box"], "vagrant-virtualbox.box"), -+ # .iso images can be of many types - boot, cd, dvd, live... - -+ # so 'boot' is just a default guess. 'iso' is not a valid -+ # productmd image type -+ ("boot", [".iso"], "iso"), - ] - - -diff --git a/pungi/phases/osbuild.py b/pungi/phases/osbuild.py -index 6c5b7e5..a3b5c9b 100644 ---- a/pungi/phases/osbuild.py -+++ b/pungi/phases/osbuild.py -@@ -11,6 +11,16 @@ from ..linker import Linker - from ..wrappers import kojiwrapper - from .image_build import EXTENSIONS - -+# copy and modify EXTENSIONS with some that osbuild produces but which -+# do not exist as `koji image-build` formats -+OSBUILDEXTENSIONS = EXTENSIONS.copy() -+OSBUILDEXTENSIONS.update( -+ { -+ "iso": ["iso"], -+ "vhd-compressed": ["vhd.gz", "vhd.xz"], -+ } -+) -+ - - class OSBuildPhase( - base.PhaseLoggerMixin, base.ImageConfigMixin, base.ConfigGuardedPhase -@@ -203,7 +213,7 @@ class RunOSBuildThread(WorkerThread): - # architecture, but we don't verify that. - build_info = koji.koji_proxy.getBuild(build_id) - for archive in koji.koji_proxy.listArchives(buildID=build_id): -- if archive["type_name"] not in EXTENSIONS: -+ if archive["type_name"] not in OSBUILDEXTENSIONS: - # Ignore values that are not of required types. - continue - -@@ -241,7 +251,7 @@ class RunOSBuildThread(WorkerThread): - - linker.link(src_file, image_dest, link_type=compose.conf["link_type"]) - -- for suffix in EXTENSIONS[archive["type_name"]]: -+ for suffix in OSBUILDEXTENSIONS[archive["type_name"]]: - if archive["filename"].endswith(suffix): - break - else: - -From 739062ed3c471e74ba9c5144c4047f67f9fbe8c8 Mon Sep 17 00:00:00 2001 -From: Adam Williamson -Date: Sep 20 2024 06:03:58 +0000 -Subject: [PATCH 2/2] image_build: drop .tar.gz as an expected extension for docker - - -Koji's image-build command has not been capable of producing a -docker image with .tar.gz as its extension since 2015: - -https://pagure.io/koji/c/b489f282bee7a008108534404dd2e78efb2256e7?branch=master - -as that commit message implies, the files have not actually been -gzip-compressed for even longer: - -https://pagure.io/koji/c/82a405c7943192e3bba3340efe7a8d07a0e26b70?branch=master - -so there's no point to having this any more. It is causing the -wrong productmd 'type' to be set for GCE cloud images, which *do* -have the .tar.gz extension - because docker appears in this dict -before tar-gz, their type is being set as 'docker' not 'tar-gz'. - -Signed-off-by: Adam Williamson - ---- - -diff --git a/pungi/phases/image_build.py b/pungi/phases/image_build.py -index cd034cd..c28ccc2 100644 ---- a/pungi/phases/image_build.py -+++ b/pungi/phases/image_build.py -@@ -28,7 +28,7 @@ from productmd.rpms import Rpms - # values. This dict is imported and extended for similar purposes by other - # phases (at least osbuild and kiwibuild). - EXTENSIONS = { -- "docker": ["tar.gz", "tar.xz"], -+ "docker": ["tar.xz"], - "liveimg-squashfs": ["liveimg.squashfs"], - "qcow": ["qcow"], - "qcow2": ["qcow2"], -diff --git a/tests/test_imagebuildphase.py b/tests/test_imagebuildphase.py -index 90f10ef..895f040 100644 ---- a/tests/test_imagebuildphase.py -+++ b/tests/test_imagebuildphase.py -@@ -846,12 +846,12 @@ class TestCreateImageBuildThread(PungiTestCase): - "amd64": [ - "/koji/task/1235/tdl-amd64.xml", - "/koji/task/1235/Fedora-Docker-Base-20160103.amd64.qcow2", -- "/koji/task/1235/Fedora-Docker-Base-20160103.amd64.tar.gz", -+ "/koji/task/1235/Fedora-Docker-Base-20160103.amd64.tar.xz", - ], - "x86_64": [ - "/koji/task/1235/tdl-x86_64.xml", - "/koji/task/1235/Fedora-Docker-Base-20160103.x86_64.qcow2", -- "/koji/task/1235/Fedora-Docker-Base-20160103.x86_64.tar.gz", -+ "/koji/task/1235/Fedora-Docker-Base-20160103.x86_64.tar.xz", - ], - } - -@@ -896,9 +896,9 @@ class TestCreateImageBuildThread(PungiTestCase): - link_type="hardlink-or-copy", - ), - mock.call.link( -- "/koji/task/1235/Fedora-Docker-Base-20160103.amd64.tar.gz", -+ "/koji/task/1235/Fedora-Docker-Base-20160103.amd64.tar.xz", - self.topdir -- + "/compose/Client/amd64/images/Fedora-Docker-Base-20160103.amd64.tar.gz", # noqa: E501 -+ + "/compose/Client/amd64/images/Fedora-Docker-Base-20160103.amd64.tar.xz", # noqa: E501 - link_type="hardlink-or-copy", - ), - mock.call.link( -@@ -908,9 +908,9 @@ class TestCreateImageBuildThread(PungiTestCase): - link_type="hardlink-or-copy", - ), - mock.call.link( -- "/koji/task/1235/Fedora-Docker-Base-20160103.x86_64.tar.gz", -+ "/koji/task/1235/Fedora-Docker-Base-20160103.x86_64.tar.xz", - self.topdir -- + "/compose/Client/x86_64/images/Fedora-Docker-Base-20160103.x86_64.tar.gz", # noqa: E501 -+ + "/compose/Client/x86_64/images/Fedora-Docker-Base-20160103.x86_64.tar.xz", # noqa: E501 - link_type="hardlink-or-copy", - ), - ], -@@ -922,8 +922,8 @@ class TestCreateImageBuildThread(PungiTestCase): - "type": "qcow2", - "arch": "amd64", - }, -- "image_dir/Client/amd64/Fedora-Docker-Base-20160103.amd64.tar.gz": { -- "format": "tar.gz", -+ "image_dir/Client/amd64/Fedora-Docker-Base-20160103.amd64.tar.xz": { -+ "format": "tar.xz", - "type": "docker", - "arch": "amd64", - }, -@@ -932,8 +932,8 @@ class TestCreateImageBuildThread(PungiTestCase): - "type": "qcow2", - "arch": "x86_64", - }, -- "image_dir/Client/x86_64/Fedora-Docker-Base-20160103.x86_64.tar.gz": { -- "format": "tar.gz", -+ "image_dir/Client/x86_64/Fedora-Docker-Base-20160103.x86_64.tar.xz": { -+ "format": "tar.xz", - "type": "docker", - "arch": "x86_64", - }, - diff --git a/1789.patch b/1789.patch deleted file mode 100644 index 9796a5d7..00000000 --- a/1789.patch +++ /dev/null @@ -1,146 +0,0 @@ -From 391a5eaed5198e5ee2941dac4ae43a2fe057eedd Mon Sep 17 00:00:00 2001 -From: Adam Williamson -Date: Oct 09 2024 23:28:54 +0000 -Subject: [PATCH 1/2] Correct subvariant handling for ostree_container phase - - -The image metadata construction code allows for subvariant to be -set in the image config dict, but checks.py doesn't expect it; -fix that. Also, when a subvariant is set, use it in the image -name template rather than the variant; otherwise you can't -build more than one subvariant in any variant (they will have -identical names, which isn't alllowed). - -Signed-off-by: Adam Williamson - ---- - -diff --git a/doc/configuration.rst b/doc/configuration.rst -index ded479e..90621bb 100644 ---- a/doc/configuration.rst -+++ b/doc/configuration.rst -@@ -1886,6 +1886,10 @@ will thus create a new OCI archive image *from scratch*. - reference will not be created. - * ``runroot_packages`` -- (*list*) A list of additional package names to be - installed in the runroot environment in Koji. -+ * ``subvariant`` -- (*str*) The subvariant value to be used in the metadata -+ for the image. Also used in the image's filename. Defaults to being the -+ same as the variant. If building more than one ostree container in a -+ variant, each must have a unique subvariant. - - Example config - -------------- -diff --git a/pungi/checks.py b/pungi/checks.py -index cc16d12..e23babc 100644 ---- a/pungi/checks.py -+++ b/pungi/checks.py -@@ -1101,6 +1101,7 @@ def make_schema(): - "runroot_packages": { - "$ref": "#/definitions/list_of_strings", - }, -+ "subvariant": {"type": "string"}, - }, - "required": [ - "treefile", -diff --git a/pungi/phases/ostree_container.py b/pungi/phases/ostree_container.py -index 38a647e..e111998 100644 ---- a/pungi/phases/ostree_container.py -+++ b/pungi/phases/ostree_container.py -@@ -119,12 +119,13 @@ class OSTreeContainerThread(WorkerThread): - def _run_ostree_container_cmd( - self, compose, variant, arch, config, config_repo, extra_config_file=None - ): -+ subvariant = config.get("subvariant", variant.uid) - target_dir = compose.paths.compose.image_dir(variant) % {"arch": arch} - util.makedirs(target_dir) - version = util.version_generator(compose, config.get("version")) - archive_name = "%s-%s-%s" % ( - compose.conf["release_short"], -- variant.uid, -+ subvariant, - version, - ) - -@@ -177,7 +178,7 @@ class OSTreeContainerThread(WorkerThread): - img.disc_number = 1 - img.disc_count = 1 - img.bootable = False -- img.subvariant = config.get("subvariant", variant.uid) -+ img.subvariant = subvariant - setattr(img, "can_fail", self.can_fail) - setattr(img, "deliverable", "ostree-container") - compose.im.add(variant=variant.uid, arch=arch, image=img) - -From aea8da5225aeb31b4c5dd413f0a31b6ab395a9ac Mon Sep 17 00:00:00 2001 -From: Adam Williamson -Date: Oct 09 2024 23:33:48 +0000 -Subject: [PATCH 2/2] ostree_container: make filename configurable, include arch - - -The default base name is probably fine in most cases, but there -are some where we might want to tweak it. We already allow this -for other phases (e.g. the livemedia phase). - -Also, we should include the arch in the image filename. Not doing -this doesn't blow up the compose as, while they have identical -filenames, the images for different arches are in different paths, -but it's confusing for people who actually download and use the -images. - -Signed-off-by: Adam Williamson - ---- - -diff --git a/doc/configuration.rst b/doc/configuration.rst -index 90621bb..14947fe 100644 ---- a/doc/configuration.rst -+++ b/doc/configuration.rst -@@ -1887,9 +1887,14 @@ will thus create a new OCI archive image *from scratch*. - * ``runroot_packages`` -- (*list*) A list of additional package names to be - installed in the runroot environment in Koji. - * ``subvariant`` -- (*str*) The subvariant value to be used in the metadata -- for the image. Also used in the image's filename. Defaults to being the -- same as the variant. If building more than one ostree container in a -- variant, each must have a unique subvariant. -+ for the image. Also used in the image's filename, unless overridden by -+ ``name``. Defaults to being the same as the variant. If building more -+ than one ostree container in a variant, each must have a unique -+ subvariant. -+ * ``name`` -- (*str*) The base for the image's filename. To produce the -+ complete filename, the image's architecture, the version string, and the -+ format suffix are appended to this. Defaults to the value of -+ ``release_short`` and the subvariant, joined by a dash. - - Example config - -------------- -diff --git a/pungi/checks.py b/pungi/checks.py -index e23babc..0cdb619 100644 ---- a/pungi/checks.py -+++ b/pungi/checks.py -@@ -1102,6 +1102,7 @@ def make_schema(): - "$ref": "#/definitions/list_of_strings", - }, - "subvariant": {"type": "string"}, -+ "name": {"type": "string"}, - }, - "required": [ - "treefile", -diff --git a/pungi/phases/ostree_container.py b/pungi/phases/ostree_container.py -index e111998..c50633d 100644 ---- a/pungi/phases/ostree_container.py -+++ b/pungi/phases/ostree_container.py -@@ -123,11 +123,8 @@ class OSTreeContainerThread(WorkerThread): - target_dir = compose.paths.compose.image_dir(variant) % {"arch": arch} - util.makedirs(target_dir) - version = util.version_generator(compose, config.get("version")) -- archive_name = "%s-%s-%s" % ( -- compose.conf["release_short"], -- subvariant, -- version, -- ) -+ anb = config.get("name", "%s-%s" % (compose.conf["release_short"], subvariant)) -+ archive_name = "%s-%s-%s" % (anb, arch, version) - - # Run the pungi-make-ostree command locally to create a script to - # execute in runroot environment. - diff --git a/1790.patch b/1790.patch deleted file mode 100644 index 371035ae..00000000 --- a/1790.patch +++ /dev/null @@ -1,82 +0,0 @@ -From b83ac496b5a20bd0493328999fc619711c6a512a Mon Sep 17 00:00:00 2001 -From: Adam Williamson -Date: Oct 11 2024 18:18:48 +0000 -Subject: Don't block main image build phase on ostree_install phase - - -I did a time map of a Fedora compose today, and noticed that we -spend about an hour waiting for the ostree_install phase to -complete before we start up the compose_images_phase which does -all the other image builds. - -This is unnecessary. Nothing else depends on ostree_install; it -should be fine to start up the extra_phase (which contains -compose_images_phase) while the ostree stuff is still running. - -This implements that by splitting the ostree phases out of the -essentials_phase which contains the real precursors to the -extra_phase. We start the essentials and ostree phases together, -but only wait for the essentials phase to complete before -kicking off extra_phase, so it can start while the ostree -phase is still running. - -One tweak we have to make to accommodate this is to move -image_checksum_phase out of extra_phase, to avoid it potentially -running before all ostree installer images are built. The -checksum phase is quite fast - it takes about five minutes - -and any time benefit of running it in parallel with the osbs and -repoclosure phases seems like it must be smaller than the time -loss of waiting for ostree_install before kicking off extra. - -Signed-off-by: Adam Williamson - ---- - -diff --git a/pungi/scripts/pungi_koji.py b/pungi/scripts/pungi_koji.py -index c643d53..b2382b0 100644 ---- a/pungi/scripts/pungi_koji.py -+++ b/pungi/scripts/pungi_koji.py -@@ -476,11 +476,15 @@ def run_compose( - buildinstall_phase, - (gather_phase, createrepo_phase), - extrafiles_phase, -+ ) -+ ostree_schema = ( - (ostree_phase, ostree_installer_phase), - ostree_container_phase, - ) - essentials_phase = pungi.phases.WeaverPhase(compose, essentials_schema) -+ ostree_phase = pungi.phases.WeaverPhase(compose, ostree_schema) - essentials_phase.start() -+ ostree_phase.start() - essentials_phase.stop() - - # write treeinfo before ISOs are created -@@ -507,12 +511,9 @@ def run_compose( - osbuild_phase, - kiwibuild_phase, - ) -- post_image_phase = pungi.phases.WeaverPhase( -- compose, (image_checksum_phase, image_container_phase) -- ) - compose_images_phase = pungi.phases.WeaverPhase(compose, compose_images_schema) - extra_phase_schema = ( -- (compose_images_phase, post_image_phase), -+ (compose_images_phase, image_container_phase), - osbs_phase, - repoclosure_phase, - ) -@@ -520,6 +521,12 @@ def run_compose( - - extra_phase.start() - extra_phase.stop() -+ # wait for ostree phase here too - it can happily run in parallel -+ # with all of the other stuff -+ ostree_phase.stop() -+ # now we do checksums as all images are done -+ image_checksum_phase.start() -+ image_checksum_phase.stop() - - pungi.metadata.write_compose_info(compose) - if not ( - diff --git a/1796.patch b/1796.patch deleted file mode 100644 index 30d9a5c4..00000000 --- a/1796.patch +++ /dev/null @@ -1,60 +0,0 @@ -From c8fe99b1aa5a9a9b941b7515cda367d24829dedf Mon Sep 17 00:00:00 2001 -From: Adam Williamson -Date: Nov 19 2024 08:22:09 +0000 -Subject: pkgset: optimize cache check (saves 20 minutes) - - -The pkgset phase takes around 35 minutes in current composes. -Around 20 minutes of that is spent creating these per-arch -subsets of the global package set. In a rather roundabout way -(see #1794 ), I figured out that almost all of this time is -spent in this cache check, which is broken for a subtle reason. - -Python's `in` keyword works by first attempting to call the -container's magic `__contains__` method. If the container does -not implement `__contains__`, it falls back to iteration - it -tries to iterate over the container until it either hits what -it's looking for, or runs out. (If the container implements -neither, you get an error). - -The FileCache instance's `file_cache` is a plain Python dict. -dicts have a very efficient `__contains__` implementation, so -doing `foo in (somedict)` is basically always very fast no matter -how huge the dict is. FileCache itself, though, implements -`__iter__` by returning an iterator over the `file_cache` dict's -keys, but it does *not* implement `__contains__`. So when we do -`foo in self.file_cache`, Python has to iterate over every key -in the dict until it hits foo or runs out. This is massively -slower than `foo in self.file_cache.file_cache`, which uses the -efficient `__contains__` method. - -Because these package sets are so huge, and we're looping over -*one* huge set and checking each package from it against the cache -of another, increasingly huge, set, this effect becomes massive. -To make it even worse, I ran a few tests where I added a debug log -if we ever hit the cache, and it looks like we never actually do - -so every check has to iterate through the entire dict. - -We could probably remove this entirely, but changing it to check -the dict instead of the FileCache instance makes it just about as -fast as taking it out, so I figured let's go with that in case -there's some unusual scenario in which the cache does work here. - -Signed-off-by: Adam Williamson - ---- - -diff --git a/pungi/phases/pkgset/pkgsets.py b/pungi/phases/pkgset/pkgsets.py -index 81c090c..ee12c3c 100644 ---- a/pungi/phases/pkgset/pkgsets.py -+++ b/pungi/phases/pkgset/pkgsets.py -@@ -265,7 +265,7 @@ class PackageSetBase(kobo.log.LoggingBase): - for arch in arch_list: - self.rpms_by_arch.setdefault(arch, []) - for i in other.rpms_by_arch.get(arch, []): -- if i.file_path in self.file_cache: -+ if i.file_path in self.file_cache.file_cache: - # TODO: test if it really works - continue - if inherit_to_noarch and exclusivearch_list and arch == "noarch": - diff --git a/1798.patch b/1798.patch deleted file mode 100644 index fb55d2d0..00000000 --- a/1798.patch +++ /dev/null @@ -1,28 +0,0 @@ -From eb4ba5f637153f0037f05981adea8b35fc0f6b25 Mon Sep 17 00:00:00 2001 -From: Adam Williamson -Date: Nov 20 2024 00:49:30 +0000 -Subject: kiwibuild: extend productmd type/format detection for FEX images - - -Signed-off-by: Adam Williamson - ---- - -diff --git a/pungi/phases/kiwibuild.py b/pungi/phases/kiwibuild.py -index 15732ce..400ea6a 100644 ---- a/pungi/phases/kiwibuild.py -+++ b/pungi/phases/kiwibuild.py -@@ -19,6 +19,12 @@ KIWIEXTENSIONS = [ - # so 'boot' is just a default guess. 'iso' is not a valid - # productmd image type - ("boot", [".iso"], "iso"), -+ ("fex", ["erofs.xz"], "erofs.xz"), -+ ("fex", ["erofs.gz"], "erofs.gz"), -+ ("fex", ["erofs"], "erofs"), -+ ("fex", ["squashfs.xz"], "squashfs.xz"), -+ ("fex", ["squashfs.gz"], "squashfs.gz"), -+ ("fex", ["squashfs"], "squashfs"), - ] - - - diff --git a/pungi.spec b/pungi.spec index 86aa3367..6858ad9b 100644 --- a/pungi.spec +++ b/pungi.spec @@ -1,20 +1,13 @@ %{?python_enable_dependency_generator} Name: pungi -Version: 4.7.0 -Release: 8%{?dist}.alma.1 +Version: 4.8.0 +Release: 1%{?dist}.alma.1 Summary: Distribution compose tool License: GPL-2.0-only URL: https://pagure.io/pungi Source0: %{name}-%{version}.tar.bz2 -Patch: https://pagure.io/pungi/pull-request/1780.patch -Patch: https://pagure.io/pungi/pull-request/1782.patch -Patch: https://pagure.io/pungi/pull-request/1788.patch -Patch: https://pagure.io/pungi/pull-request/1789.patch -Patch: https://pagure.io/pungi/pull-request/1790.patch -Patch: https://pagure.io/pungi/pull-request/1796.patch -Patch: https://pagure.io/pungi/pull-request/1798.patch BuildRequires: make BuildRequires: python3-pytest @@ -141,8 +134,6 @@ gzip _build/man/pungi.1 %{__install} -d %{buildroot}%{_mandir}/man1 %{__install} -m 0644 doc/_build/man/pungi.1.gz %{buildroot}%{_mandir}/man1 -rm %{buildroot}%{_bindir}/pungi - %check %pytest @@ -181,6 +172,21 @@ rm %{buildroot}%{_bindir}/pungi %{_bindir}/%{name}-cache-cleanup %changelog +* Fri Nov 29 2024 Lubomír Sedlář - 4.8.0-1 +- Drop spec file (lsedlar) +- Remove python 2.7 from tox configuration (lsedlar) +- Remove forgotten multilib module for yum (lsedlar) +- Drop usage of six (lsedlar) +- Ensure ostree phase threads are stopped (lsedlar) +- scm: Clone git submodules (lsedlar) +- Drop unittest2 (lsedlar) +- Remove pungi/gather.py and associated code (lsedlar) +- Reduce legacy pungi script to gather phase only (#1792) (awilliam) +- Install dnf4 into test image (lsedlar) +- ostree_container: make filename configurable, include arch (awilliam) +- Correct subvariant handling for ostree_container phase (awilliam) +- Drop compatibility helper for dnf.Package.source_name (lsedlar) + * Tue Nov 19 2024 Adam Williamson - 4.7.0-8 - Backport #1798 to infer types/formats for new FEX backing images diff --git a/sources b/sources index 4f8b3065..3de551b2 100644 --- a/sources +++ b/sources @@ -1 +1 @@ -SHA512 (pungi-4.7.0.tar.bz2) = 55c7527a0dff6efa8ed13b1ccdfd3628686fadb55b78fb456e552f4972b831aa96f3ff37ac54837462d91df834157f38426e6b66b52216e1e5861628df724eca +SHA512 (pungi-4.8.0.tar.bz2) = 8ba996b2aeed47ef3b155fc3523519a5570fc191b0a4d779afa3ec3a0fb028ea6a21804ab195ef11eb97010042007f4aa483de51c36568653c8a2fe178b7b4ef