From e42e65783d51a803d6b5fdced8c38b17e9861b9e Mon Sep 17 00:00:00 2001 From: Haibo Lin Date: Fri, 20 Aug 2021 19:35:36 +0800 Subject: [PATCH] image_build: Allow reusing old image_build results JIRA: RHELCMP-5970 Signed-off-by: Haibo Lin --- pungi/checks.py | 1 + pungi/phases/image_build.py | 225 ++++++++++++++++++++++++- pungi/scripts/pungi_koji.py | 2 +- tests/test_imagebuildphase.py | 307 +++++++++++++++++----------------- 4 files changed, 369 insertions(+), 166 deletions(-) diff --git a/pungi/checks.py b/pungi/checks.py index a2434fce..c79f2bbc 100644 --- a/pungi/checks.py +++ b/pungi/checks.py @@ -1080,6 +1080,7 @@ def make_schema(): "live_images": _variant_arch_mapping( _one_or_list({"$ref": "#/definitions/live_image_config"}) ), + "image_build_allow_reuse": {"type": "boolean", "default": False}, "image_build": { "type": "object", "patternProperties": { diff --git a/pungi/phases/image_build.py b/pungi/phases/image_build.py index 127ecf1d..e0dcb02b 100644 --- a/pungi/phases/image_build.py +++ b/pungi/phases/image_build.py @@ -1,18 +1,22 @@ # -*- coding: utf-8 -*- import copy +import hashlib +import json import os +import shutil import time from kobo import shortcuts from pungi.util import makedirs, get_mtime, get_file_size, failable, log_failed_task -from pungi.util import translate_path, get_repo_urls, version_generator +from pungi.util import as_local_file, translate_path, get_repo_urls, version_generator from pungi.phases import base from pungi.linker import Linker from pungi.wrappers.kojiwrapper import KojiWrapper from kobo.threads import ThreadPool, WorkerThread from kobo.shortcuts import force_list from productmd.images import Image +from productmd.rpms import Rpms # This is a mapping from formats to file extensions. The format is what koji @@ -46,9 +50,10 @@ class ImageBuildPhase( name = "image_build" - def __init__(self, compose): + def __init__(self, compose, buildinstall_phase=None): super(ImageBuildPhase, self).__init__(compose) self.pool = ThreadPool(logger=self.logger) + self.buildinstall_phase = buildinstall_phase def _get_install_tree(self, image_conf, variant): """ @@ -117,6 +122,7 @@ class ImageBuildPhase( # prevent problems in next iteration where the original # value is needed. image_conf = copy.deepcopy(image_conf) + original_image_conf = copy.deepcopy(image_conf) # image_conf is passed to get_image_build_cmd as dict @@ -167,6 +173,7 @@ class ImageBuildPhase( image_conf["image-build"]["can_fail"] = sorted(can_fail) cmd = { + "original_image_conf": original_image_conf, "image_conf": image_conf, "conf_file": self.compose.paths.work.image_build_conf( image_conf["image-build"]["variant"], @@ -182,7 +189,7 @@ class ImageBuildPhase( "scratch": image_conf["image-build"].pop("scratch", False), } self.pool.add(CreateImageBuildThread(self.pool)) - self.pool.queue_put((self.compose, cmd)) + self.pool.queue_put((self.compose, cmd, self.buildinstall_phase)) self.pool.start() @@ -192,7 +199,7 @@ class CreateImageBuildThread(WorkerThread): self.pool.log_error("CreateImageBuild failed.") def process(self, item, num): - compose, cmd = item + compose, cmd, buildinstall_phase = item variant = cmd["image_conf"]["image-build"]["variant"] subvariant = cmd["image_conf"]["image-build"].get("subvariant", variant.uid) self.failable_arches = cmd["image_conf"]["image-build"].get("can_fail", "") @@ -208,15 +215,47 @@ class CreateImageBuildThread(WorkerThread): subvariant, logger=self.pool._logger, ): - self.worker(num, compose, variant, subvariant, cmd) + self.worker(num, compose, variant, subvariant, cmd, buildinstall_phase) - def worker(self, num, compose, variant, subvariant, cmd): + def worker(self, num, compose, variant, subvariant, cmd, buildinstall_phase): arches = cmd["image_conf"]["image-build"]["arches"] formats = "-".join(cmd["image_conf"]["image-build"]["format"]) dash_arches = "-".join(arches) log_file = compose.paths.log.log_file( dash_arches, "imagebuild-%s-%s-%s" % (variant.uid, subvariant, formats) ) + metadata_file = log_file[:-4] + ".reuse.json" + + external_repo_checksum = {} + try: + for repo in cmd["original_image_conf"]["image-build"]["repo"]: + if repo in compose.all_variants: + continue + with as_local_file( + os.path.join(repo, "repodata/repomd.xml") + ) as filename: + with open(filename, "rb") as f: + external_repo_checksum[repo] = hashlib.sha256( + f.read() + ).hexdigest() + except Exception as e: + external_repo_checksum = None + self.pool.log_info( + "Can't calculate checksum of repomd.xml of external repo - %s" % str(e) + ) + + if self._try_to_reuse( + compose, + variant, + subvariant, + metadata_file, + log_file, + cmd, + external_repo_checksum, + buildinstall_phase, + ): + return + msg = ( "Creating image (formats: %s, arches: %s, variant: %s, subvariant: %s)" % (formats, dash_arches, variant, subvariant) @@ -275,6 +314,22 @@ class CreateImageBuildThread(WorkerThread): ) break + self._link_images(compose, variant, subvariant, cmd, image_infos) + self._write_reuse_metadata( + compose, metadata_file, cmd, image_infos, external_repo_checksum + ) + + self.pool.log_info("[DONE ] %s (task id: %s)" % (msg, output["task_id"])) + + def _link_images(self, compose, variant, subvariant, cmd, image_infos): + """Link images to compose and update image manifest. + + :param Compose compose: Current compose. + :param Variant variant: Current variant. + :param str subvariant: + :param dict cmd: Dict of params for image-build. + :param dict image_infos: Dict contains image info. + """ # The usecase here is that you can run koji image-build with multiple --format # It's ok to do it serialized since we're talking about max 2 images per single # image_build record @@ -308,4 +363,160 @@ class CreateImageBuildThread(WorkerThread): setattr(img, "deliverable", "image-build") compose.im.add(variant=variant.uid, arch=image_info["arch"], image=img) - self.pool.log_info("[DONE ] %s (task id: %s)" % (msg, output["task_id"])) + def _try_to_reuse( + self, + compose, + variant, + subvariant, + metadata_file, + log_file, + cmd, + external_repo_checksum, + buildinstall_phase, + ): + """Try to reuse images from old compose. + + :param Compose compose: Current compose. + :param Variant variant: Current variant. + :param str subvariant: + :param str metadata_file: Path to reuse metadata file. + :param str log_file: Path to log file. + :param dict cmd: Dict of params for image-build. + :param dict external_repo_checksum: Dict contains checksum of repomd.xml + or None if can't get checksum. + :param BuildinstallPhase buildinstall_phase: buildinstall phase of + current compose. + """ + log_msg = "Cannot reuse old image_build phase results - %s" + if not compose.conf["image_build_allow_reuse"]: + self.pool.log_info( + log_msg % "reuse of old image_build results is disabled." + ) + return False + + if external_repo_checksum is None: + self.pool.log_info( + log_msg % "Can't ensure that external repo is not changed." + ) + return False + + old_metadata_file = compose.paths.old_compose_path(metadata_file) + if not old_metadata_file: + self.pool.log_info(log_msg % "Can't find old reuse metadata file") + return False + + try: + old_metadata = self._load_reuse_metadata(old_metadata_file) + except Exception as e: + self.pool.log_info( + log_msg % "Can't load old reuse metadata file: %s" % str(e) + ) + return False + + if old_metadata["cmd"]["original_image_conf"] != cmd["original_image_conf"]: + self.pool.log_info(log_msg % "image_build config changed") + return False + + # Make sure external repo does not change + if ( + old_metadata["external_repo_checksum"] is None + or old_metadata["external_repo_checksum"] != external_repo_checksum + ): + self.pool.log_info(log_msg % "External repo may be changed") + return False + + # Make sure buildinstall phase is reused + for arch in cmd["image_conf"]["image-build"]["arches"]: + if buildinstall_phase and not buildinstall_phase.reused(variant, arch): + self.pool.log_info(log_msg % "buildinstall phase changed") + return False + + # Make sure packages in variant not change + rpm_manifest_file = compose.paths.compose.metadata("rpms.json") + rpm_manifest = Rpms() + rpm_manifest.load(rpm_manifest_file) + + old_rpm_manifest_file = compose.paths.old_compose_path(rpm_manifest_file) + old_rpm_manifest = Rpms() + old_rpm_manifest.load(old_rpm_manifest_file) + + for repo in cmd["original_image_conf"]["image-build"]["repo"]: + if repo not in compose.all_variants: + # External repos are checked using other logic. + continue + for arch in cmd["image_conf"]["image-build"]["arches"]: + if ( + rpm_manifest.rpms[variant.uid][arch] + != old_rpm_manifest.rpms[variant.uid][arch] + ): + self.pool.log_info( + log_msg % "Packages in %s.%s changed." % (variant.uid, arch) + ) + return False + + self.pool.log_info( + "Reusing images from old compose for variant %s" % variant.uid + ) + try: + self._link_images( + compose, variant, subvariant, cmd, old_metadata["image_infos"] + ) + except Exception as e: + self.pool.log_info(log_msg % "Can't link images %s" % str(e)) + return False + + old_log_file = compose.paths.old_compose_path(log_file) + try: + shutil.copy2(old_log_file, log_file) + except Exception as e: + self.pool.log_info( + log_msg % "Can't copy old log_file: %s %s" % (old_log_file, str(e)) + ) + return False + + self._write_reuse_metadata( + compose, + metadata_file, + cmd, + old_metadata["image_infos"], + external_repo_checksum, + ) + + return True + + def _write_reuse_metadata( + self, compose, metadata_file, cmd, image_infos, external_repo_checksum + ): + """Write metadata file. + + :param Compose compose: Current compose. + :param str metadata_file: Path to reuse metadata file. + :param dict cmd: Dict of params for image-build. + :param dict image_infos: Dict contains image info. + :param dict external_repo_checksum: Dict contains checksum of repomd.xml + or None if can't get checksum. + """ + msg = "Writing reuse metadata file: %s" % metadata_file + self.pool.log_info(msg) + + cmd_copy = copy.deepcopy(cmd) + del cmd_copy["image_conf"]["image-build"]["variant"] + + data = { + "cmd": cmd_copy, + "image_infos": image_infos, + "external_repo_checksum": external_repo_checksum, + } + try: + with open(metadata_file, "w") as f: + json.dump(data, f, indent=4) + except Exception as e: + self.pool.log_info("%s Failed: %s" % (msg, str(e))) + + def _load_reuse_metadata(self, metadata_file): + """Load metadata file. + + :param str metadata_file: Path to reuse metadata file. + """ + with open(metadata_file, "r") as f: + return json.load(f) diff --git a/pungi/scripts/pungi_koji.py b/pungi/scripts/pungi_koji.py index 07f8cf83..ed3b0815 100644 --- a/pungi/scripts/pungi_koji.py +++ b/pungi/scripts/pungi_koji.py @@ -406,7 +406,7 @@ def run_compose( extra_isos_phase = pungi.phases.ExtraIsosPhase(compose) liveimages_phase = pungi.phases.LiveImagesPhase(compose) livemedia_phase = pungi.phases.LiveMediaPhase(compose) - image_build_phase = pungi.phases.ImageBuildPhase(compose) + image_build_phase = pungi.phases.ImageBuildPhase(compose, buildinstall_phase) osbuild_phase = pungi.phases.OSBuildPhase(compose) osbs_phase = pungi.phases.OSBSPhase(compose) image_container_phase = pungi.phases.ImageContainerPhase(compose) diff --git a/tests/test_imagebuildphase.py b/tests/test_imagebuildphase.py index 711e0f7d..c0f5ac7d 100644 --- a/tests/test_imagebuildphase.py +++ b/tests/test_imagebuildphase.py @@ -17,26 +17,23 @@ class TestImageBuildPhase(PungiTestCase): @mock.patch("pungi.phases.image_build.ThreadPool") def test_image_build(self, ThreadPool): + original_image_conf = { + "image-build": { + "format": [("docker", "tar.xz")], + "name": "Fedora-Docker-Base", + "target": "f24", + "version": "Rawhide", + "ksurl": "git://git.fedorahosted.org/git/spin-kickstarts.git", # noqa: E501 + "kickstart": "fedora-docker-base.ks", + "distro": "Fedora-20", + "disk_size": 3, + "failable": ["x86_64"], + } + } compose = DummyCompose( self.topdir, { - "image_build": { - "^Client|Server$": [ - { - "image-build": { - "format": [("docker", "tar.xz")], - "name": "Fedora-Docker-Base", - "target": "f24", - "version": "Rawhide", - "ksurl": "git://git.fedorahosted.org/git/spin-kickstarts.git", # noqa: E501 - "kickstart": "fedora-docker-base.ks", - "distro": "Fedora-20", - "disk_size": 3, - "failable": ["x86_64"], - } - } - ] - }, + "image_build": {"^Client|Server$": [original_image_conf]}, "koji_profile": "koji", }, ) @@ -50,6 +47,7 @@ class TestImageBuildPhase(PungiTestCase): # assert at least one thread was started self.assertTrue(phase.pool.add.called) client_args = { + "original_image_conf": original_image_conf, "image_conf": { "image-build": { "install_tree": self.topdir + "/compose/Client/$arch/os", @@ -75,6 +73,7 @@ class TestImageBuildPhase(PungiTestCase): "scratch": False, } server_args = { + "original_image_conf": original_image_conf, "image_conf": { "image-build": { "install_tree": self.topdir + "/compose/Server/$arch/os", @@ -102,11 +101,23 @@ class TestImageBuildPhase(PungiTestCase): six.assertCountEqual( self, phase.pool.queue_put.mock_calls, - [mock.call((compose, client_args)), mock.call((compose, server_args))], + [ + mock.call((compose, client_args, phase.buildinstall_phase)), + mock.call((compose, server_args, phase.buildinstall_phase)), + ], ) @mock.patch("pungi.phases.image_build.ThreadPool") def test_image_build_phase_global_options(self, ThreadPool): + original_image_conf = { + "image-build": { + "format": ["docker"], + "name": "Fedora-Docker-Base", + "kickstart": "fedora-docker-base.ks", + "distro": "Fedora-20", + "disk_size": 3, + } + } compose = DummyCompose( self.topdir, { @@ -114,19 +125,7 @@ class TestImageBuildPhase(PungiTestCase): "image_build_release": "!RELEASE_FROM_LABEL_DATE_TYPE_RESPIN", "image_build_target": "f24", "image_build_version": "Rawhide", - "image_build": { - "^Server$": [ - { - "image-build": { - "format": ["docker"], - "name": "Fedora-Docker-Base", - "kickstart": "fedora-docker-base.ks", - "distro": "Fedora-20", - "disk_size": 3, - } - } - ] - }, + "image_build": {"^Server$": [original_image_conf]}, "koji_profile": "koji", }, ) @@ -140,6 +139,7 @@ class TestImageBuildPhase(PungiTestCase): # assert at least one thread was started self.assertTrue(phase.pool.add.called) server_args = { + "original_image_conf": original_image_conf, "image_conf": { "image-build": { "install_tree": self.topdir + "/compose/Server/$arch/os", @@ -165,30 +165,28 @@ class TestImageBuildPhase(PungiTestCase): "scratch": False, } self.assertEqual( - phase.pool.queue_put.mock_calls, [mock.call((compose, server_args))] + phase.pool.queue_put.mock_calls, + [mock.call((compose, server_args, phase.buildinstall_phase))], ) @mock.patch("pungi.phases.image_build.ThreadPool") def test_image_build_phase_missing_version(self, ThreadPool): + original_image_conf = { + "image-build": { + "format": "docker", + "name": "Fedora-Docker-Base", + "kickstart": "fedora-docker-base.ks", + "distro": "Fedora-20", + "disk_size": 3, + } + } compose = DummyCompose( self.topdir, { "image_build_ksurl": "git://git.fedorahosted.org/git/spin-kickstarts.git", # noqa: E501 "image_build_release": "!RELEASE_FROM_LABEL_DATE_TYPE_RESPIN", "image_build_target": "f24", - "image_build": { - "^Server$": [ - { - "image-build": { - "format": "docker", - "name": "Fedora-Docker-Base", - "kickstart": "fedora-docker-base.ks", - "distro": "Fedora-20", - "disk_size": 3, - } - } - ] - }, + "image_build": {"^Server$": [original_image_conf]}, "koji_profile": "koji", }, ) @@ -200,6 +198,7 @@ class TestImageBuildPhase(PungiTestCase): # assert at least one thread was started self.assertTrue(phase.pool.add.called) server_args = { + "original_image_conf": original_image_conf, "image_conf": { "image-build": { "install_tree": self.topdir + "/compose/Server/$arch/os", @@ -225,7 +224,8 @@ class TestImageBuildPhase(PungiTestCase): "scratch": False, } self.assertEqual( - phase.pool.queue_put.mock_calls, [mock.call((compose, server_args))] + phase.pool.queue_put.mock_calls, + [mock.call((compose, server_args, phase.buildinstall_phase))], ) @mock.patch("pungi.phases.image_build.ThreadPool") @@ -266,27 +266,25 @@ class TestImageBuildPhase(PungiTestCase): @mock.patch("pungi.phases.image_build.ThreadPool") def test_image_build_set_install_tree(self, ThreadPool): + original_image_conf = { + "image-build": { + "format": ["docker"], + "name": "Fedora-Docker-Base", + "target": "f24", + "version": "Rawhide", + "ksurl": "git://git.fedorahosted.org/git/spin-kickstarts.git", # noqa: E501 + "kickstart": "fedora-docker-base.ks", + "distro": "Fedora-20", + "disk_size": 3, + "arches": ["x86_64"], + "install_tree_from": "Server-optional", + } + } + compose = DummyCompose( self.topdir, { - "image_build": { - "^Server$": [ - { - "image-build": { - "format": ["docker"], - "name": "Fedora-Docker-Base", - "target": "f24", - "version": "Rawhide", - "ksurl": "git://git.fedorahosted.org/git/spin-kickstarts.git", # noqa: E501 - "kickstart": "fedora-docker-base.ks", - "distro": "Fedora-20", - "disk_size": 3, - "arches": ["x86_64"], - "install_tree_from": "Server-optional", - } - } - ] - }, + "image_build": {"^Server$": [original_image_conf]}, "koji_profile": "koji", }, ) @@ -307,6 +305,7 @@ class TestImageBuildPhase(PungiTestCase): self.assertDictEqual( args[0][1], { + "original_image_conf": original_image_conf, "image_conf": { "image-build": { "install_tree": self.topdir @@ -335,27 +334,24 @@ class TestImageBuildPhase(PungiTestCase): @mock.patch("pungi.phases.image_build.ThreadPool") def test_image_build_set_install_tree_from_path(self, ThreadPool): + original_image_conf = { + "image-build": { + "format": ["docker"], + "name": "Fedora-Docker-Base", + "target": "f24", + "version": "Rawhide", + "ksurl": "git://git.fedorahosted.org/git/spin-kickstarts.git", # noqa: E501 + "kickstart": "fedora-docker-base.ks", + "distro": "Fedora-20", + "disk_size": 3, + "arches": ["x86_64"], + "install_tree_from": "/my/tree", + } + } compose = DummyCompose( self.topdir, { - "image_build": { - "^Server$": [ - { - "image-build": { - "format": ["docker"], - "name": "Fedora-Docker-Base", - "target": "f24", - "version": "Rawhide", - "ksurl": "git://git.fedorahosted.org/git/spin-kickstarts.git", # noqa: E501 - "kickstart": "fedora-docker-base.ks", - "distro": "Fedora-20", - "disk_size": 3, - "arches": ["x86_64"], - "install_tree_from": "/my/tree", - } - } - ] - }, + "image_build": {"^Server$": [original_image_conf]}, "koji_profile": "koji", "translate_paths": [("/my", "http://example.com")], }, @@ -376,6 +372,7 @@ class TestImageBuildPhase(PungiTestCase): self.assertDictEqual( args[0][1], { + "original_image_conf": original_image_conf, "image_conf": { "image-build": { "install_tree": "http://example.com/tree", @@ -403,27 +400,24 @@ class TestImageBuildPhase(PungiTestCase): @mock.patch("pungi.phases.image_build.ThreadPool") def test_image_build_set_extra_repos(self, ThreadPool): + original_image_conf = { + "image-build": { + "format": ["docker"], + "name": "Fedora-Docker-Base", + "target": "f24", + "version": "Rawhide", + "ksurl": "git://git.fedorahosted.org/git/spin-kickstarts.git", # noqa: E501 + "kickstart": "fedora-docker-base.ks", + "distro": "Fedora-20", + "disk_size": 3, + "arches": ["x86_64"], + "repo_from": ["Everything", "Server-optional"], + } + } compose = DummyCompose( self.topdir, { - "image_build": { - "^Server$": [ - { - "image-build": { - "format": ["docker"], - "name": "Fedora-Docker-Base", - "target": "f24", - "version": "Rawhide", - "ksurl": "git://git.fedorahosted.org/git/spin-kickstarts.git", # noqa: E501 - "kickstart": "fedora-docker-base.ks", - "distro": "Fedora-20", - "disk_size": 3, - "arches": ["x86_64"], - "repo_from": ["Everything", "Server-optional"], - } - } - ] - }, + "image_build": {"^Server$": [original_image_conf]}, "koji_profile": "koji", }, ) @@ -444,6 +438,7 @@ class TestImageBuildPhase(PungiTestCase): self.assertDictEqual( args[0][1], { + "original_image_conf": original_image_conf, "image_conf": { "image-build": { "install_tree": self.topdir + "/compose/Server/$arch/os", @@ -477,27 +472,24 @@ class TestImageBuildPhase(PungiTestCase): @mock.patch("pungi.phases.image_build.ThreadPool") def test_image_build_set_external_install_tree(self, ThreadPool): + original_image_conf = { + "image-build": { + "format": ["docker"], + "name": "Fedora-Docker-Base", + "target": "f24", + "version": "Rawhide", + "ksurl": "git://git.fedorahosted.org/git/spin-kickstarts.git", # noqa: E501 + "kickstart": "fedora-docker-base.ks", + "distro": "Fedora-20", + "disk_size": 3, + "arches": ["x86_64"], + "install_tree_from": "http://example.com/install-tree/", + } + } compose = DummyCompose( self.topdir, { - "image_build": { - "^Server$": [ - { - "image-build": { - "format": ["docker"], - "name": "Fedora-Docker-Base", - "target": "f24", - "version": "Rawhide", - "ksurl": "git://git.fedorahosted.org/git/spin-kickstarts.git", # noqa: E501 - "kickstart": "fedora-docker-base.ks", - "distro": "Fedora-20", - "disk_size": 3, - "arches": ["x86_64"], - "install_tree_from": "http://example.com/install-tree/", - } - } - ] - }, + "image_build": {"^Server$": [original_image_conf]}, "koji_profile": "koji", }, ) @@ -517,6 +509,7 @@ class TestImageBuildPhase(PungiTestCase): self.assertDictEqual( args[0][1], { + "original_image_conf": original_image_conf, "image_conf": { "image-build": { "install_tree": "http://example.com/install-tree/", @@ -670,26 +663,23 @@ class TestImageBuildPhase(PungiTestCase): @mock.patch("pungi.phases.image_build.ThreadPool") def test_image_build_optional(self, ThreadPool): + original_image_conf = { + "image-build": { + "format": ["docker"], + "name": "Fedora-Docker-Base", + "target": "f24", + "version": "Rawhide", + "ksurl": "git://git.fedorahosted.org/git/spin-kickstarts.git", # noqa: E501 + "kickstart": "fedora-docker-base.ks", + "distro": "Fedora-20", + "disk_size": 3, + "failable": ["x86_64"], + } + } compose = DummyCompose( self.topdir, { - "image_build": { - "^Server-optional$": [ - { - "image-build": { - "format": ["docker"], - "name": "Fedora-Docker-Base", - "target": "f24", - "version": "Rawhide", - "ksurl": "git://git.fedorahosted.org/git/spin-kickstarts.git", # noqa: E501 - "kickstart": "fedora-docker-base.ks", - "distro": "Fedora-20", - "disk_size": 3, - "failable": ["x86_64"], - } - } - ] - }, + "image_build": {"^Server-optional$": [original_image_conf]}, "koji_profile": "koji", }, ) @@ -704,6 +694,7 @@ class TestImageBuildPhase(PungiTestCase): # assert at least one thread was started self.assertTrue(phase.pool.add.called) server_args = { + "original_image_conf": original_image_conf, "image_conf": { "image-build": { "install_tree": self.topdir + "/compose/Server/$arch/os", @@ -729,31 +720,29 @@ class TestImageBuildPhase(PungiTestCase): "scratch": False, } self.assertEqual( - phase.pool.queue_put.mock_calls, [mock.call((compose, server_args))] + phase.pool.queue_put.mock_calls, + [mock.call((compose, server_args, phase.buildinstall_phase))], ) @mock.patch("pungi.phases.image_build.ThreadPool") def test_failable_star(self, ThreadPool): + original_image_conf = { + "image-build": { + "format": ["docker"], + "name": "Fedora-Docker-Base", + "target": "f24", + "version": "Rawhide", + "ksurl": "git://git.fedorahosted.org/git/spin-kickstarts.git", # noqa: E501 + "kickstart": "fedora-docker-base.ks", + "distro": "Fedora-20", + "disk_size": 3, + "failable": ["*"], + } + } compose = DummyCompose( self.topdir, { - "image_build": { - "^Server$": [ - { - "image-build": { - "format": ["docker"], - "name": "Fedora-Docker-Base", - "target": "f24", - "version": "Rawhide", - "ksurl": "git://git.fedorahosted.org/git/spin-kickstarts.git", # noqa: E501 - "kickstart": "fedora-docker-base.ks", - "distro": "Fedora-20", - "disk_size": 3, - "failable": ["*"], - } - } - ] - }, + "image_build": {"^Server$": [original_image_conf]}, "koji_profile": "koji", }, ) @@ -768,6 +757,7 @@ class TestImageBuildPhase(PungiTestCase): # assert at least one thread was started self.assertTrue(phase.pool.add.called) server_args = { + "original_image_conf": original_image_conf, "image_conf": { "image-build": { "install_tree": self.topdir + "/compose/Server/$arch/os", @@ -793,7 +783,8 @@ class TestImageBuildPhase(PungiTestCase): "scratch": False, } self.assertEqual( - phase.pool.queue_put.mock_calls, [mock.call((compose, server_args))] + phase.pool.queue_put.mock_calls, + [mock.call((compose, server_args, phase.buildinstall_phase))], ) @@ -854,7 +845,7 @@ class TestCreateImageBuildThread(PungiTestCase): t = CreateImageBuildThread(pool) with mock.patch("time.sleep"): - t.process((compose, cmd), 1) + t.process((compose, cmd, None), 1) self.assertEqual( koji_wrapper.get_image_build_cmd.call_args_list, @@ -987,7 +978,7 @@ class TestCreateImageBuildThread(PungiTestCase): t = CreateImageBuildThread(pool) with mock.patch("time.sleep"): - t.process((compose, cmd), 1) + t.process((compose, cmd, None), 1) pool._logger.error.assert_has_calls( [ @@ -1041,7 +1032,7 @@ class TestCreateImageBuildThread(PungiTestCase): t = CreateImageBuildThread(pool) with mock.patch("time.sleep"): - t.process((compose, cmd), 1) + t.process((compose, cmd, None), 1) pool._logger.error.assert_has_calls( [ @@ -1092,4 +1083,4 @@ class TestCreateImageBuildThread(PungiTestCase): t = CreateImageBuildThread(pool) with self.assertRaises(RuntimeError): with mock.patch("time.sleep"): - t.process((compose, cmd), 1) + t.process((compose, cmd, None), 1)