image_build: Allow reusing old image_build results

JIRA: RHELCMP-5970
Signed-off-by: Haibo Lin <hlin@redhat.com>
This commit is contained in:
Haibo Lin 2021-08-20 19:35:36 +08:00
parent 7475d2a3a9
commit e42e65783d
4 changed files with 369 additions and 166 deletions

View File

@ -1080,6 +1080,7 @@ def make_schema():
"live_images": _variant_arch_mapping(
_one_or_list({"$ref": "#/definitions/live_image_config"})
),
"image_build_allow_reuse": {"type": "boolean", "default": False},
"image_build": {
"type": "object",
"patternProperties": {

View File

@ -1,18 +1,22 @@
# -*- coding: utf-8 -*-
import copy
import hashlib
import json
import os
import shutil
import time
from kobo import shortcuts
from pungi.util import makedirs, get_mtime, get_file_size, failable, log_failed_task
from pungi.util import translate_path, get_repo_urls, version_generator
from pungi.util import as_local_file, translate_path, get_repo_urls, version_generator
from pungi.phases import base
from pungi.linker import Linker
from pungi.wrappers.kojiwrapper import KojiWrapper
from kobo.threads import ThreadPool, WorkerThread
from kobo.shortcuts import force_list
from productmd.images import Image
from productmd.rpms import Rpms
# This is a mapping from formats to file extensions. The format is what koji
@ -46,9 +50,10 @@ class ImageBuildPhase(
name = "image_build"
def __init__(self, compose):
def __init__(self, compose, buildinstall_phase=None):
super(ImageBuildPhase, self).__init__(compose)
self.pool = ThreadPool(logger=self.logger)
self.buildinstall_phase = buildinstall_phase
def _get_install_tree(self, image_conf, variant):
"""
@ -117,6 +122,7 @@ class ImageBuildPhase(
# prevent problems in next iteration where the original
# value is needed.
image_conf = copy.deepcopy(image_conf)
original_image_conf = copy.deepcopy(image_conf)
# image_conf is passed to get_image_build_cmd as dict
@ -167,6 +173,7 @@ class ImageBuildPhase(
image_conf["image-build"]["can_fail"] = sorted(can_fail)
cmd = {
"original_image_conf": original_image_conf,
"image_conf": image_conf,
"conf_file": self.compose.paths.work.image_build_conf(
image_conf["image-build"]["variant"],
@ -182,7 +189,7 @@ class ImageBuildPhase(
"scratch": image_conf["image-build"].pop("scratch", False),
}
self.pool.add(CreateImageBuildThread(self.pool))
self.pool.queue_put((self.compose, cmd))
self.pool.queue_put((self.compose, cmd, self.buildinstall_phase))
self.pool.start()
@ -192,7 +199,7 @@ class CreateImageBuildThread(WorkerThread):
self.pool.log_error("CreateImageBuild failed.")
def process(self, item, num):
compose, cmd = item
compose, cmd, buildinstall_phase = item
variant = cmd["image_conf"]["image-build"]["variant"]
subvariant = cmd["image_conf"]["image-build"].get("subvariant", variant.uid)
self.failable_arches = cmd["image_conf"]["image-build"].get("can_fail", "")
@ -208,15 +215,47 @@ class CreateImageBuildThread(WorkerThread):
subvariant,
logger=self.pool._logger,
):
self.worker(num, compose, variant, subvariant, cmd)
self.worker(num, compose, variant, subvariant, cmd, buildinstall_phase)
def worker(self, num, compose, variant, subvariant, cmd):
def worker(self, num, compose, variant, subvariant, cmd, buildinstall_phase):
arches = cmd["image_conf"]["image-build"]["arches"]
formats = "-".join(cmd["image_conf"]["image-build"]["format"])
dash_arches = "-".join(arches)
log_file = compose.paths.log.log_file(
dash_arches, "imagebuild-%s-%s-%s" % (variant.uid, subvariant, formats)
)
metadata_file = log_file[:-4] + ".reuse.json"
external_repo_checksum = {}
try:
for repo in cmd["original_image_conf"]["image-build"]["repo"]:
if repo in compose.all_variants:
continue
with as_local_file(
os.path.join(repo, "repodata/repomd.xml")
) as filename:
with open(filename, "rb") as f:
external_repo_checksum[repo] = hashlib.sha256(
f.read()
).hexdigest()
except Exception as e:
external_repo_checksum = None
self.pool.log_info(
"Can't calculate checksum of repomd.xml of external repo - %s" % str(e)
)
if self._try_to_reuse(
compose,
variant,
subvariant,
metadata_file,
log_file,
cmd,
external_repo_checksum,
buildinstall_phase,
):
return
msg = (
"Creating image (formats: %s, arches: %s, variant: %s, subvariant: %s)"
% (formats, dash_arches, variant, subvariant)
@ -275,6 +314,22 @@ class CreateImageBuildThread(WorkerThread):
)
break
self._link_images(compose, variant, subvariant, cmd, image_infos)
self._write_reuse_metadata(
compose, metadata_file, cmd, image_infos, external_repo_checksum
)
self.pool.log_info("[DONE ] %s (task id: %s)" % (msg, output["task_id"]))
def _link_images(self, compose, variant, subvariant, cmd, image_infos):
"""Link images to compose and update image manifest.
:param Compose compose: Current compose.
:param Variant variant: Current variant.
:param str subvariant:
:param dict cmd: Dict of params for image-build.
:param dict image_infos: Dict contains image info.
"""
# The usecase here is that you can run koji image-build with multiple --format
# It's ok to do it serialized since we're talking about max 2 images per single
# image_build record
@ -308,4 +363,160 @@ class CreateImageBuildThread(WorkerThread):
setattr(img, "deliverable", "image-build")
compose.im.add(variant=variant.uid, arch=image_info["arch"], image=img)
self.pool.log_info("[DONE ] %s (task id: %s)" % (msg, output["task_id"]))
def _try_to_reuse(
self,
compose,
variant,
subvariant,
metadata_file,
log_file,
cmd,
external_repo_checksum,
buildinstall_phase,
):
"""Try to reuse images from old compose.
:param Compose compose: Current compose.
:param Variant variant: Current variant.
:param str subvariant:
:param str metadata_file: Path to reuse metadata file.
:param str log_file: Path to log file.
:param dict cmd: Dict of params for image-build.
:param dict external_repo_checksum: Dict contains checksum of repomd.xml
or None if can't get checksum.
:param BuildinstallPhase buildinstall_phase: buildinstall phase of
current compose.
"""
log_msg = "Cannot reuse old image_build phase results - %s"
if not compose.conf["image_build_allow_reuse"]:
self.pool.log_info(
log_msg % "reuse of old image_build results is disabled."
)
return False
if external_repo_checksum is None:
self.pool.log_info(
log_msg % "Can't ensure that external repo is not changed."
)
return False
old_metadata_file = compose.paths.old_compose_path(metadata_file)
if not old_metadata_file:
self.pool.log_info(log_msg % "Can't find old reuse metadata file")
return False
try:
old_metadata = self._load_reuse_metadata(old_metadata_file)
except Exception as e:
self.pool.log_info(
log_msg % "Can't load old reuse metadata file: %s" % str(e)
)
return False
if old_metadata["cmd"]["original_image_conf"] != cmd["original_image_conf"]:
self.pool.log_info(log_msg % "image_build config changed")
return False
# Make sure external repo does not change
if (
old_metadata["external_repo_checksum"] is None
or old_metadata["external_repo_checksum"] != external_repo_checksum
):
self.pool.log_info(log_msg % "External repo may be changed")
return False
# Make sure buildinstall phase is reused
for arch in cmd["image_conf"]["image-build"]["arches"]:
if buildinstall_phase and not buildinstall_phase.reused(variant, arch):
self.pool.log_info(log_msg % "buildinstall phase changed")
return False
# Make sure packages in variant not change
rpm_manifest_file = compose.paths.compose.metadata("rpms.json")
rpm_manifest = Rpms()
rpm_manifest.load(rpm_manifest_file)
old_rpm_manifest_file = compose.paths.old_compose_path(rpm_manifest_file)
old_rpm_manifest = Rpms()
old_rpm_manifest.load(old_rpm_manifest_file)
for repo in cmd["original_image_conf"]["image-build"]["repo"]:
if repo not in compose.all_variants:
# External repos are checked using other logic.
continue
for arch in cmd["image_conf"]["image-build"]["arches"]:
if (
rpm_manifest.rpms[variant.uid][arch]
!= old_rpm_manifest.rpms[variant.uid][arch]
):
self.pool.log_info(
log_msg % "Packages in %s.%s changed." % (variant.uid, arch)
)
return False
self.pool.log_info(
"Reusing images from old compose for variant %s" % variant.uid
)
try:
self._link_images(
compose, variant, subvariant, cmd, old_metadata["image_infos"]
)
except Exception as e:
self.pool.log_info(log_msg % "Can't link images %s" % str(e))
return False
old_log_file = compose.paths.old_compose_path(log_file)
try:
shutil.copy2(old_log_file, log_file)
except Exception as e:
self.pool.log_info(
log_msg % "Can't copy old log_file: %s %s" % (old_log_file, str(e))
)
return False
self._write_reuse_metadata(
compose,
metadata_file,
cmd,
old_metadata["image_infos"],
external_repo_checksum,
)
return True
def _write_reuse_metadata(
self, compose, metadata_file, cmd, image_infos, external_repo_checksum
):
"""Write metadata file.
:param Compose compose: Current compose.
:param str metadata_file: Path to reuse metadata file.
:param dict cmd: Dict of params for image-build.
:param dict image_infos: Dict contains image info.
:param dict external_repo_checksum: Dict contains checksum of repomd.xml
or None if can't get checksum.
"""
msg = "Writing reuse metadata file: %s" % metadata_file
self.pool.log_info(msg)
cmd_copy = copy.deepcopy(cmd)
del cmd_copy["image_conf"]["image-build"]["variant"]
data = {
"cmd": cmd_copy,
"image_infos": image_infos,
"external_repo_checksum": external_repo_checksum,
}
try:
with open(metadata_file, "w") as f:
json.dump(data, f, indent=4)
except Exception as e:
self.pool.log_info("%s Failed: %s" % (msg, str(e)))
def _load_reuse_metadata(self, metadata_file):
"""Load metadata file.
:param str metadata_file: Path to reuse metadata file.
"""
with open(metadata_file, "r") as f:
return json.load(f)

View File

@ -406,7 +406,7 @@ def run_compose(
extra_isos_phase = pungi.phases.ExtraIsosPhase(compose)
liveimages_phase = pungi.phases.LiveImagesPhase(compose)
livemedia_phase = pungi.phases.LiveMediaPhase(compose)
image_build_phase = pungi.phases.ImageBuildPhase(compose)
image_build_phase = pungi.phases.ImageBuildPhase(compose, buildinstall_phase)
osbuild_phase = pungi.phases.OSBuildPhase(compose)
osbs_phase = pungi.phases.OSBSPhase(compose)
image_container_phase = pungi.phases.ImageContainerPhase(compose)

View File

@ -17,12 +17,7 @@ class TestImageBuildPhase(PungiTestCase):
@mock.patch("pungi.phases.image_build.ThreadPool")
def test_image_build(self, ThreadPool):
compose = DummyCompose(
self.topdir,
{
"image_build": {
"^Client|Server$": [
{
original_image_conf = {
"image-build": {
"format": [("docker", "tar.xz")],
"name": "Fedora-Docker-Base",
@ -35,8 +30,10 @@ class TestImageBuildPhase(PungiTestCase):
"failable": ["x86_64"],
}
}
]
},
compose = DummyCompose(
self.topdir,
{
"image_build": {"^Client|Server$": [original_image_conf]},
"koji_profile": "koji",
},
)
@ -50,6 +47,7 @@ class TestImageBuildPhase(PungiTestCase):
# assert at least one thread was started
self.assertTrue(phase.pool.add.called)
client_args = {
"original_image_conf": original_image_conf,
"image_conf": {
"image-build": {
"install_tree": self.topdir + "/compose/Client/$arch/os",
@ -75,6 +73,7 @@ class TestImageBuildPhase(PungiTestCase):
"scratch": False,
}
server_args = {
"original_image_conf": original_image_conf,
"image_conf": {
"image-build": {
"install_tree": self.topdir + "/compose/Server/$arch/os",
@ -102,21 +101,15 @@ class TestImageBuildPhase(PungiTestCase):
six.assertCountEqual(
self,
phase.pool.queue_put.mock_calls,
[mock.call((compose, client_args)), mock.call((compose, server_args))],
[
mock.call((compose, client_args, phase.buildinstall_phase)),
mock.call((compose, server_args, phase.buildinstall_phase)),
],
)
@mock.patch("pungi.phases.image_build.ThreadPool")
def test_image_build_phase_global_options(self, ThreadPool):
compose = DummyCompose(
self.topdir,
{
"image_build_ksurl": "git://git.fedorahosted.org/git/spin-kickstarts.git", # noqa: E501
"image_build_release": "!RELEASE_FROM_LABEL_DATE_TYPE_RESPIN",
"image_build_target": "f24",
"image_build_version": "Rawhide",
"image_build": {
"^Server$": [
{
original_image_conf = {
"image-build": {
"format": ["docker"],
"name": "Fedora-Docker-Base",
@ -125,8 +118,14 @@ class TestImageBuildPhase(PungiTestCase):
"disk_size": 3,
}
}
]
},
compose = DummyCompose(
self.topdir,
{
"image_build_ksurl": "git://git.fedorahosted.org/git/spin-kickstarts.git", # noqa: E501
"image_build_release": "!RELEASE_FROM_LABEL_DATE_TYPE_RESPIN",
"image_build_target": "f24",
"image_build_version": "Rawhide",
"image_build": {"^Server$": [original_image_conf]},
"koji_profile": "koji",
},
)
@ -140,6 +139,7 @@ class TestImageBuildPhase(PungiTestCase):
# assert at least one thread was started
self.assertTrue(phase.pool.add.called)
server_args = {
"original_image_conf": original_image_conf,
"image_conf": {
"image-build": {
"install_tree": self.topdir + "/compose/Server/$arch/os",
@ -165,20 +165,13 @@ class TestImageBuildPhase(PungiTestCase):
"scratch": False,
}
self.assertEqual(
phase.pool.queue_put.mock_calls, [mock.call((compose, server_args))]
phase.pool.queue_put.mock_calls,
[mock.call((compose, server_args, phase.buildinstall_phase))],
)
@mock.patch("pungi.phases.image_build.ThreadPool")
def test_image_build_phase_missing_version(self, ThreadPool):
compose = DummyCompose(
self.topdir,
{
"image_build_ksurl": "git://git.fedorahosted.org/git/spin-kickstarts.git", # noqa: E501
"image_build_release": "!RELEASE_FROM_LABEL_DATE_TYPE_RESPIN",
"image_build_target": "f24",
"image_build": {
"^Server$": [
{
original_image_conf = {
"image-build": {
"format": "docker",
"name": "Fedora-Docker-Base",
@ -187,8 +180,13 @@ class TestImageBuildPhase(PungiTestCase):
"disk_size": 3,
}
}
]
},
compose = DummyCompose(
self.topdir,
{
"image_build_ksurl": "git://git.fedorahosted.org/git/spin-kickstarts.git", # noqa: E501
"image_build_release": "!RELEASE_FROM_LABEL_DATE_TYPE_RESPIN",
"image_build_target": "f24",
"image_build": {"^Server$": [original_image_conf]},
"koji_profile": "koji",
},
)
@ -200,6 +198,7 @@ class TestImageBuildPhase(PungiTestCase):
# assert at least one thread was started
self.assertTrue(phase.pool.add.called)
server_args = {
"original_image_conf": original_image_conf,
"image_conf": {
"image-build": {
"install_tree": self.topdir + "/compose/Server/$arch/os",
@ -225,7 +224,8 @@ class TestImageBuildPhase(PungiTestCase):
"scratch": False,
}
self.assertEqual(
phase.pool.queue_put.mock_calls, [mock.call((compose, server_args))]
phase.pool.queue_put.mock_calls,
[mock.call((compose, server_args, phase.buildinstall_phase))],
)
@mock.patch("pungi.phases.image_build.ThreadPool")
@ -266,12 +266,7 @@ class TestImageBuildPhase(PungiTestCase):
@mock.patch("pungi.phases.image_build.ThreadPool")
def test_image_build_set_install_tree(self, ThreadPool):
compose = DummyCompose(
self.topdir,
{
"image_build": {
"^Server$": [
{
original_image_conf = {
"image-build": {
"format": ["docker"],
"name": "Fedora-Docker-Base",
@ -285,8 +280,11 @@ class TestImageBuildPhase(PungiTestCase):
"install_tree_from": "Server-optional",
}
}
]
},
compose = DummyCompose(
self.topdir,
{
"image_build": {"^Server$": [original_image_conf]},
"koji_profile": "koji",
},
)
@ -307,6 +305,7 @@ class TestImageBuildPhase(PungiTestCase):
self.assertDictEqual(
args[0][1],
{
"original_image_conf": original_image_conf,
"image_conf": {
"image-build": {
"install_tree": self.topdir
@ -335,12 +334,7 @@ class TestImageBuildPhase(PungiTestCase):
@mock.patch("pungi.phases.image_build.ThreadPool")
def test_image_build_set_install_tree_from_path(self, ThreadPool):
compose = DummyCompose(
self.topdir,
{
"image_build": {
"^Server$": [
{
original_image_conf = {
"image-build": {
"format": ["docker"],
"name": "Fedora-Docker-Base",
@ -354,8 +348,10 @@ class TestImageBuildPhase(PungiTestCase):
"install_tree_from": "/my/tree",
}
}
]
},
compose = DummyCompose(
self.topdir,
{
"image_build": {"^Server$": [original_image_conf]},
"koji_profile": "koji",
"translate_paths": [("/my", "http://example.com")],
},
@ -376,6 +372,7 @@ class TestImageBuildPhase(PungiTestCase):
self.assertDictEqual(
args[0][1],
{
"original_image_conf": original_image_conf,
"image_conf": {
"image-build": {
"install_tree": "http://example.com/tree",
@ -403,12 +400,7 @@ class TestImageBuildPhase(PungiTestCase):
@mock.patch("pungi.phases.image_build.ThreadPool")
def test_image_build_set_extra_repos(self, ThreadPool):
compose = DummyCompose(
self.topdir,
{
"image_build": {
"^Server$": [
{
original_image_conf = {
"image-build": {
"format": ["docker"],
"name": "Fedora-Docker-Base",
@ -422,8 +414,10 @@ class TestImageBuildPhase(PungiTestCase):
"repo_from": ["Everything", "Server-optional"],
}
}
]
},
compose = DummyCompose(
self.topdir,
{
"image_build": {"^Server$": [original_image_conf]},
"koji_profile": "koji",
},
)
@ -444,6 +438,7 @@ class TestImageBuildPhase(PungiTestCase):
self.assertDictEqual(
args[0][1],
{
"original_image_conf": original_image_conf,
"image_conf": {
"image-build": {
"install_tree": self.topdir + "/compose/Server/$arch/os",
@ -477,12 +472,7 @@ class TestImageBuildPhase(PungiTestCase):
@mock.patch("pungi.phases.image_build.ThreadPool")
def test_image_build_set_external_install_tree(self, ThreadPool):
compose = DummyCompose(
self.topdir,
{
"image_build": {
"^Server$": [
{
original_image_conf = {
"image-build": {
"format": ["docker"],
"name": "Fedora-Docker-Base",
@ -496,8 +486,10 @@ class TestImageBuildPhase(PungiTestCase):
"install_tree_from": "http://example.com/install-tree/",
}
}
]
},
compose = DummyCompose(
self.topdir,
{
"image_build": {"^Server$": [original_image_conf]},
"koji_profile": "koji",
},
)
@ -517,6 +509,7 @@ class TestImageBuildPhase(PungiTestCase):
self.assertDictEqual(
args[0][1],
{
"original_image_conf": original_image_conf,
"image_conf": {
"image-build": {
"install_tree": "http://example.com/install-tree/",
@ -670,12 +663,7 @@ class TestImageBuildPhase(PungiTestCase):
@mock.patch("pungi.phases.image_build.ThreadPool")
def test_image_build_optional(self, ThreadPool):
compose = DummyCompose(
self.topdir,
{
"image_build": {
"^Server-optional$": [
{
original_image_conf = {
"image-build": {
"format": ["docker"],
"name": "Fedora-Docker-Base",
@ -688,8 +676,10 @@ class TestImageBuildPhase(PungiTestCase):
"failable": ["x86_64"],
}
}
]
},
compose = DummyCompose(
self.topdir,
{
"image_build": {"^Server-optional$": [original_image_conf]},
"koji_profile": "koji",
},
)
@ -704,6 +694,7 @@ class TestImageBuildPhase(PungiTestCase):
# assert at least one thread was started
self.assertTrue(phase.pool.add.called)
server_args = {
"original_image_conf": original_image_conf,
"image_conf": {
"image-build": {
"install_tree": self.topdir + "/compose/Server/$arch/os",
@ -729,17 +720,13 @@ class TestImageBuildPhase(PungiTestCase):
"scratch": False,
}
self.assertEqual(
phase.pool.queue_put.mock_calls, [mock.call((compose, server_args))]
phase.pool.queue_put.mock_calls,
[mock.call((compose, server_args, phase.buildinstall_phase))],
)
@mock.patch("pungi.phases.image_build.ThreadPool")
def test_failable_star(self, ThreadPool):
compose = DummyCompose(
self.topdir,
{
"image_build": {
"^Server$": [
{
original_image_conf = {
"image-build": {
"format": ["docker"],
"name": "Fedora-Docker-Base",
@ -752,8 +739,10 @@ class TestImageBuildPhase(PungiTestCase):
"failable": ["*"],
}
}
]
},
compose = DummyCompose(
self.topdir,
{
"image_build": {"^Server$": [original_image_conf]},
"koji_profile": "koji",
},
)
@ -768,6 +757,7 @@ class TestImageBuildPhase(PungiTestCase):
# assert at least one thread was started
self.assertTrue(phase.pool.add.called)
server_args = {
"original_image_conf": original_image_conf,
"image_conf": {
"image-build": {
"install_tree": self.topdir + "/compose/Server/$arch/os",
@ -793,7 +783,8 @@ class TestImageBuildPhase(PungiTestCase):
"scratch": False,
}
self.assertEqual(
phase.pool.queue_put.mock_calls, [mock.call((compose, server_args))]
phase.pool.queue_put.mock_calls,
[mock.call((compose, server_args, phase.buildinstall_phase))],
)
@ -854,7 +845,7 @@ class TestCreateImageBuildThread(PungiTestCase):
t = CreateImageBuildThread(pool)
with mock.patch("time.sleep"):
t.process((compose, cmd), 1)
t.process((compose, cmd, None), 1)
self.assertEqual(
koji_wrapper.get_image_build_cmd.call_args_list,
@ -987,7 +978,7 @@ class TestCreateImageBuildThread(PungiTestCase):
t = CreateImageBuildThread(pool)
with mock.patch("time.sleep"):
t.process((compose, cmd), 1)
t.process((compose, cmd, None), 1)
pool._logger.error.assert_has_calls(
[
@ -1041,7 +1032,7 @@ class TestCreateImageBuildThread(PungiTestCase):
t = CreateImageBuildThread(pool)
with mock.patch("time.sleep"):
t.process((compose, cmd), 1)
t.process((compose, cmd, None), 1)
pool._logger.error.assert_has_calls(
[
@ -1092,4 +1083,4 @@ class TestCreateImageBuildThread(PungiTestCase):
t = CreateImageBuildThread(pool)
with self.assertRaises(RuntimeError):
with mock.patch("time.sleep"):
t.process((compose, cmd), 1)
t.process((compose, cmd, None), 1)