Update from upstream #11
@ -1080,6 +1080,7 @@ def make_schema():
|
|||||||
"live_images": _variant_arch_mapping(
|
"live_images": _variant_arch_mapping(
|
||||||
_one_or_list({"$ref": "#/definitions/live_image_config"})
|
_one_or_list({"$ref": "#/definitions/live_image_config"})
|
||||||
),
|
),
|
||||||
|
"image_build_allow_reuse": {"type": "boolean", "default": False},
|
||||||
"image_build": {
|
"image_build": {
|
||||||
"type": "object",
|
"type": "object",
|
||||||
"patternProperties": {
|
"patternProperties": {
|
||||||
|
@ -1,18 +1,22 @@
|
|||||||
# -*- coding: utf-8 -*-
|
# -*- coding: utf-8 -*-
|
||||||
|
|
||||||
import copy
|
import copy
|
||||||
|
import hashlib
|
||||||
|
import json
|
||||||
import os
|
import os
|
||||||
|
import shutil
|
||||||
import time
|
import time
|
||||||
from kobo import shortcuts
|
from kobo import shortcuts
|
||||||
|
|
||||||
from pungi.util import makedirs, get_mtime, get_file_size, failable, log_failed_task
|
from pungi.util import makedirs, get_mtime, get_file_size, failable, log_failed_task
|
||||||
from pungi.util import translate_path, get_repo_urls, version_generator
|
from pungi.util import as_local_file, translate_path, get_repo_urls, version_generator
|
||||||
from pungi.phases import base
|
from pungi.phases import base
|
||||||
from pungi.linker import Linker
|
from pungi.linker import Linker
|
||||||
from pungi.wrappers.kojiwrapper import KojiWrapper
|
from pungi.wrappers.kojiwrapper import KojiWrapper
|
||||||
from kobo.threads import ThreadPool, WorkerThread
|
from kobo.threads import ThreadPool, WorkerThread
|
||||||
from kobo.shortcuts import force_list
|
from kobo.shortcuts import force_list
|
||||||
from productmd.images import Image
|
from productmd.images import Image
|
||||||
|
from productmd.rpms import Rpms
|
||||||
|
|
||||||
|
|
||||||
# This is a mapping from formats to file extensions. The format is what koji
|
# This is a mapping from formats to file extensions. The format is what koji
|
||||||
@ -46,9 +50,10 @@ class ImageBuildPhase(
|
|||||||
|
|
||||||
name = "image_build"
|
name = "image_build"
|
||||||
|
|
||||||
def __init__(self, compose):
|
def __init__(self, compose, buildinstall_phase=None):
|
||||||
super(ImageBuildPhase, self).__init__(compose)
|
super(ImageBuildPhase, self).__init__(compose)
|
||||||
self.pool = ThreadPool(logger=self.logger)
|
self.pool = ThreadPool(logger=self.logger)
|
||||||
|
self.buildinstall_phase = buildinstall_phase
|
||||||
|
|
||||||
def _get_install_tree(self, image_conf, variant):
|
def _get_install_tree(self, image_conf, variant):
|
||||||
"""
|
"""
|
||||||
@ -117,6 +122,7 @@ class ImageBuildPhase(
|
|||||||
# prevent problems in next iteration where the original
|
# prevent problems in next iteration where the original
|
||||||
# value is needed.
|
# value is needed.
|
||||||
image_conf = copy.deepcopy(image_conf)
|
image_conf = copy.deepcopy(image_conf)
|
||||||
|
original_image_conf = copy.deepcopy(image_conf)
|
||||||
|
|
||||||
# image_conf is passed to get_image_build_cmd as dict
|
# image_conf is passed to get_image_build_cmd as dict
|
||||||
|
|
||||||
@ -167,6 +173,7 @@ class ImageBuildPhase(
|
|||||||
image_conf["image-build"]["can_fail"] = sorted(can_fail)
|
image_conf["image-build"]["can_fail"] = sorted(can_fail)
|
||||||
|
|
||||||
cmd = {
|
cmd = {
|
||||||
|
"original_image_conf": original_image_conf,
|
||||||
"image_conf": image_conf,
|
"image_conf": image_conf,
|
||||||
"conf_file": self.compose.paths.work.image_build_conf(
|
"conf_file": self.compose.paths.work.image_build_conf(
|
||||||
image_conf["image-build"]["variant"],
|
image_conf["image-build"]["variant"],
|
||||||
@ -182,7 +189,7 @@ class ImageBuildPhase(
|
|||||||
"scratch": image_conf["image-build"].pop("scratch", False),
|
"scratch": image_conf["image-build"].pop("scratch", False),
|
||||||
}
|
}
|
||||||
self.pool.add(CreateImageBuildThread(self.pool))
|
self.pool.add(CreateImageBuildThread(self.pool))
|
||||||
self.pool.queue_put((self.compose, cmd))
|
self.pool.queue_put((self.compose, cmd, self.buildinstall_phase))
|
||||||
|
|
||||||
self.pool.start()
|
self.pool.start()
|
||||||
|
|
||||||
@ -192,7 +199,7 @@ class CreateImageBuildThread(WorkerThread):
|
|||||||
self.pool.log_error("CreateImageBuild failed.")
|
self.pool.log_error("CreateImageBuild failed.")
|
||||||
|
|
||||||
def process(self, item, num):
|
def process(self, item, num):
|
||||||
compose, cmd = item
|
compose, cmd, buildinstall_phase = item
|
||||||
variant = cmd["image_conf"]["image-build"]["variant"]
|
variant = cmd["image_conf"]["image-build"]["variant"]
|
||||||
subvariant = cmd["image_conf"]["image-build"].get("subvariant", variant.uid)
|
subvariant = cmd["image_conf"]["image-build"].get("subvariant", variant.uid)
|
||||||
self.failable_arches = cmd["image_conf"]["image-build"].get("can_fail", "")
|
self.failable_arches = cmd["image_conf"]["image-build"].get("can_fail", "")
|
||||||
@ -208,15 +215,47 @@ class CreateImageBuildThread(WorkerThread):
|
|||||||
subvariant,
|
subvariant,
|
||||||
logger=self.pool._logger,
|
logger=self.pool._logger,
|
||||||
):
|
):
|
||||||
self.worker(num, compose, variant, subvariant, cmd)
|
self.worker(num, compose, variant, subvariant, cmd, buildinstall_phase)
|
||||||
|
|
||||||
def worker(self, num, compose, variant, subvariant, cmd):
|
def worker(self, num, compose, variant, subvariant, cmd, buildinstall_phase):
|
||||||
arches = cmd["image_conf"]["image-build"]["arches"]
|
arches = cmd["image_conf"]["image-build"]["arches"]
|
||||||
formats = "-".join(cmd["image_conf"]["image-build"]["format"])
|
formats = "-".join(cmd["image_conf"]["image-build"]["format"])
|
||||||
dash_arches = "-".join(arches)
|
dash_arches = "-".join(arches)
|
||||||
log_file = compose.paths.log.log_file(
|
log_file = compose.paths.log.log_file(
|
||||||
dash_arches, "imagebuild-%s-%s-%s" % (variant.uid, subvariant, formats)
|
dash_arches, "imagebuild-%s-%s-%s" % (variant.uid, subvariant, formats)
|
||||||
)
|
)
|
||||||
|
metadata_file = log_file[:-4] + ".reuse.json"
|
||||||
|
|
||||||
|
external_repo_checksum = {}
|
||||||
|
try:
|
||||||
|
for repo in cmd["original_image_conf"]["image-build"]["repo"]:
|
||||||
|
if repo in compose.all_variants:
|
||||||
|
continue
|
||||||
|
with as_local_file(
|
||||||
|
os.path.join(repo, "repodata/repomd.xml")
|
||||||
|
) as filename:
|
||||||
|
with open(filename, "rb") as f:
|
||||||
|
external_repo_checksum[repo] = hashlib.sha256(
|
||||||
|
f.read()
|
||||||
|
).hexdigest()
|
||||||
|
except Exception as e:
|
||||||
|
external_repo_checksum = None
|
||||||
|
self.pool.log_info(
|
||||||
|
"Can't calculate checksum of repomd.xml of external repo - %s" % str(e)
|
||||||
|
)
|
||||||
|
|
||||||
|
if self._try_to_reuse(
|
||||||
|
compose,
|
||||||
|
variant,
|
||||||
|
subvariant,
|
||||||
|
metadata_file,
|
||||||
|
log_file,
|
||||||
|
cmd,
|
||||||
|
external_repo_checksum,
|
||||||
|
buildinstall_phase,
|
||||||
|
):
|
||||||
|
return
|
||||||
|
|
||||||
msg = (
|
msg = (
|
||||||
"Creating image (formats: %s, arches: %s, variant: %s, subvariant: %s)"
|
"Creating image (formats: %s, arches: %s, variant: %s, subvariant: %s)"
|
||||||
% (formats, dash_arches, variant, subvariant)
|
% (formats, dash_arches, variant, subvariant)
|
||||||
@ -275,6 +314,22 @@ class CreateImageBuildThread(WorkerThread):
|
|||||||
)
|
)
|
||||||
break
|
break
|
||||||
|
|
||||||
|
self._link_images(compose, variant, subvariant, cmd, image_infos)
|
||||||
|
self._write_reuse_metadata(
|
||||||
|
compose, metadata_file, cmd, image_infos, external_repo_checksum
|
||||||
|
)
|
||||||
|
|
||||||
|
self.pool.log_info("[DONE ] %s (task id: %s)" % (msg, output["task_id"]))
|
||||||
|
|
||||||
|
def _link_images(self, compose, variant, subvariant, cmd, image_infos):
|
||||||
|
"""Link images to compose and update image manifest.
|
||||||
|
|
||||||
|
:param Compose compose: Current compose.
|
||||||
|
:param Variant variant: Current variant.
|
||||||
|
:param str subvariant:
|
||||||
|
:param dict cmd: Dict of params for image-build.
|
||||||
|
:param dict image_infos: Dict contains image info.
|
||||||
|
"""
|
||||||
# The usecase here is that you can run koji image-build with multiple --format
|
# The usecase here is that you can run koji image-build with multiple --format
|
||||||
# It's ok to do it serialized since we're talking about max 2 images per single
|
# It's ok to do it serialized since we're talking about max 2 images per single
|
||||||
# image_build record
|
# image_build record
|
||||||
@ -308,4 +363,160 @@ class CreateImageBuildThread(WorkerThread):
|
|||||||
setattr(img, "deliverable", "image-build")
|
setattr(img, "deliverable", "image-build")
|
||||||
compose.im.add(variant=variant.uid, arch=image_info["arch"], image=img)
|
compose.im.add(variant=variant.uid, arch=image_info["arch"], image=img)
|
||||||
|
|
||||||
self.pool.log_info("[DONE ] %s (task id: %s)" % (msg, output["task_id"]))
|
def _try_to_reuse(
|
||||||
|
self,
|
||||||
|
compose,
|
||||||
|
variant,
|
||||||
|
subvariant,
|
||||||
|
metadata_file,
|
||||||
|
log_file,
|
||||||
|
cmd,
|
||||||
|
external_repo_checksum,
|
||||||
|
buildinstall_phase,
|
||||||
|
):
|
||||||
|
"""Try to reuse images from old compose.
|
||||||
|
|
||||||
|
:param Compose compose: Current compose.
|
||||||
|
:param Variant variant: Current variant.
|
||||||
|
:param str subvariant:
|
||||||
|
:param str metadata_file: Path to reuse metadata file.
|
||||||
|
:param str log_file: Path to log file.
|
||||||
|
:param dict cmd: Dict of params for image-build.
|
||||||
|
:param dict external_repo_checksum: Dict contains checksum of repomd.xml
|
||||||
|
or None if can't get checksum.
|
||||||
|
:param BuildinstallPhase buildinstall_phase: buildinstall phase of
|
||||||
|
current compose.
|
||||||
|
"""
|
||||||
|
log_msg = "Cannot reuse old image_build phase results - %s"
|
||||||
|
if not compose.conf["image_build_allow_reuse"]:
|
||||||
|
self.pool.log_info(
|
||||||
|
log_msg % "reuse of old image_build results is disabled."
|
||||||
|
)
|
||||||
|
return False
|
||||||
|
|
||||||
|
if external_repo_checksum is None:
|
||||||
|
self.pool.log_info(
|
||||||
|
log_msg % "Can't ensure that external repo is not changed."
|
||||||
|
)
|
||||||
|
return False
|
||||||
|
|
||||||
|
old_metadata_file = compose.paths.old_compose_path(metadata_file)
|
||||||
|
if not old_metadata_file:
|
||||||
|
self.pool.log_info(log_msg % "Can't find old reuse metadata file")
|
||||||
|
return False
|
||||||
|
|
||||||
|
try:
|
||||||
|
old_metadata = self._load_reuse_metadata(old_metadata_file)
|
||||||
|
except Exception as e:
|
||||||
|
self.pool.log_info(
|
||||||
|
log_msg % "Can't load old reuse metadata file: %s" % str(e)
|
||||||
|
)
|
||||||
|
return False
|
||||||
|
|
||||||
|
if old_metadata["cmd"]["original_image_conf"] != cmd["original_image_conf"]:
|
||||||
|
self.pool.log_info(log_msg % "image_build config changed")
|
||||||
|
return False
|
||||||
|
|
||||||
|
# Make sure external repo does not change
|
||||||
|
if (
|
||||||
|
old_metadata["external_repo_checksum"] is None
|
||||||
|
or old_metadata["external_repo_checksum"] != external_repo_checksum
|
||||||
|
):
|
||||||
|
self.pool.log_info(log_msg % "External repo may be changed")
|
||||||
|
return False
|
||||||
|
|
||||||
|
# Make sure buildinstall phase is reused
|
||||||
|
for arch in cmd["image_conf"]["image-build"]["arches"]:
|
||||||
|
if buildinstall_phase and not buildinstall_phase.reused(variant, arch):
|
||||||
|
self.pool.log_info(log_msg % "buildinstall phase changed")
|
||||||
|
return False
|
||||||
|
|
||||||
|
# Make sure packages in variant not change
|
||||||
|
rpm_manifest_file = compose.paths.compose.metadata("rpms.json")
|
||||||
|
rpm_manifest = Rpms()
|
||||||
|
rpm_manifest.load(rpm_manifest_file)
|
||||||
|
|
||||||
|
old_rpm_manifest_file = compose.paths.old_compose_path(rpm_manifest_file)
|
||||||
|
old_rpm_manifest = Rpms()
|
||||||
|
old_rpm_manifest.load(old_rpm_manifest_file)
|
||||||
|
|
||||||
|
for repo in cmd["original_image_conf"]["image-build"]["repo"]:
|
||||||
|
if repo not in compose.all_variants:
|
||||||
|
# External repos are checked using other logic.
|
||||||
|
continue
|
||||||
|
for arch in cmd["image_conf"]["image-build"]["arches"]:
|
||||||
|
if (
|
||||||
|
rpm_manifest.rpms[variant.uid][arch]
|
||||||
|
!= old_rpm_manifest.rpms[variant.uid][arch]
|
||||||
|
):
|
||||||
|
self.pool.log_info(
|
||||||
|
log_msg % "Packages in %s.%s changed." % (variant.uid, arch)
|
||||||
|
)
|
||||||
|
return False
|
||||||
|
|
||||||
|
self.pool.log_info(
|
||||||
|
"Reusing images from old compose for variant %s" % variant.uid
|
||||||
|
)
|
||||||
|
try:
|
||||||
|
self._link_images(
|
||||||
|
compose, variant, subvariant, cmd, old_metadata["image_infos"]
|
||||||
|
)
|
||||||
|
except Exception as e:
|
||||||
|
self.pool.log_info(log_msg % "Can't link images %s" % str(e))
|
||||||
|
return False
|
||||||
|
|
||||||
|
old_log_file = compose.paths.old_compose_path(log_file)
|
||||||
|
try:
|
||||||
|
shutil.copy2(old_log_file, log_file)
|
||||||
|
except Exception as e:
|
||||||
|
self.pool.log_info(
|
||||||
|
log_msg % "Can't copy old log_file: %s %s" % (old_log_file, str(e))
|
||||||
|
)
|
||||||
|
return False
|
||||||
|
|
||||||
|
self._write_reuse_metadata(
|
||||||
|
compose,
|
||||||
|
metadata_file,
|
||||||
|
cmd,
|
||||||
|
old_metadata["image_infos"],
|
||||||
|
external_repo_checksum,
|
||||||
|
)
|
||||||
|
|
||||||
|
return True
|
||||||
|
|
||||||
|
def _write_reuse_metadata(
|
||||||
|
self, compose, metadata_file, cmd, image_infos, external_repo_checksum
|
||||||
|
):
|
||||||
|
"""Write metadata file.
|
||||||
|
|
||||||
|
:param Compose compose: Current compose.
|
||||||
|
:param str metadata_file: Path to reuse metadata file.
|
||||||
|
:param dict cmd: Dict of params for image-build.
|
||||||
|
:param dict image_infos: Dict contains image info.
|
||||||
|
:param dict external_repo_checksum: Dict contains checksum of repomd.xml
|
||||||
|
or None if can't get checksum.
|
||||||
|
"""
|
||||||
|
msg = "Writing reuse metadata file: %s" % metadata_file
|
||||||
|
self.pool.log_info(msg)
|
||||||
|
|
||||||
|
cmd_copy = copy.deepcopy(cmd)
|
||||||
|
del cmd_copy["image_conf"]["image-build"]["variant"]
|
||||||
|
|
||||||
|
data = {
|
||||||
|
"cmd": cmd_copy,
|
||||||
|
"image_infos": image_infos,
|
||||||
|
"external_repo_checksum": external_repo_checksum,
|
||||||
|
}
|
||||||
|
try:
|
||||||
|
with open(metadata_file, "w") as f:
|
||||||
|
json.dump(data, f, indent=4)
|
||||||
|
except Exception as e:
|
||||||
|
self.pool.log_info("%s Failed: %s" % (msg, str(e)))
|
||||||
|
|
||||||
|
def _load_reuse_metadata(self, metadata_file):
|
||||||
|
"""Load metadata file.
|
||||||
|
|
||||||
|
:param str metadata_file: Path to reuse metadata file.
|
||||||
|
"""
|
||||||
|
with open(metadata_file, "r") as f:
|
||||||
|
return json.load(f)
|
||||||
|
@ -406,7 +406,7 @@ def run_compose(
|
|||||||
extra_isos_phase = pungi.phases.ExtraIsosPhase(compose)
|
extra_isos_phase = pungi.phases.ExtraIsosPhase(compose)
|
||||||
liveimages_phase = pungi.phases.LiveImagesPhase(compose)
|
liveimages_phase = pungi.phases.LiveImagesPhase(compose)
|
||||||
livemedia_phase = pungi.phases.LiveMediaPhase(compose)
|
livemedia_phase = pungi.phases.LiveMediaPhase(compose)
|
||||||
image_build_phase = pungi.phases.ImageBuildPhase(compose)
|
image_build_phase = pungi.phases.ImageBuildPhase(compose, buildinstall_phase)
|
||||||
osbuild_phase = pungi.phases.OSBuildPhase(compose)
|
osbuild_phase = pungi.phases.OSBuildPhase(compose)
|
||||||
osbs_phase = pungi.phases.OSBSPhase(compose)
|
osbs_phase = pungi.phases.OSBSPhase(compose)
|
||||||
image_container_phase = pungi.phases.ImageContainerPhase(compose)
|
image_container_phase = pungi.phases.ImageContainerPhase(compose)
|
||||||
|
@ -17,12 +17,7 @@ class TestImageBuildPhase(PungiTestCase):
|
|||||||
|
|
||||||
@mock.patch("pungi.phases.image_build.ThreadPool")
|
@mock.patch("pungi.phases.image_build.ThreadPool")
|
||||||
def test_image_build(self, ThreadPool):
|
def test_image_build(self, ThreadPool):
|
||||||
compose = DummyCompose(
|
original_image_conf = {
|
||||||
self.topdir,
|
|
||||||
{
|
|
||||||
"image_build": {
|
|
||||||
"^Client|Server$": [
|
|
||||||
{
|
|
||||||
"image-build": {
|
"image-build": {
|
||||||
"format": [("docker", "tar.xz")],
|
"format": [("docker", "tar.xz")],
|
||||||
"name": "Fedora-Docker-Base",
|
"name": "Fedora-Docker-Base",
|
||||||
@ -35,8 +30,10 @@ class TestImageBuildPhase(PungiTestCase):
|
|||||||
"failable": ["x86_64"],
|
"failable": ["x86_64"],
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
]
|
compose = DummyCompose(
|
||||||
},
|
self.topdir,
|
||||||
|
{
|
||||||
|
"image_build": {"^Client|Server$": [original_image_conf]},
|
||||||
"koji_profile": "koji",
|
"koji_profile": "koji",
|
||||||
},
|
},
|
||||||
)
|
)
|
||||||
@ -50,6 +47,7 @@ class TestImageBuildPhase(PungiTestCase):
|
|||||||
# assert at least one thread was started
|
# assert at least one thread was started
|
||||||
self.assertTrue(phase.pool.add.called)
|
self.assertTrue(phase.pool.add.called)
|
||||||
client_args = {
|
client_args = {
|
||||||
|
"original_image_conf": original_image_conf,
|
||||||
"image_conf": {
|
"image_conf": {
|
||||||
"image-build": {
|
"image-build": {
|
||||||
"install_tree": self.topdir + "/compose/Client/$arch/os",
|
"install_tree": self.topdir + "/compose/Client/$arch/os",
|
||||||
@ -75,6 +73,7 @@ class TestImageBuildPhase(PungiTestCase):
|
|||||||
"scratch": False,
|
"scratch": False,
|
||||||
}
|
}
|
||||||
server_args = {
|
server_args = {
|
||||||
|
"original_image_conf": original_image_conf,
|
||||||
"image_conf": {
|
"image_conf": {
|
||||||
"image-build": {
|
"image-build": {
|
||||||
"install_tree": self.topdir + "/compose/Server/$arch/os",
|
"install_tree": self.topdir + "/compose/Server/$arch/os",
|
||||||
@ -102,21 +101,15 @@ class TestImageBuildPhase(PungiTestCase):
|
|||||||
six.assertCountEqual(
|
six.assertCountEqual(
|
||||||
self,
|
self,
|
||||||
phase.pool.queue_put.mock_calls,
|
phase.pool.queue_put.mock_calls,
|
||||||
[mock.call((compose, client_args)), mock.call((compose, server_args))],
|
[
|
||||||
|
mock.call((compose, client_args, phase.buildinstall_phase)),
|
||||||
|
mock.call((compose, server_args, phase.buildinstall_phase)),
|
||||||
|
],
|
||||||
)
|
)
|
||||||
|
|
||||||
@mock.patch("pungi.phases.image_build.ThreadPool")
|
@mock.patch("pungi.phases.image_build.ThreadPool")
|
||||||
def test_image_build_phase_global_options(self, ThreadPool):
|
def test_image_build_phase_global_options(self, ThreadPool):
|
||||||
compose = DummyCompose(
|
original_image_conf = {
|
||||||
self.topdir,
|
|
||||||
{
|
|
||||||
"image_build_ksurl": "git://git.fedorahosted.org/git/spin-kickstarts.git", # noqa: E501
|
|
||||||
"image_build_release": "!RELEASE_FROM_LABEL_DATE_TYPE_RESPIN",
|
|
||||||
"image_build_target": "f24",
|
|
||||||
"image_build_version": "Rawhide",
|
|
||||||
"image_build": {
|
|
||||||
"^Server$": [
|
|
||||||
{
|
|
||||||
"image-build": {
|
"image-build": {
|
||||||
"format": ["docker"],
|
"format": ["docker"],
|
||||||
"name": "Fedora-Docker-Base",
|
"name": "Fedora-Docker-Base",
|
||||||
@ -125,8 +118,14 @@ class TestImageBuildPhase(PungiTestCase):
|
|||||||
"disk_size": 3,
|
"disk_size": 3,
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
]
|
compose = DummyCompose(
|
||||||
},
|
self.topdir,
|
||||||
|
{
|
||||||
|
"image_build_ksurl": "git://git.fedorahosted.org/git/spin-kickstarts.git", # noqa: E501
|
||||||
|
"image_build_release": "!RELEASE_FROM_LABEL_DATE_TYPE_RESPIN",
|
||||||
|
"image_build_target": "f24",
|
||||||
|
"image_build_version": "Rawhide",
|
||||||
|
"image_build": {"^Server$": [original_image_conf]},
|
||||||
"koji_profile": "koji",
|
"koji_profile": "koji",
|
||||||
},
|
},
|
||||||
)
|
)
|
||||||
@ -140,6 +139,7 @@ class TestImageBuildPhase(PungiTestCase):
|
|||||||
# assert at least one thread was started
|
# assert at least one thread was started
|
||||||
self.assertTrue(phase.pool.add.called)
|
self.assertTrue(phase.pool.add.called)
|
||||||
server_args = {
|
server_args = {
|
||||||
|
"original_image_conf": original_image_conf,
|
||||||
"image_conf": {
|
"image_conf": {
|
||||||
"image-build": {
|
"image-build": {
|
||||||
"install_tree": self.topdir + "/compose/Server/$arch/os",
|
"install_tree": self.topdir + "/compose/Server/$arch/os",
|
||||||
@ -165,20 +165,13 @@ class TestImageBuildPhase(PungiTestCase):
|
|||||||
"scratch": False,
|
"scratch": False,
|
||||||
}
|
}
|
||||||
self.assertEqual(
|
self.assertEqual(
|
||||||
phase.pool.queue_put.mock_calls, [mock.call((compose, server_args))]
|
phase.pool.queue_put.mock_calls,
|
||||||
|
[mock.call((compose, server_args, phase.buildinstall_phase))],
|
||||||
)
|
)
|
||||||
|
|
||||||
@mock.patch("pungi.phases.image_build.ThreadPool")
|
@mock.patch("pungi.phases.image_build.ThreadPool")
|
||||||
def test_image_build_phase_missing_version(self, ThreadPool):
|
def test_image_build_phase_missing_version(self, ThreadPool):
|
||||||
compose = DummyCompose(
|
original_image_conf = {
|
||||||
self.topdir,
|
|
||||||
{
|
|
||||||
"image_build_ksurl": "git://git.fedorahosted.org/git/spin-kickstarts.git", # noqa: E501
|
|
||||||
"image_build_release": "!RELEASE_FROM_LABEL_DATE_TYPE_RESPIN",
|
|
||||||
"image_build_target": "f24",
|
|
||||||
"image_build": {
|
|
||||||
"^Server$": [
|
|
||||||
{
|
|
||||||
"image-build": {
|
"image-build": {
|
||||||
"format": "docker",
|
"format": "docker",
|
||||||
"name": "Fedora-Docker-Base",
|
"name": "Fedora-Docker-Base",
|
||||||
@ -187,8 +180,13 @@ class TestImageBuildPhase(PungiTestCase):
|
|||||||
"disk_size": 3,
|
"disk_size": 3,
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
]
|
compose = DummyCompose(
|
||||||
},
|
self.topdir,
|
||||||
|
{
|
||||||
|
"image_build_ksurl": "git://git.fedorahosted.org/git/spin-kickstarts.git", # noqa: E501
|
||||||
|
"image_build_release": "!RELEASE_FROM_LABEL_DATE_TYPE_RESPIN",
|
||||||
|
"image_build_target": "f24",
|
||||||
|
"image_build": {"^Server$": [original_image_conf]},
|
||||||
"koji_profile": "koji",
|
"koji_profile": "koji",
|
||||||
},
|
},
|
||||||
)
|
)
|
||||||
@ -200,6 +198,7 @@ class TestImageBuildPhase(PungiTestCase):
|
|||||||
# assert at least one thread was started
|
# assert at least one thread was started
|
||||||
self.assertTrue(phase.pool.add.called)
|
self.assertTrue(phase.pool.add.called)
|
||||||
server_args = {
|
server_args = {
|
||||||
|
"original_image_conf": original_image_conf,
|
||||||
"image_conf": {
|
"image_conf": {
|
||||||
"image-build": {
|
"image-build": {
|
||||||
"install_tree": self.topdir + "/compose/Server/$arch/os",
|
"install_tree": self.topdir + "/compose/Server/$arch/os",
|
||||||
@ -225,7 +224,8 @@ class TestImageBuildPhase(PungiTestCase):
|
|||||||
"scratch": False,
|
"scratch": False,
|
||||||
}
|
}
|
||||||
self.assertEqual(
|
self.assertEqual(
|
||||||
phase.pool.queue_put.mock_calls, [mock.call((compose, server_args))]
|
phase.pool.queue_put.mock_calls,
|
||||||
|
[mock.call((compose, server_args, phase.buildinstall_phase))],
|
||||||
)
|
)
|
||||||
|
|
||||||
@mock.patch("pungi.phases.image_build.ThreadPool")
|
@mock.patch("pungi.phases.image_build.ThreadPool")
|
||||||
@ -266,12 +266,7 @@ class TestImageBuildPhase(PungiTestCase):
|
|||||||
|
|
||||||
@mock.patch("pungi.phases.image_build.ThreadPool")
|
@mock.patch("pungi.phases.image_build.ThreadPool")
|
||||||
def test_image_build_set_install_tree(self, ThreadPool):
|
def test_image_build_set_install_tree(self, ThreadPool):
|
||||||
compose = DummyCompose(
|
original_image_conf = {
|
||||||
self.topdir,
|
|
||||||
{
|
|
||||||
"image_build": {
|
|
||||||
"^Server$": [
|
|
||||||
{
|
|
||||||
"image-build": {
|
"image-build": {
|
||||||
"format": ["docker"],
|
"format": ["docker"],
|
||||||
"name": "Fedora-Docker-Base",
|
"name": "Fedora-Docker-Base",
|
||||||
@ -285,8 +280,11 @@ class TestImageBuildPhase(PungiTestCase):
|
|||||||
"install_tree_from": "Server-optional",
|
"install_tree_from": "Server-optional",
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
]
|
|
||||||
},
|
compose = DummyCompose(
|
||||||
|
self.topdir,
|
||||||
|
{
|
||||||
|
"image_build": {"^Server$": [original_image_conf]},
|
||||||
"koji_profile": "koji",
|
"koji_profile": "koji",
|
||||||
},
|
},
|
||||||
)
|
)
|
||||||
@ -307,6 +305,7 @@ class TestImageBuildPhase(PungiTestCase):
|
|||||||
self.assertDictEqual(
|
self.assertDictEqual(
|
||||||
args[0][1],
|
args[0][1],
|
||||||
{
|
{
|
||||||
|
"original_image_conf": original_image_conf,
|
||||||
"image_conf": {
|
"image_conf": {
|
||||||
"image-build": {
|
"image-build": {
|
||||||
"install_tree": self.topdir
|
"install_tree": self.topdir
|
||||||
@ -335,12 +334,7 @@ class TestImageBuildPhase(PungiTestCase):
|
|||||||
|
|
||||||
@mock.patch("pungi.phases.image_build.ThreadPool")
|
@mock.patch("pungi.phases.image_build.ThreadPool")
|
||||||
def test_image_build_set_install_tree_from_path(self, ThreadPool):
|
def test_image_build_set_install_tree_from_path(self, ThreadPool):
|
||||||
compose = DummyCompose(
|
original_image_conf = {
|
||||||
self.topdir,
|
|
||||||
{
|
|
||||||
"image_build": {
|
|
||||||
"^Server$": [
|
|
||||||
{
|
|
||||||
"image-build": {
|
"image-build": {
|
||||||
"format": ["docker"],
|
"format": ["docker"],
|
||||||
"name": "Fedora-Docker-Base",
|
"name": "Fedora-Docker-Base",
|
||||||
@ -354,8 +348,10 @@ class TestImageBuildPhase(PungiTestCase):
|
|||||||
"install_tree_from": "/my/tree",
|
"install_tree_from": "/my/tree",
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
]
|
compose = DummyCompose(
|
||||||
},
|
self.topdir,
|
||||||
|
{
|
||||||
|
"image_build": {"^Server$": [original_image_conf]},
|
||||||
"koji_profile": "koji",
|
"koji_profile": "koji",
|
||||||
"translate_paths": [("/my", "http://example.com")],
|
"translate_paths": [("/my", "http://example.com")],
|
||||||
},
|
},
|
||||||
@ -376,6 +372,7 @@ class TestImageBuildPhase(PungiTestCase):
|
|||||||
self.assertDictEqual(
|
self.assertDictEqual(
|
||||||
args[0][1],
|
args[0][1],
|
||||||
{
|
{
|
||||||
|
"original_image_conf": original_image_conf,
|
||||||
"image_conf": {
|
"image_conf": {
|
||||||
"image-build": {
|
"image-build": {
|
||||||
"install_tree": "http://example.com/tree",
|
"install_tree": "http://example.com/tree",
|
||||||
@ -403,12 +400,7 @@ class TestImageBuildPhase(PungiTestCase):
|
|||||||
|
|
||||||
@mock.patch("pungi.phases.image_build.ThreadPool")
|
@mock.patch("pungi.phases.image_build.ThreadPool")
|
||||||
def test_image_build_set_extra_repos(self, ThreadPool):
|
def test_image_build_set_extra_repos(self, ThreadPool):
|
||||||
compose = DummyCompose(
|
original_image_conf = {
|
||||||
self.topdir,
|
|
||||||
{
|
|
||||||
"image_build": {
|
|
||||||
"^Server$": [
|
|
||||||
{
|
|
||||||
"image-build": {
|
"image-build": {
|
||||||
"format": ["docker"],
|
"format": ["docker"],
|
||||||
"name": "Fedora-Docker-Base",
|
"name": "Fedora-Docker-Base",
|
||||||
@ -422,8 +414,10 @@ class TestImageBuildPhase(PungiTestCase):
|
|||||||
"repo_from": ["Everything", "Server-optional"],
|
"repo_from": ["Everything", "Server-optional"],
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
]
|
compose = DummyCompose(
|
||||||
},
|
self.topdir,
|
||||||
|
{
|
||||||
|
"image_build": {"^Server$": [original_image_conf]},
|
||||||
"koji_profile": "koji",
|
"koji_profile": "koji",
|
||||||
},
|
},
|
||||||
)
|
)
|
||||||
@ -444,6 +438,7 @@ class TestImageBuildPhase(PungiTestCase):
|
|||||||
self.assertDictEqual(
|
self.assertDictEqual(
|
||||||
args[0][1],
|
args[0][1],
|
||||||
{
|
{
|
||||||
|
"original_image_conf": original_image_conf,
|
||||||
"image_conf": {
|
"image_conf": {
|
||||||
"image-build": {
|
"image-build": {
|
||||||
"install_tree": self.topdir + "/compose/Server/$arch/os",
|
"install_tree": self.topdir + "/compose/Server/$arch/os",
|
||||||
@ -477,12 +472,7 @@ class TestImageBuildPhase(PungiTestCase):
|
|||||||
|
|
||||||
@mock.patch("pungi.phases.image_build.ThreadPool")
|
@mock.patch("pungi.phases.image_build.ThreadPool")
|
||||||
def test_image_build_set_external_install_tree(self, ThreadPool):
|
def test_image_build_set_external_install_tree(self, ThreadPool):
|
||||||
compose = DummyCompose(
|
original_image_conf = {
|
||||||
self.topdir,
|
|
||||||
{
|
|
||||||
"image_build": {
|
|
||||||
"^Server$": [
|
|
||||||
{
|
|
||||||
"image-build": {
|
"image-build": {
|
||||||
"format": ["docker"],
|
"format": ["docker"],
|
||||||
"name": "Fedora-Docker-Base",
|
"name": "Fedora-Docker-Base",
|
||||||
@ -496,8 +486,10 @@ class TestImageBuildPhase(PungiTestCase):
|
|||||||
"install_tree_from": "http://example.com/install-tree/",
|
"install_tree_from": "http://example.com/install-tree/",
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
]
|
compose = DummyCompose(
|
||||||
},
|
self.topdir,
|
||||||
|
{
|
||||||
|
"image_build": {"^Server$": [original_image_conf]},
|
||||||
"koji_profile": "koji",
|
"koji_profile": "koji",
|
||||||
},
|
},
|
||||||
)
|
)
|
||||||
@ -517,6 +509,7 @@ class TestImageBuildPhase(PungiTestCase):
|
|||||||
self.assertDictEqual(
|
self.assertDictEqual(
|
||||||
args[0][1],
|
args[0][1],
|
||||||
{
|
{
|
||||||
|
"original_image_conf": original_image_conf,
|
||||||
"image_conf": {
|
"image_conf": {
|
||||||
"image-build": {
|
"image-build": {
|
||||||
"install_tree": "http://example.com/install-tree/",
|
"install_tree": "http://example.com/install-tree/",
|
||||||
@ -670,12 +663,7 @@ class TestImageBuildPhase(PungiTestCase):
|
|||||||
|
|
||||||
@mock.patch("pungi.phases.image_build.ThreadPool")
|
@mock.patch("pungi.phases.image_build.ThreadPool")
|
||||||
def test_image_build_optional(self, ThreadPool):
|
def test_image_build_optional(self, ThreadPool):
|
||||||
compose = DummyCompose(
|
original_image_conf = {
|
||||||
self.topdir,
|
|
||||||
{
|
|
||||||
"image_build": {
|
|
||||||
"^Server-optional$": [
|
|
||||||
{
|
|
||||||
"image-build": {
|
"image-build": {
|
||||||
"format": ["docker"],
|
"format": ["docker"],
|
||||||
"name": "Fedora-Docker-Base",
|
"name": "Fedora-Docker-Base",
|
||||||
@ -688,8 +676,10 @@ class TestImageBuildPhase(PungiTestCase):
|
|||||||
"failable": ["x86_64"],
|
"failable": ["x86_64"],
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
]
|
compose = DummyCompose(
|
||||||
},
|
self.topdir,
|
||||||
|
{
|
||||||
|
"image_build": {"^Server-optional$": [original_image_conf]},
|
||||||
"koji_profile": "koji",
|
"koji_profile": "koji",
|
||||||
},
|
},
|
||||||
)
|
)
|
||||||
@ -704,6 +694,7 @@ class TestImageBuildPhase(PungiTestCase):
|
|||||||
# assert at least one thread was started
|
# assert at least one thread was started
|
||||||
self.assertTrue(phase.pool.add.called)
|
self.assertTrue(phase.pool.add.called)
|
||||||
server_args = {
|
server_args = {
|
||||||
|
"original_image_conf": original_image_conf,
|
||||||
"image_conf": {
|
"image_conf": {
|
||||||
"image-build": {
|
"image-build": {
|
||||||
"install_tree": self.topdir + "/compose/Server/$arch/os",
|
"install_tree": self.topdir + "/compose/Server/$arch/os",
|
||||||
@ -729,17 +720,13 @@ class TestImageBuildPhase(PungiTestCase):
|
|||||||
"scratch": False,
|
"scratch": False,
|
||||||
}
|
}
|
||||||
self.assertEqual(
|
self.assertEqual(
|
||||||
phase.pool.queue_put.mock_calls, [mock.call((compose, server_args))]
|
phase.pool.queue_put.mock_calls,
|
||||||
|
[mock.call((compose, server_args, phase.buildinstall_phase))],
|
||||||
)
|
)
|
||||||
|
|
||||||
@mock.patch("pungi.phases.image_build.ThreadPool")
|
@mock.patch("pungi.phases.image_build.ThreadPool")
|
||||||
def test_failable_star(self, ThreadPool):
|
def test_failable_star(self, ThreadPool):
|
||||||
compose = DummyCompose(
|
original_image_conf = {
|
||||||
self.topdir,
|
|
||||||
{
|
|
||||||
"image_build": {
|
|
||||||
"^Server$": [
|
|
||||||
{
|
|
||||||
"image-build": {
|
"image-build": {
|
||||||
"format": ["docker"],
|
"format": ["docker"],
|
||||||
"name": "Fedora-Docker-Base",
|
"name": "Fedora-Docker-Base",
|
||||||
@ -752,8 +739,10 @@ class TestImageBuildPhase(PungiTestCase):
|
|||||||
"failable": ["*"],
|
"failable": ["*"],
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
]
|
compose = DummyCompose(
|
||||||
},
|
self.topdir,
|
||||||
|
{
|
||||||
|
"image_build": {"^Server$": [original_image_conf]},
|
||||||
"koji_profile": "koji",
|
"koji_profile": "koji",
|
||||||
},
|
},
|
||||||
)
|
)
|
||||||
@ -768,6 +757,7 @@ class TestImageBuildPhase(PungiTestCase):
|
|||||||
# assert at least one thread was started
|
# assert at least one thread was started
|
||||||
self.assertTrue(phase.pool.add.called)
|
self.assertTrue(phase.pool.add.called)
|
||||||
server_args = {
|
server_args = {
|
||||||
|
"original_image_conf": original_image_conf,
|
||||||
"image_conf": {
|
"image_conf": {
|
||||||
"image-build": {
|
"image-build": {
|
||||||
"install_tree": self.topdir + "/compose/Server/$arch/os",
|
"install_tree": self.topdir + "/compose/Server/$arch/os",
|
||||||
@ -793,7 +783,8 @@ class TestImageBuildPhase(PungiTestCase):
|
|||||||
"scratch": False,
|
"scratch": False,
|
||||||
}
|
}
|
||||||
self.assertEqual(
|
self.assertEqual(
|
||||||
phase.pool.queue_put.mock_calls, [mock.call((compose, server_args))]
|
phase.pool.queue_put.mock_calls,
|
||||||
|
[mock.call((compose, server_args, phase.buildinstall_phase))],
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
@ -854,7 +845,7 @@ class TestCreateImageBuildThread(PungiTestCase):
|
|||||||
|
|
||||||
t = CreateImageBuildThread(pool)
|
t = CreateImageBuildThread(pool)
|
||||||
with mock.patch("time.sleep"):
|
with mock.patch("time.sleep"):
|
||||||
t.process((compose, cmd), 1)
|
t.process((compose, cmd, None), 1)
|
||||||
|
|
||||||
self.assertEqual(
|
self.assertEqual(
|
||||||
koji_wrapper.get_image_build_cmd.call_args_list,
|
koji_wrapper.get_image_build_cmd.call_args_list,
|
||||||
@ -987,7 +978,7 @@ class TestCreateImageBuildThread(PungiTestCase):
|
|||||||
|
|
||||||
t = CreateImageBuildThread(pool)
|
t = CreateImageBuildThread(pool)
|
||||||
with mock.patch("time.sleep"):
|
with mock.patch("time.sleep"):
|
||||||
t.process((compose, cmd), 1)
|
t.process((compose, cmd, None), 1)
|
||||||
|
|
||||||
pool._logger.error.assert_has_calls(
|
pool._logger.error.assert_has_calls(
|
||||||
[
|
[
|
||||||
@ -1041,7 +1032,7 @@ class TestCreateImageBuildThread(PungiTestCase):
|
|||||||
|
|
||||||
t = CreateImageBuildThread(pool)
|
t = CreateImageBuildThread(pool)
|
||||||
with mock.patch("time.sleep"):
|
with mock.patch("time.sleep"):
|
||||||
t.process((compose, cmd), 1)
|
t.process((compose, cmd, None), 1)
|
||||||
|
|
||||||
pool._logger.error.assert_has_calls(
|
pool._logger.error.assert_has_calls(
|
||||||
[
|
[
|
||||||
@ -1092,4 +1083,4 @@ class TestCreateImageBuildThread(PungiTestCase):
|
|||||||
t = CreateImageBuildThread(pool)
|
t = CreateImageBuildThread(pool)
|
||||||
with self.assertRaises(RuntimeError):
|
with self.assertRaises(RuntimeError):
|
||||||
with mock.patch("time.sleep"):
|
with mock.patch("time.sleep"):
|
||||||
t.process((compose, cmd), 1)
|
t.process((compose, cmd, None), 1)
|
||||||
|
Loading…
Reference in New Issue
Block a user