osbs: Reuse images from old compose

JIRA: RHELCMP-5972
Signed-off-by: Haibo Lin <hlin@redhat.com>
This commit is contained in:
Haibo Lin 2021-09-09 13:58:32 +08:00
parent e42e65783d
commit 8133676270
4 changed files with 264 additions and 29 deletions

View File

@ -1202,6 +1202,7 @@ def make_schema():
"anyOf": [{"type": "string"}, {"type": "number"}],
"default": 10 * 1024 * 1024,
},
"osbs_allow_reuse": {"type": "boolean", "default": False},
"osbs": {
"type": "object",
"patternProperties": {

View File

@ -1,23 +1,29 @@
# -*- coding: utf-8 -*-
import copy
import fnmatch
import json
import os
from kobo.threads import ThreadPool, WorkerThread
from kobo import shortcuts
from productmd.rpms import Rpms
from six.moves import configparser
from .base import ConfigGuardedPhase, PhaseLoggerMixin
from .. import util
from ..wrappers import kojiwrapper
from ..wrappers.scm import get_file_from_scm
class OSBSPhase(PhaseLoggerMixin, ConfigGuardedPhase):
name = "osbs"
def __init__(self, compose):
def __init__(self, compose, pkgset_phase, buildinstall_phase):
super(OSBSPhase, self).__init__(compose)
self.pool = ThreadPool(logger=self.logger)
self.pool.registries = {}
self.pool.pkgset_phase = pkgset_phase
self.pool.buildinstall_phase = buildinstall_phase
def run(self):
for variant in self.compose.get_variants():
@ -77,8 +83,8 @@ class OSBSThread(WorkerThread):
def worker(self, compose, variant, config):
msg = "OSBS task for variant %s" % variant.uid
self.pool.log_info("[BEGIN] %s" % msg)
koji = kojiwrapper.KojiWrapper(compose)
koji.login()
original_config = copy.deepcopy(config)
# Start task
source = config.pop("url")
@ -94,33 +100,99 @@ class OSBSThread(WorkerThread):
config["yum_repourls"] = repos
task_id = koji.koji_proxy.buildContainer(
source, target, config, priority=priority
)
koji.save_task_id(task_id)
# Wait for it to finish and capture the output into log file (even
# though there is not much there).
log_dir = os.path.join(compose.paths.log.topdir(), "osbs")
util.makedirs(log_dir)
log_file = os.path.join(
log_dir, "%s-%s-watch-task.log" % (variant.uid, self.num)
)
reuse_file = log_file[:-4] + ".reuse.json"
try:
image_conf = self._get_image_conf(compose, original_config)
except Exception as e:
image_conf = None
self.pool.log_info(
"Can't get image-build.conf for variant: %s source: %s - %s"
% (variant.uid, source, str(e))
)
koji = kojiwrapper.KojiWrapper(compose)
koji.login()
task_id = self._try_to_reuse(
compose, variant, original_config, image_conf, reuse_file
)
if not task_id:
task_id = koji.koji_proxy.buildContainer(
source, target, config, priority=priority
)
koji.save_task_id(task_id)
# Wait for it to finish and capture the output into log file (even
# though there is not much there).
if koji.watch_task(task_id, log_file) != 0:
raise RuntimeError(
"OSBS: task %s failed: see %s for details" % (task_id, log_file)
)
scratch = config.get("scratch", False)
nvr = add_metadata(variant, task_id, compose, scratch)
nvr, archive_ids = add_metadata(variant, task_id, compose, scratch)
if nvr:
registry = get_registry(compose, nvr, registry)
if registry:
self.pool.registries[nvr] = registry
self._write_reuse_metadata(
compose,
variant,
original_config,
image_conf,
task_id,
archive_ids,
reuse_file,
)
self.pool.log_info("[DONE ] %s" % msg)
def _get_image_conf(self, compose, config):
"""Get image-build.conf from git repo.
:param Compose compose: Current compose.
:param dict config: One osbs config item of compose.conf["osbs"][$variant]
"""
tmp_dir = compose.mkdtemp(prefix="osbs_")
url = config["url"].split("#")
if len(url) == 1:
url.append(config["git_branch"])
filename = "image-build.conf"
get_file_from_scm(
{
"scm": "git",
"repo": url[0],
"branch": url[1],
"file": [filename],
},
tmp_dir,
)
c = configparser.ConfigParser()
c.read(os.path.join(tmp_dir, filename))
return c
def _get_ksurl(self, image_conf):
"""Get ksurl from image-build.conf"""
ksurl = image_conf.get("image-build", "ksurl")
if ksurl:
resolver = util.GitUrlResolver(offline=False)
return resolver(ksurl)
else:
return None
def _get_repo(self, compose, repo, gpgkey=None):
"""
Return repo file URL of repo, if repo contains "://", it's already a
@ -177,6 +249,151 @@ class OSBSThread(WorkerThread):
return util.translate_path(compose, repo_file)
def _try_to_reuse(self, compose, variant, config, image_conf, reuse_file):
"""Try to reuse results of old compose.
:param Compose compose: Current compose.
:param Variant variant: Current variant.
:param dict config: One osbs config item of compose.conf["osbs"][$variant]
:param ConfigParser image_conf: ConfigParser obj of image-build.conf.
:param str reuse_file: Path to reuse metadata file
"""
log_msg = "Cannot reuse old osbs phase results - %s"
if not compose.conf["osbs_allow_reuse"]:
self.pool.log_info(log_msg % "reuse of old osbs results is disabled.")
return False
old_reuse_file = compose.paths.old_compose_path(reuse_file)
if not old_reuse_file:
self.pool.log_info(log_msg % "Can't find old reuse metadata file")
return False
try:
with open(old_reuse_file) as f:
old_reuse_metadata = json.load(f)
except Exception as e:
self.pool.log_info(
log_msg % "Can't load old reuse metadata file: %s" % str(e)
)
return False
if old_reuse_metadata["config"] != config:
self.pool.log_info(log_msg % "osbs config changed")
return False
if not image_conf:
self.pool.log_info(log_msg % "Can't get image-build.conf")
return False
# Make sure ksurl not change
try:
ksurl = self._get_ksurl(image_conf)
except Exception as e:
self.pool.log_info(
log_msg % "Can't get ksurl from image-build.conf - %s" % str(e)
)
return False
if not old_reuse_metadata["ksurl"]:
self.pool.log_info(
log_msg % "Can't get ksurl from old compose reuse metadata."
)
return False
if ksurl != old_reuse_metadata["ksurl"]:
self.pool.log_info(log_msg % "ksurl changed")
return False
# Make sure buildinstall phase is reused
try:
arches = image_conf.get("image-build", "arches").split(",")
except Exception as e:
self.pool.log_info(
log_msg % "Can't get arches from image-build.conf - %s" % str(e)
)
for arch in arches:
if not self.pool.buildinstall_phase.reused(variant, arch):
self.pool.log_info(
log_msg % "buildinstall phase changed %s.%s" % (variant, arch)
)
return False
# Make sure rpms installed in image exists in current compose
rpm_manifest_file = compose.paths.compose.metadata("rpms.json")
rpm_manifest = Rpms()
rpm_manifest.load(rpm_manifest_file)
rpms = set()
for variant in rpm_manifest.rpms:
for arch in rpm_manifest.rpms[variant]:
for src in rpm_manifest.rpms[variant][arch]:
for nevra in rpm_manifest.rpms[variant][arch][src]:
rpms.add(nevra)
for nevra in old_reuse_metadata["rpmlist"]:
if nevra not in rpms:
self.pool.log_info(
log_msg % "%s does not exist in current compose" % nevra
)
return False
self.pool.log_info(
"Reusing old OSBS task %d result" % old_reuse_file["task_id"]
)
return old_reuse_file["task_id"]
def _write_reuse_metadata(
self, compose, variant, config, image_conf, task_id, archive_ids, reuse_file
):
"""Write metadata to file for reusing.
:param Compose compose: Current compose.
:param Variant variant: Current variant.
:param dict config: One osbs config item of compose.conf["osbs"][$variant]
:param ConfigParser image_conf: ConfigParser obj of image-build.conf.
:param int task_id: Koji task id of osbs task.
:param list archive_ids: List of koji archive id
:param str reuse_file: Path to reuse metadata file.
"""
msg = "Writing reuse metadata file %s" % reuse_file
compose.log_info(msg)
rpmlist = set()
koji = kojiwrapper.KojiWrapper(compose)
for archive_id in archive_ids:
rpms = koji.koji_proxy.listRPMs(imageID=archive_id)
for item in rpms:
if item["epoch"]:
rpmlist.add(
"%s:%s-%s-%s.%s"
% (
item["name"],
item["epoch"],
item["version"],
item["release"],
item["arch"],
)
)
else:
rpmlist.add("%s.%s" % (item["nvr"], item["arch"]))
try:
ksurl = self._get_ksurl(image_conf)
except Exception:
ksurl = None
data = {
"config": config,
"ksurl": ksurl,
"rpmlist": sorted(rpmlist),
"task_id": task_id,
}
try:
with open(reuse_file, "w") as f:
json.dump(data, f, indent=4)
except Exception as e:
compose.log_info(msg + " failed - %s" % str(e))
def add_metadata(variant, task_id, compose, is_scratch):
"""Given a task ID, find details about the container and add it to global
@ -200,7 +417,7 @@ def add_metadata(variant, task_id, compose, is_scratch):
compose.containers_metadata.setdefault(variant.uid, {}).setdefault(
"scratch", []
).append(metadata)
return None
return None, []
else:
build_id = int(result["koji_builds"][0])
@ -218,6 +435,7 @@ def add_metadata(variant, task_id, compose, is_scratch):
"creation_time": buildinfo["creation_time"],
}
)
archive_ids = []
for archive in archives:
data = {
"filename": archive["filename"],
@ -234,4 +452,5 @@ def add_metadata(variant, task_id, compose, is_scratch):
compose.containers_metadata.setdefault(variant.uid, {}).setdefault(
arch, []
).append(data)
return nvr
archive_ids.append(archive["id"])
return nvr, archive_ids

View File

@ -408,7 +408,7 @@ def run_compose(
livemedia_phase = pungi.phases.LiveMediaPhase(compose)
image_build_phase = pungi.phases.ImageBuildPhase(compose, buildinstall_phase)
osbuild_phase = pungi.phases.OSBuildPhase(compose)
osbs_phase = pungi.phases.OSBSPhase(compose)
osbs_phase = pungi.phases.OSBSPhase(compose, pkgset_phase, buildinstall_phase)
image_container_phase = pungi.phases.ImageContainerPhase(compose)
image_checksum_phase = pungi.phases.ImageChecksumPhase(compose)
repoclosure_phase = pungi.phases.RepoclosurePhase(compose)

View File

@ -19,7 +19,7 @@ class OSBSPhaseTest(helpers.PungiTestCase):
pool = ThreadPool.return_value
phase = osbs.OSBSPhase(compose)
phase = osbs.OSBSPhase(compose, None, None)
phase.run()
self.assertEqual(len(pool.add.call_args_list), 1)
@ -33,7 +33,7 @@ class OSBSPhaseTest(helpers.PungiTestCase):
compose = helpers.DummyCompose(self.topdir, {})
compose.just_phases = None
compose.skip_phases = []
phase = osbs.OSBSPhase(compose)
phase = osbs.OSBSPhase(compose, None, None)
self.assertTrue(phase.skip())
@mock.patch("pungi.phases.osbs.ThreadPool")
@ -42,7 +42,7 @@ class OSBSPhaseTest(helpers.PungiTestCase):
compose.just_phases = None
compose.skip_phases = []
compose.notifier = mock.Mock()
phase = osbs.OSBSPhase(compose)
phase = osbs.OSBSPhase(compose, None, None)
phase.start()
phase.stop()
phase.pool.registries = {"foo": "bar"}
@ -139,6 +139,8 @@ METADATA = {
}
}
RPMS = []
SCRATCH_TASK_RESULT = {
"koji_builds": [],
"repositories": [
@ -182,6 +184,7 @@ class OSBSThreadTest(helpers.PungiTestCase):
self.wrapper.koji_proxy.getTaskResult.return_value = TASK_RESULT
self.wrapper.koji_proxy.getBuild.return_value = BUILD_INFO
self.wrapper.koji_proxy.listArchives.return_value = ARCHIVES
self.wrapper.koji_proxy.listRPMs.return_value = RPMS
self.wrapper.koji_proxy.getLatestBuilds.return_value = [
mock.Mock(),
mock.Mock(),
@ -233,6 +236,7 @@ class OSBSThreadTest(helpers.PungiTestCase):
[
mock.call.koji_proxy.getBuild(54321),
mock.call.koji_proxy.listArchives(54321),
mock.call.koji_proxy.listRPMs(imageID=1436049),
]
)
self.assertEqual(self.wrapper.mock_calls, expect_calls)
@ -269,8 +273,9 @@ class OSBSThreadTest(helpers.PungiTestCase):
self.assertIn(" Possible reason: %r is a required property" % key, errors)
self.assertEqual([], warnings)
@mock.patch("pungi.phases.osbs.get_file_from_scm")
@mock.patch("pungi.phases.osbs.kojiwrapper.KojiWrapper")
def test_minimal_run(self, KojiWrapper):
def test_minimal_run(self, KojiWrapper, get_file_from_scm):
cfg = {
"url": "git://example.com/repo?#BEEFCAFE",
"target": "f24-docker-candidate",
@ -285,8 +290,9 @@ class OSBSThreadTest(helpers.PungiTestCase):
self._assertCorrectMetadata()
self._assertRepoFile()
@mock.patch("pungi.phases.osbs.get_file_from_scm")
@mock.patch("pungi.phases.osbs.kojiwrapper.KojiWrapper")
def test_run_failable(self, KojiWrapper):
def test_run_failable(self, KojiWrapper, get_file_from_scm):
cfg = {
"url": "git://example.com/repo?#BEEFCAFE",
"target": "f24-docker-candidate",
@ -302,8 +308,9 @@ class OSBSThreadTest(helpers.PungiTestCase):
self._assertCorrectMetadata()
self._assertRepoFile()
@mock.patch("pungi.phases.osbs.get_file_from_scm")
@mock.patch("pungi.phases.osbs.kojiwrapper.KojiWrapper")
def test_run_with_more_args(self, KojiWrapper):
def test_run_with_more_args(self, KojiWrapper, get_file_from_scm):
cfg = {
"url": "git://example.com/repo?#BEEFCAFE",
"target": "f24-docker-candidate",
@ -322,8 +329,9 @@ class OSBSThreadTest(helpers.PungiTestCase):
self._assertCorrectMetadata()
self._assertRepoFile()
@mock.patch("pungi.phases.osbs.get_file_from_scm")
@mock.patch("pungi.phases.osbs.kojiwrapper.KojiWrapper")
def test_run_with_extra_repos(self, KojiWrapper):
def test_run_with_extra_repos(self, KojiWrapper, get_file_from_scm):
cfg = {
"url": "git://example.com/repo?#BEEFCAFE",
"target": "f24-docker-candidate",
@ -395,8 +403,9 @@ class OSBSThreadTest(helpers.PungiTestCase):
self._assertCorrectCalls(options)
self._assertCorrectMetadata()
@mock.patch("pungi.phases.osbs.get_file_from_scm")
@mock.patch("pungi.phases.osbs.kojiwrapper.KojiWrapper")
def test_run_with_deprecated_registry(self, KojiWrapper):
def test_run_with_deprecated_registry(self, KojiWrapper, get_file_from_scm):
cfg = {
"url": "git://example.com/repo?#BEEFCAFE",
"target": "f24-docker-candidate",
@ -426,8 +435,9 @@ class OSBSThreadTest(helpers.PungiTestCase):
self._assertRepoFile(["Server", "Everything"])
self.assertEqual(self.t.pool.registries, {"my-name-1.0-1": {"foo": "bar"}})
@mock.patch("pungi.phases.osbs.get_file_from_scm")
@mock.patch("pungi.phases.osbs.kojiwrapper.KojiWrapper")
def test_run_with_registry(self, KojiWrapper):
def test_run_with_registry(self, KojiWrapper, get_file_from_scm):
cfg = {
"url": "git://example.com/repo?#BEEFCAFE",
"target": "f24-docker-candidate",
@ -457,8 +467,9 @@ class OSBSThreadTest(helpers.PungiTestCase):
self._assertRepoFile(["Server", "Everything"])
self.assertEqual(self.t.pool.registries, {"my-name-1.0-1": [{"foo": "bar"}]})
@mock.patch("pungi.phases.osbs.get_file_from_scm")
@mock.patch("pungi.phases.osbs.kojiwrapper.KojiWrapper")
def test_run_with_extra_repos_in_list(self, KojiWrapper):
def test_run_with_extra_repos_in_list(self, KojiWrapper, get_file_from_scm):
cfg = {
"url": "git://example.com/repo?#BEEFCAFE",
"target": "f24-docker-candidate",
@ -487,8 +498,9 @@ class OSBSThreadTest(helpers.PungiTestCase):
self._assertCorrectMetadata()
self._assertRepoFile(["Server", "Everything", "Client"])
@mock.patch("pungi.phases.osbs.get_file_from_scm")
@mock.patch("pungi.phases.osbs.kojiwrapper.KojiWrapper")
def test_run_with_gpgkey_enabled(self, KojiWrapper):
def test_run_with_gpgkey_enabled(self, KojiWrapper, get_file_from_scm):
gpgkey = "file:///etc/pki/rpm-gpg/RPM-GPG-KEY-redhat-release"
cfg = {
"url": "git://example.com/repo?#BEEFCAFE",
@ -547,8 +559,9 @@ class OSBSThreadTest(helpers.PungiTestCase):
}
self._assertConfigMissing(cfg, "git_branch")
@mock.patch("pungi.phases.osbs.get_file_from_scm")
@mock.patch("pungi.phases.osbs.kojiwrapper.KojiWrapper")
def test_failing_task(self, KojiWrapper):
def test_failing_task(self, KojiWrapper, get_file_from_scm):
cfg = {
"url": "git://example.com/repo?#BEEFCAFE",
"target": "fedora-24-docker-candidate",
@ -563,8 +576,9 @@ class OSBSThreadTest(helpers.PungiTestCase):
self.assertRegex(str(ctx.exception), r"task 12345 failed: see .+ for details")
@mock.patch("pungi.phases.osbs.get_file_from_scm")
@mock.patch("pungi.phases.osbs.kojiwrapper.KojiWrapper")
def test_failing_task_with_failable(self, KojiWrapper):
def test_failing_task_with_failable(self, KojiWrapper, get_file_from_scm):
cfg = {
"url": "git://example.com/repo?#BEEFCAFE",
"target": "fedora-24-docker-candidate",
@ -577,8 +591,9 @@ class OSBSThreadTest(helpers.PungiTestCase):
self.t.process((self.compose, self.compose.variants["Server"], cfg), 1)
@mock.patch("pungi.phases.osbs.get_file_from_scm")
@mock.patch("pungi.phases.osbs.kojiwrapper.KojiWrapper")
def test_scratch_metadata(self, KojiWrapper):
def test_scratch_metadata(self, KojiWrapper, get_file_from_scm):
cfg = {
"url": "git://example.com/repo?#BEEFCAFE",
"target": "f24-docker-candidate",