almalinux/pungi
14
1
Fork 0
Code Issues Pull Requests Releases Activity

Allow using Pungi Koji plugin for ostree phases.

Browse Source 
This commits changes `ostree` and `ostree_installer` phases
so they can run with Koji Pungi plugin instead of the plain runroot.

It is similar to `buildinstall` phase running with Koji plugin.

Signed-off-by: Jan Kaluza <jkaluza@redhat.com>
This commit is contained in:
Jan Kaluza 2020-05-13 07:58:07 +02:00 committed by lsedlar
parent f1eea0b5a6
commit b8c3ca1abe
8 changed files with 380 additions and 73 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

11
doc/configuration.rst
View File

@ -1489,7 +1489,6 @@ repository with a new commit.
reference will not be created.
* ``ostree_ref`` -- (*str*) To override value ``ref`` from ``treefile``.
Example config
--------------
::
@ -1514,6 +1513,11 @@ Example config
}
}
**ostree_use_koji_plugin** = False
(*bool*) -- When set to ``True``, the Koji pungi_ostree task will be
used to execute rpm-ostree instead of runroot. Use only if the Koji instance
has the pungi_ostree plugin installed.
Ostree Installer Settings
=========================
@ -1566,6 +1570,11 @@ an OSTree repository. This always runs in Koji as a ``runroot`` task.
With this option it is possible to opt-in for the overwriting. The
traditional ``boot.iso`` will be in the ``iso/`` subdirectory.
**ostree_installer_use_koji_plugin** = False
(*bool*) -- When set to ``True``, the Koji pungi_buildinstall task will be
used to execute Lorax instead of runroot. Use only if the Koji instance
has the pungi_buildinstall plugin installed.
Example config
--------------

2
pungi/checks.py
View File

@ -1032,6 +1032,8 @@ def make_schema():
"additionalProperties": False,
}
),
"ostree_use_koji_plugin": {"type": "boolean", "default": False},
"ostree_installer_use_koji_plugin": {"type": "boolean", "default": False},
"ostree_installer_overwrite": {"type": "boolean", "default": False},
"live_images": _variant_arch_mapping(
_one_or_list({"$ref": "#/definitions/live_image_config"})

75
pungi/phases/ostree.py
View File

@ -5,6 +5,7 @@ import json
import os
from kobo import shortcuts
from kobo.threads import ThreadPool, WorkerThread
from collections import OrderedDict
from pungi.arch_utils import getBaseArch
from pungi.runroot import Runroot
@ -154,45 +155,49 @@ class OSTreeThread(WorkerThread):
def _run_ostree_cmd(
self, compose, variant, arch, config, config_repo, extra_config_file=None
):
cmd = [
"pungi-make-ostree",
"tree",
"--repo=%s" % config["ostree_repo"],
"--log-dir=%s" % self.logdir,
"--treefile=%s" % os.path.join(config_repo, config["treefile"]),
]
version = util.version_generator(compose, config.get("version"))
if version:
cmd.append("--version=%s" % version)
if extra_config_file:
cmd.append("--extra-config=%s" % extra_config_file)
if config.get("update_summary", False):
cmd.append("--update-summary")
ostree_ref = config.get("ostree_ref")
if ostree_ref:
cmd.append("--ostree-ref=%s" % ostree_ref)
if config.get("force_new_commit", False):
cmd.append("--force-new-commit")
args = OrderedDict(
[
("repo", config["ostree_repo"]),
("log-dir", self.logdir),
("treefile", os.path.join(config_repo, config["treefile"])),
("version", util.version_generator(compose, config.get("version"))),
("extra-config", extra_config_file),
("update-summary", config.get("update_summary", False)),
("ostree-ref", config.get("ostree_ref")),
("force-new-commit", config.get("force_new_commit", False)),
]
)
packages = ["pungi", "ostree", "rpm-ostree"]
log_file = os.path.join(self.logdir, "runroot.log")
mounts = [compose.topdir, config["ostree_repo"]]
runroot = Runroot(compose, phase="ostree")
runroot.run(
cmd,
log_file=log_file,
arch=arch,
packages=packages,
mounts=mounts,
new_chroot=True,
weight=compose.conf["runroot_weights"].get("ostree"),
)
if compose.conf["ostree_use_koji_plugin"]:
runroot.run_pungi_ostree(
dict(args),
log_file=log_file,
arch=arch,
packages=packages,
mounts=mounts,
weight=compose.conf["runroot_weights"].get("ostree"),
)
else:
cmd = ["pungi-make-ostree", "tree"]
for key, value in args.items():
if value is True:
cmd.append("--%s" % key)
elif value:
cmd.append("--%s=%s" % (key, value))
runroot.run(
cmd,
log_file=log_file,
arch=arch,
packages=packages,
mounts=mounts,
new_chroot=True,
weight=compose.conf["runroot_weights"].get("ostree"),
)
def _clone_repo(self, compose, repodir, url, branch):
scm.get_dir_from_scm(

121
pungi/phases/ostree_installer.py
View File

@ -10,7 +10,14 @@ from kobo import shortcuts
from .base import ConfigGuardedPhase, PhaseLoggerMixin
from .. import util
from ..arch import get_valid_arches
from ..util import get_volid, get_repo_urls, version_generator, translate_path
from ..util import (
get_volid,
get_repo_urls,
version_generator,
translate_path,
move_all,
makedirs,
)
from ..wrappers import iso, lorax, scm
from ..runroot import Runroot
@ -200,43 +207,83 @@ class OstreeInstallerThread(WorkerThread):
def _run_ostree_cmd(
self, compose, variant, arch, config, source_repo, output_dir, volid
):
lorax_wrapper = lorax.LoraxWrapper()
lorax_cmd = lorax_wrapper.get_lorax_cmd(
compose.conf["release_name"],
compose.conf["release_version"],
self._get_release(compose, config),
repo_baseurl=source_repo,
output_dir=output_dir,
variant=variant.uid,
nomacboot=True,
volid=volid,
buildarch=get_valid_arches(arch)[0],
buildinstallpackages=config.get("installpkgs"),
add_template=self._get_templates(config, "add_template"),
add_arch_template=self._get_templates(config, "add_arch_template"),
add_template_var=config.get("add_template_var"),
add_arch_template_var=config.get("add_arch_template_var"),
rootfs_size=config.get("rootfs_size"),
is_final=compose.supported,
log_dir=self.logdir,
)
cmd = "rm -rf %s && %s" % (
shlex_quote(output_dir),
" ".join([shlex_quote(x) for x in lorax_cmd]),
)
packages = ["pungi", "lorax", "ostree"]
packages += config.get("extra_runroot_pkgs", [])
log_file = os.path.join(self.logdir, "runroot.log")
runroot = Runroot(compose, phase="ostree_installer")
runroot.run(
cmd,
log_file=log_file,
arch=arch,
packages=packages,
mounts=[compose.topdir],
chown_paths=[output_dir],
weight=compose.conf["runroot_weights"].get("ostree_installer"),
)
if compose.conf["ostree_installer_use_koji_plugin"]:
args = {
"product": compose.conf["release_name"],
"version": compose.conf["release_version"],
"release": self._get_release(compose, config),
"sources": shortcuts.force_list(source_repo),
"variant": variant.uid,
"nomacboot": True,
"volid": volid,
"buildarch": get_valid_arches(arch)[0],
"installpkgs": config.get("installpkgs"),
"add-template": self._get_templates(config, "add_template"),
"add-arch-template": self._get_templates(config, "add_arch_template"),
"add-template-var": config.get("add_template_var"),
"add-arch-template-var": config.get("add_arch_template_var"),
"rootfs-size": config.get("rootfs_size"),
"isfinal": compose.supported,
"outputdir": output_dir,
}
runroot.run_pungi_buildinstall(
args,
log_file=log_file,
arch=arch,
packages=packages,
mounts=[compose.topdir],
weight=compose.conf["runroot_weights"].get("ostree_installer"),
)
# If Koji pungi-buildinstall is used, then the buildinstall results are
# not stored directly in `output_dir` dir, but in "results" and "logs"
# subdirectories. We need to move them to final_output_dir.
results_dir = os.path.join(output_dir, "results")
move_all(results_dir, output_dir, rm_src_dir=True)
# Get the log_dir into which we should copy the resulting log files.
if not os.path.exists(self.logdir):
makedirs(self.logdir)
log_dir = os.path.join(output_dir, "logs")
move_all(log_dir, self.logdir, rm_src_dir=True)
else:
lorax_wrapper = lorax.LoraxWrapper()
lorax_cmd = lorax_wrapper.get_lorax_cmd(
compose.conf["release_name"],
compose.conf["release_version"],
self._get_release(compose, config),
repo_baseurl=source_repo,
output_dir=output_dir,
variant=variant.uid,
nomacboot=True,
volid=volid,
buildarch=get_valid_arches(arch)[0],
buildinstallpackages=config.get("installpkgs"),
add_template=self._get_templates(config, "add_template"),
add_arch_template=self._get_templates(config, "add_arch_template"),
add_template_var=config.get("add_template_var"),
add_arch_template_var=config.get("add_arch_template_var"),
rootfs_size=config.get("rootfs_size"),
is_final=compose.supported,
log_dir=self.logdir,
)
cmd = "rm -rf %s && %s" % (
shlex_quote(output_dir),
" ".join([shlex_quote(x) for x in lorax_cmd]),
)
runroot.run(
cmd,
log_file=log_file,
arch=arch,
packages=packages,
mounts=[compose.topdir],
chown_paths=[output_dir],
weight=compose.conf["runroot_weights"].get("ostree_installer"),
)

29
pungi/runroot.py
View File

@ -283,6 +283,35 @@ class Runroot(kobo.log.LoggingBase):
)
self._result = output
def run_pungi_ostree(self, args, log_file=None, arch=None, **kwargs):
"""
Runs the OStree runroot command using the Pungi OSTree
Koji plugin as pungi_ostree task.
The **kwargs are optional and matches the
`KojiWrapper.get_pungi_buildinstall_cmd()` kwargs.
:param dict args: Arguments for the pungi_ostree Koji task.
:param str log_file: Log file into which the output of the task will
be logged.
:param str arch: Architecture on which the task should be executed.
"""
runroot_channel = self.compose.conf.get("runroot_channel")
runroot_tag = self.compose.conf["runroot_tag"]
koji_wrapper = kojiwrapper.KojiWrapper(self.compose.conf["koji_profile"])
koji_cmd = koji_wrapper.get_pungi_ostree_cmd(
runroot_tag, arch, args, channel=runroot_channel, **kwargs
)
output = koji_wrapper.run_runroot_cmd(koji_cmd, log_file=log_file)
if output["retcode"] != 0:
raise RuntimeError(
"Pungi-buildinstall task failed: %s. See %s for more details."
% (output["task_id"], log_file)
)
self._result = output
def get_buildroot_rpms(self):
"""
Returns the list of RPMs installed in a buildroot in which the runroot

38
pungi/wrappers/kojiwrapper.py
View File

@ -201,6 +201,44 @@ class KojiWrapper(object):
return cmd
def get_pungi_ostree_cmd(
self, target, arch, args, channel=None, packages=None, mounts=None, weight=None,
):
cmd = self._get_cmd("pungi-ostree", "--nowait", "--task-id")
if channel:
cmd.append("--channel-override=%s" % channel)
else:
cmd.append("--channel-override=runroot-local")
if weight:
cmd.append("--weight=%s" % int(weight))
for package in packages or []:
cmd.append("--package=%s" % package)
for mount in mounts or []:
# directories are *not* created here
cmd.append("--mount=%s" % mount)
# IMPORTANT: all --opts have to be provided *before* args
cmd.append(target)
# i686 -> i386 etc.
arch = getBaseArch(arch)
cmd.append(arch)
for k, v in args.items():
if v:
if isinstance(v, bool):
cmd.append(k)
else:
for arg in force_list(v):
cmd.append("%s=%s" % (k, shlex_quote(arg)))
return cmd
@contextlib.contextmanager
def get_koji_cmd_env(self):
"""Get environment variables for running a koji command.

105
tests/test_ostree_installer_phase.py
View File

@ -266,6 +266,111 @@ class OstreeThreadTest(helpers.PungiTestCase):
self.assertImageAdded(self.compose, ImageCls, iso)
self.assertAllCopied(copy_all)
@mock.patch("pungi.util.copy_all")
@mock.patch("productmd.images.Image")
@mock.patch("pungi.util.get_mtime")
@mock.patch("pungi.util.get_file_size")
@mock.patch("pungi.phases.ostree_installer.iso")
@mock.patch("os.link")
@mock.patch("pungi.wrappers.kojiwrapper.KojiWrapper")
@mock.patch("pungi.phases.ostree_installer.move_all")
def test_run_koji_plugin(
self,
move_all,
KojiWrapper,
link,
iso,
get_file_size,
get_mtime,
ImageCls,
copy_all,
):
self.compose.supported = False
self.compose.conf["ostree_installer_use_koji_plugin"] = True
pool = mock.Mock()
cfg = {
"repo": "Everything", # this variant-type repo is deprecated, in result will be replaced with default repo # noqa: E501
"release": "20160321.n.0",
}
koji = KojiWrapper.return_value
koji.run_runroot_cmd.return_value = {
"task_id": 1234,
"retcode": 0,
"output": "Foo bar\n",
}
get_file_size.return_value = 1024
get_mtime.return_value = 13579
final_iso_path = self.topdir + "/compose/Everything/x86_64/iso/image-name"
t = ostree.OstreeInstallerThread(pool, ["http://example.com/repo/1"])
t.process((self.compose, self.compose.variants["Everything"], "x86_64", cfg), 1)
args = {
"product": "Fedora",
"version": "Rawhide",
"release": "20160321.n.0",
"sources": [
"http://example.com/repo/1",
"http://example.com/work/$basearch/comps_repo_Everything",
],
"variant": "Everything",
"nomacboot": True,
"volid": "test-Everything-x86_64",
"buildarch": "x86_64",
"installpkgs": None,
"add-template": [],
"add-arch-template": [],
"add-template-var": None,
"add-arch-template-var": None,
"rootfs-size": None,
"isfinal": False,
"outputdir": self.topdir + "/work/x86_64/Everything/ostree_installer",
}
self.assertEqual(
koji.get_pungi_buildinstall_cmd.mock_calls,
[
mock.call(
"rrt",
"x86_64",
args,
channel=None,
packages=["pungi", "lorax", "ostree"],
mounts=[self.topdir],
weight=None,
chown_uid=os.getuid(),
)
],
)
self.assertEqual(
koji.run_runroot_cmd.mock_calls,
[
mock.call(
koji.get_pungi_buildinstall_cmd.return_value,
log_file=os.path.join(self.topdir, LOG_PATH, "runroot.log"),
)
],
)
self.assertEqual(
move_all.mock_calls,
[
mock.call(
self.topdir + "/work/x86_64/Everything/ostree_installer/results",
self.topdir + "/work/x86_64/Everything/ostree_installer",
rm_src_dir=True,
),
mock.call(
self.topdir + "/work/x86_64/Everything/ostree_installer/logs",
os.path.join(self.topdir, LOG_PATH),
rm_src_dir=True,
),
],
)
self.assertIsoLinked(link, get_file_size, get_mtime, final_iso_path)
self.assertImageAdded(self.compose, ImageCls, iso)
self.assertAllCopied(copy_all)
@mock.patch("pungi.util.copy_all")
@mock.patch("productmd.images.Image")
@mock.patch("pungi.util.get_mtime")

72
tests/test_ostree_phase.py
View File

@ -277,6 +277,78 @@ class OSTreeThreadTest(helpers.PungiTestCase):
)
self.assertTrue(os.path.isdir(self.repo))
@mock.patch("pungi.wrappers.scm.get_dir_from_scm")
@mock.patch("pungi.wrappers.kojiwrapper.KojiWrapper")
def test_run_use_koji_plugin(self, KojiWrapper, get_dir_from_scm):
get_dir_from_scm.side_effect = self._dummy_config_repo
self.compose.conf["runroot_weights"] = {"ostree": 123}
self.compose.conf["ostree_use_koji_plugin"] = True
koji = KojiWrapper.return_value
koji.run_runroot_cmd.side_effect = self._mock_runroot(0)
t = ostree.OSTreeThread(self.pool, ["http://example.com/repo/1"])
t.process(
(self.compose, self.compose.variants["Everything"], "x86_64", self.cfg), 1
)
self.assertEqual(
get_dir_from_scm.call_args_list,
[
mock.call(
{
"scm": "git",
"repo": "https://git.fedorahosted.org/git/fedora-atomic.git",
"branch": "f24",
"dir": ".",
},
self.topdir + "/work/ostree-1/config_repo",
compose=self.compose,
)
],
)
self.assertEqual(
koji.get_pungi_ostree_cmd.call_args_list,
[
mock.call(
"rrt",
"x86_64",
{
"repo": self.repo,
"log-dir": "%s/logs/x86_64/Everything/ostree-1" % self.topdir,
"treefile": "%s/fedora-atomic-docker-host.json"
% (self.topdir + "/work/ostree-1/config_repo"),
"extra-config": "%s/extra_config.json"
% (self.topdir + "/work/ostree-1"),
"update-summary": False,
"ostree-ref": None,
"force-new-commit": False,
"version": None,
},
channel=None,
mounts=[self.topdir, self.repo],
packages=["pungi", "ostree", "rpm-ostree"],
weight=123,
)
],
)
self.assertEqual(
koji.run_runroot_cmd.call_args_list,
[
mock.call(
koji.get_pungi_ostree_cmd.return_value,
log_file=self.topdir
+ "/logs/x86_64/Everything/ostree-1/runroot.log",
)
],
)
self.assertTrue(
os.path.isfile(os.path.join(self.topdir, "work/ostree-1/extra_config.json"))
)
self.assertTrue(os.path.isdir(self.repo))
@mock.patch("pungi.wrappers.scm.get_dir_from_scm")
@mock.patch("pungi.wrappers.kojiwrapper.KojiWrapper")
def test_run_fail(self, KojiWrapper, get_dir_from_scm):