diff --git a/doc/configuration.rst b/doc/configuration.rst index 8caca1ff..a5b927f0 100644 --- a/doc/configuration.rst +++ b/doc/configuration.rst @@ -1839,6 +1839,88 @@ Example config has the pungi_ostree plugin installed. +OSTree Native Container Settings +================================ + +The ``ostree_container`` phase of *Pungi* can create an ostree native container +image as an OCI archive. This is done by running ``rpm-ostree compose image`` +in a Koji runroot environment. + +While rpm-ostree can use information from previously built images to improve +the split in container layers, we can not use that functionnality until +https://github.com/containers/skopeo/pull/2114 is resolved. Each invocation +will thus create a new OCI archive image *from scratch*. + +**ostree_container** + (*dict*) -- a mapping of configuration for each. The format should be + ``{variant_uid_regex: config_dict}``. It is possible to use a list of + configuration dicts as well. + + The configuration dict for each variant arch pair must have these keys: + + * ``treefile`` -- (*str*) Filename of configuration for ``rpm-ostree``. + * ``config_url`` -- (*str*) URL for Git repository with the ``treefile``. + * ``repo`` -- (*str|dict|[str|dict]*) repos specified by URL or variant UID + or a dict of repo options, ``baseurl`` is required in the dict. + * ``ociarchive_path`` -- (*str*) Where to put the OCI archive. + * ``ociarchive_name`` -- (*str*) Base name to use for the ociarchive file. + Final name will be ``{name}-{version}.ociarchive`` (ommitting the version + if it is not set). + + These keys are optional: + + * ``keep_original_sources`` -- (*bool*) Keep the existing source repos in + the tree config file. If not enabled, all the original source repos will + be removed from the tree config file. + * ``config_branch`` -- (*str*) Git branch of the repo to use. Defaults to + ``main``. + * ``arches`` -- (*[str]*) List of architectures for which to generate + ostree native container images. There will be one task per architecture. + By default all architectures in the variant are used. + * ``failable`` -- (*[str]*) List of architectures for which this + deliverable is not release blocking. + * ``version`` -- (*str*) Version string to be added to the OCI archive name. + If this option is set to ``!OSTREE_VERSION_FROM_LABEL_DATE_TYPE_RESPIN``, + a value will be generated automatically as ``$VERSION.$RELEASE``. + If this option is set to ``!VERSION_FROM_VERSION_DATE_RESPIN``, + a value will be generated automatically as ``$VERSION.$DATE.$RESPIN``. + :ref:`See how those values are created `. + * ``tag_ref`` -- (*bool*, default ``True``) If set to ``False``, a git + reference will not be created. + * ``runroot_packages`` -- (*list*) A list of additional package names to be + installed in the runroot environment in Koji. + +Example config +-------------- +:: + + ostree_container = { + "^Sagano$": { + "treefile": "fedora-tier-0-38.yaml", + "config_url": "https://gitlab.com/CentOS/cloud/sagano.git", + "config_branch": "main", + "repo": [ + "Server", + "http://example.com/repo/x86_64/os", + {"baseurl": "Everything"}, + {"baseurl": "http://example.com/linux/repo", "exclude": "systemd-container"}, + ], + "ociarchive_path": "/mnt/koji/compose/ostree_container/", + # Base name to use for the ociarchive file. Final name will be {name}-{version}.ociarchive + "ociarchive_name": "sagano", + # Automatically generate a reasonable version + "version": "!OSTREE_VERSION_FROM_LABEL_DATE_TYPE_RESPIN", + # Only run this for x86_64 even if Sagano has more arches + "arches": ["x86_64"], + } + } + +**ostree_container_use_koji_plugin** = False + (*bool*) -- When set to ``True``, the Koji pungi_ostree task will be + used to execute rpm-ostree instead of runroot. Use only if the Koji instance + has the pungi_ostree plugin installed. + + Ostree Installer Settings ========================= diff --git a/doc/examples.rst b/doc/examples.rst index aef18e34..4ee85a3e 100644 --- a/doc/examples.rst +++ b/doc/examples.rst @@ -343,6 +343,27 @@ This is a shortened configuration for Fedora Radhide compose as of 2019-10-14. } } + ostree_container = { + "^Sagano$": { + "treefile": "fedora-tier-0-38.yaml", + "config_url": "https://gitlab.com/CentOS/cloud/sagano.git", + "config_branch": "main", + "repo": [ + "Server", + "http://example.com/repo/x86_64/os", + {"baseurl": "Everything"}, + {"baseurl": "http://example.com/linux/repo", "exclude": "systemd-container"}, + ], + "ociarchive_path": "/mnt/koji/compose/ostree_container/", + # Base name to use for the ociarchive file. Final name will be {name}-{version}.ociarchive + "ociarchive_name": "sagano", + # Automatically generate a reasonable version + "version": "!OSTREE_VERSION_FROM_LABEL_DATE_TYPE_RESPIN", + # Only run this for x86_64 even if Sagano has more arches + "arches": ["x86_64"], + } + } + ostree_installer = [ ("^Silverblue$", { "x86_64": { diff --git a/pungi/checks.py b/pungi/checks.py index 093046bd..d1c4c8dd 100644 --- a/pungi/checks.py +++ b/pungi/checks.py @@ -1104,6 +1104,44 @@ def make_schema(): ), ] }, + "ostree_container": { + "type": "object", + "patternProperties": { + # Warning: this pattern is a variant uid regex, but the + # format does not let us validate it as there is no regular + # expression to describe all regular expressions. + ".+": _one_or_list( + { + "type": "object", + "properties": { + "treefile": {"type": "string"}, + "config_url": {"type": "string"}, + "ociarchive_path": {"type": "string"}, + "ociarchive_name": {"type": "string"}, + "repo": {"$ref": "#/definitions/repos"}, + "keep_original_sources": {"type": "boolean"}, + "config_branch": {"type": "string"}, + "arches": {"$ref": "#/definitions/list_of_strings"}, + "failable": {"$ref": "#/definitions/list_of_strings"}, + "version": {"type": "string"}, + "tag_ref": {"type": "boolean"}, + "runroot_packages": { + "$ref": "#/definitions/list_of_strings", + }, + }, + "required": [ + "treefile", + "config_url", + "repo", + "ociarchive_path", + "ociarchive_name", + ], + "additionalProperties": False, + } + ), + }, + "additionalProperties": False, + }, "ostree_installer": _variant_arch_mapping( { "type": "object", @@ -1128,6 +1166,7 @@ def make_schema(): } ), "ostree_use_koji_plugin": {"type": "boolean", "default": False}, + "ostree_container_use_koji_plugin": {"type": "boolean", "default": False}, "ostree_installer_use_koji_plugin": {"type": "boolean", "default": False}, "ostree_installer_overwrite": {"type": "boolean", "default": False}, "live_images": _variant_arch_mapping( diff --git a/pungi/ostree/__init__.py b/pungi/ostree/__init__.py index 49162692..16263148 100644 --- a/pungi/ostree/__init__.py +++ b/pungi/ostree/__init__.py @@ -19,6 +19,7 @@ import logging from .tree import Tree from .installer import Installer +from .container import Container def main(args=None): @@ -71,6 +72,42 @@ def main(args=None): help="use unified core mode in rpm-ostree", ) + container = subparser.add_parser( + "container", help="Compose OSTree native container" + ) + container.set_defaults(_class=Container, func="run") + container.add_argument( + "--ociarchive-path", + metavar="DIR", + required=True, + help="where to output the OCI archive (required)", + ) + container.add_argument( + "--ociarchive-name", + required=True, + help="the name of the the OCI archive (required)", + ) + container.add_argument( + "--treefile", + metavar="FILE", + required=True, + help="treefile for rpm-ostree (required)", + ) + container.add_argument( + "--log-dir", + metavar="DIR", + required=True, + help="where to log output (required).", + ) + container.add_argument( + "--extra-config", metavar="FILE", help="JSON file contains extra configurations" + ) + container.add_argument( + "--version", + metavar="VERSION", + help="version string to be used for OCI archive name", + ) + installerp = subparser.add_parser( "installer", help="Create an OSTree installer image" ) diff --git a/pungi/ostree/container.py b/pungi/ostree/container.py new file mode 100644 index 00000000..b78165cd --- /dev/null +++ b/pungi/ostree/container.py @@ -0,0 +1,98 @@ +# -*- coding: utf-8 -*- + + +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; version 2 of the License. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Library General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, see . + + +import os +import json +from kobo import shortcuts + +from .base import OSTree +from .utils import ( + make_log_file, + tweak_treeconf, +) + + +class Container(OSTree): + def _make_container(self): + """Compose OSTree Container Native image""" + log_file = make_log_file(self.logdir, "create-ostree-repo") + stamp_file = os.path.join(self.logdir, "%s.stamp" % self.ociarchive_name) + cmd = [ + "rpm-ostree", + "compose", + "image", + # Always initialize for now + "--initialize", + # Touch the file if a new commit was created. This can help us tell + # if the commitid file is missing because no commit was created or + # because something went wrong. + "--touch-if-changed=%s" % stamp_file, + self.treefile, + ] + if self.version is None: + fullpath = os.path.join( + self.ociarchive_path, "%s.ociarchive" % self.ociarchive_name + ) + else: + fullpath = os.path.join( + self.ociarchive_path, + "%s-%s.ociarchive" % (self.ociarchive_name, self.version), + ) + cmd.append(fullpath) + + # Set the umask to be more permissive so directories get group write + # permissions. See https://pagure.io/releng/issue/8811#comment-629051 + oldumask = os.umask(0o0002) + try: + shortcuts.run( + cmd, + show_cmd=True, + stdout=True, + logfile=log_file, + universal_newlines=True, + ) + finally: + os.umask(oldumask) + + def run(self): + self.treefile = self.args.treefile + self.version = self.args.version + self.logdir = self.args.log_dir + self.extra_config = self.args.extra_config + self.ociarchive_path = self.args.ociarchive_path + self.ociarchive_name = self.args.ociarchive_name + + if self.extra_config: + self.extra_config = json.load(open(self.extra_config, "r")) + repos = self.extra_config.get("repo", []) + keep_original_sources = self.extra_config.get( + "keep_original_sources", False + ) + else: + # missing extra_config mustn't affect tweak_treeconf call + repos = [] + keep_original_sources = True + + update_dict = {} + + self.treefile = tweak_treeconf( + self.treefile, + source_repos=repos, + keep_original_sources=keep_original_sources, + update_dict=update_dict, + ) + + self._make_container() diff --git a/pungi/phases/ostree_container.py b/pungi/phases/ostree_container.py new file mode 100644 index 00000000..0216e95c --- /dev/null +++ b/pungi/phases/ostree_container.py @@ -0,0 +1,197 @@ +# -*- coding: utf-8 -*- + +import copy +import json +import os +from kobo import shortcuts +from kobo.threads import ThreadPool, WorkerThread +from collections import OrderedDict + +from pungi.runroot import Runroot +from .base import ConfigGuardedPhase +from .. import util +from ..util import get_repo_dicts, translate_path +from ..wrappers import scm + + +class OSTreeContainerPhase(ConfigGuardedPhase): + name = "ostree_container" + + def __init__(self, compose, pkgset_phase=None): + super(OSTreeContainerPhase, self).__init__(compose) + self.pool = ThreadPool(logger=self.compose._logger) + self.pkgset_phase = pkgset_phase + + def get_repos(self): + return [ + translate_path( + self.compose, + self.compose.paths.work.pkgset_repo(pkgset.name, "$basearch"), + ) + for pkgset in self.pkgset_phase.package_sets + ] + + def _enqueue(self, variant, arch, conf): + self.pool.add(OSTreeContainerThread(self.pool, self.get_repos())) + self.pool.queue_put((self.compose, variant, arch, conf)) + + def run(self): + if isinstance(self.compose.conf.get(self.name), dict): + for variant in self.compose.get_variants(): + for conf in self.get_config_block(variant): + for arch in conf.get("arches", []) or variant.arches: + self._enqueue(variant, arch, conf) + else: + # Legacy code path to support original configuration. + for variant in self.compose.get_variants(): + for arch in variant.arches: + for conf in self.get_config_block(variant, arch): + self._enqueue(variant, arch, conf) + + self.pool.start() + + +class OSTreeContainerThread(WorkerThread): + def __init__(self, pool, repos): + super(OSTreeContainerThread, self).__init__(pool) + self.repos = repos + + def process(self, item, num): + compose, variant, arch, config = item + self.num = num + failable_arches = config.get("failable", []) + with util.failable( + compose, + util.can_arch_fail(failable_arches, arch), + variant, + arch, + "ostree-container", + ): + self.worker(compose, variant, arch, config) + + def worker(self, compose, variant, arch, config): + msg = "OSTree phase for variant %s, arch %s" % (variant.uid, arch) + self.pool.log_info("[BEGIN] %s" % msg) + workdir = compose.paths.work.topdir("ostree-%d" % self.num) + self.logdir = compose.paths.log.topdir( + "%s/%s/ostree-container-%d" % (arch, variant.uid, self.num) + ) + repodir = os.path.join(workdir, "config_repo") + self._clone_repo( + compose, + repodir, + config["config_url"], + config.get("config_branch", "main"), + ) + + repos = shortcuts.force_list(config["repo"]) + self.repos + repos = get_repo_dicts(repos, logger=self.pool) + + # copy the original config and update before save to a json file + new_config = copy.copy(config) + + # repos in configuration can have repo url set to variant UID, + # update it to have the actual url that we just translated. + new_config.update({"repo": repos}) + + # remove unnecessary (for 'pungi-make-ostree container' script ) elements + # from config, it doesn't hurt to have them, however remove them can + # reduce confusion + for k in [ + "treefile", + "config_url", + "config_branch", + "failable", + "version", + ]: + new_config.pop(k, None) + + # write a json file to save the configuration, so 'pungi-make-ostree tree' + # can take use of it + extra_config_file = os.path.join(workdir, "extra_config.json") + with open(extra_config_file, "w") as f: + json.dump(new_config, f, indent=4) + + self._run_ostree_container_cmd( + compose, variant, arch, config, repodir, extra_config_file=extra_config_file + ) + + if compose.notifier: + # 'pungi-make-ostree container' writes to {ociarchive_name}.stamp in + # logdir if the compose succeeded. If the compose failed, an exception + # will be raised. + os.path.exists( + os.path.join(self.logdir, "%s.stamp" % config["ociarchive_name"]) + ) + if config["version"] is None: + filename = "%s.ociarchive" % config["ociarchive_name"] + else: + filename = ( + "%s-%s.ociarchive" % (config["ociarchive_name"], config["version"]), + ) + compose.notifier.send( + "ostree_container", + variant=variant.uid, + arch=arch, + filename=filename, + version=config["version"], + path=translate_path(compose, config["ociarchive_path"]), + local_path=config["ociarchive_path"], + ) + + self.pool.log_info("[DONE ] %s" % (msg)) + + def _run_ostree_container_cmd( + self, compose, variant, arch, config, config_repo, extra_config_file=None + ): + args = OrderedDict( + [ + ("log-dir", self.logdir), + ("treefile", os.path.join(config_repo, config["treefile"])), + ("version", util.version_generator(compose, config.get("version"))), + ("extra-config", extra_config_file), + ("ociarchive-path", config.get("ociarchive_path")), + ("ociarchive-name", config.get("ociarchive_name")), + ] + ) + default_packages = ["pungi", "ostree", "rpm-ostree", "selinux-policy-targeted"] + additional_packages = config.get("runroot_packages", []) + packages = default_packages + additional_packages + log_file = os.path.join(self.logdir, "runroot.log") + # TODO: Use to get previous build + mounts = [compose.topdir, config["ostree_repo"]] + runroot = Runroot(compose, phase="ostree_container") + + if compose.conf["ostree_container_use_koji_plugin"]: + runroot.run_pungi_ostree( + dict(args), + log_file=log_file, + arch=arch, + packages=packages, + mounts=mounts, + weight=compose.conf["runroot_weights"].get("ostree"), + ) + else: + cmd = ["pungi-make-ostree", "container"] + for key, value in args.items(): + if value is True: + cmd.append("--%s" % key) + elif value: + cmd.append("--%s=%s" % (key, value)) + + runroot.run( + cmd, + log_file=log_file, + arch=arch, + packages=packages, + mounts=mounts, + new_chroot=True, + weight=compose.conf["runroot_weights"].get("ostree"), + ) + + def _clone_repo(self, compose, repodir, url, branch): + scm.get_dir_from_scm( + {"scm": "git", "repo": url, "branch": branch, "dir": "."}, + repodir, + compose=compose, + )