Add ostree native container support
Add a new `ostree_container` stage to create ostree native container images as OCI archives, using rpm-ostree compose image. See: https://fedoraproject.org/wiki/Changes/OstreeNativeContainerStable See: https://gitlab.com/CentOS/cloud/issue-tracker/-/issues/1 Fixes: https://pagure.io/pungi/issue/1698 Merges: https://pagure.io/pungi/pull-request/1699 Signed-off-by: Timothée Ravier <tim@siosm.fr>
This commit is contained in:
Timothée Ravier
Lubomír Sedlář
parent
aa7fcc1c20
commit
95497d2676
@ -1839,6 +1839,88 @@ Example config
|
||||
has the pungi_ostree plugin installed.
|
||||
|
||||
|
||||
OSTree Native Container Settings
|
||||
================================
|
||||
|
||||
The ``ostree_container`` phase of *Pungi* can create an ostree native container
|
||||
image as an OCI archive. This is done by running ``rpm-ostree compose image``
|
||||
in a Koji runroot environment.
|
||||
|
||||
While rpm-ostree can use information from previously built images to improve
|
||||
the split in container layers, we can not use that functionnality until
|
||||
https://github.com/containers/skopeo/pull/2114 is resolved. Each invocation
|
||||
will thus create a new OCI archive image *from scratch*.
|
||||
|
||||
**ostree_container**
|
||||
(*dict*) -- a mapping of configuration for each. The format should be
|
||||
``{variant_uid_regex: config_dict}``. It is possible to use a list of
|
||||
configuration dicts as well.
|
||||
|
||||
The configuration dict for each variant arch pair must have these keys:
|
||||
|
||||
* ``treefile`` -- (*str*) Filename of configuration for ``rpm-ostree``.
|
||||
* ``config_url`` -- (*str*) URL for Git repository with the ``treefile``.
|
||||
* ``repo`` -- (*str|dict|[str|dict]*) repos specified by URL or variant UID
|
||||
or a dict of repo options, ``baseurl`` is required in the dict.
|
||||
* ``ociarchive_path`` -- (*str*) Where to put the OCI archive.
|
||||
* ``ociarchive_name`` -- (*str*) Base name to use for the ociarchive file.
|
||||
Final name will be ``{name}-{version}.ociarchive`` (ommitting the version
|
||||
if it is not set).
|
||||
|
||||
These keys are optional:
|
||||
|
||||
* ``keep_original_sources`` -- (*bool*) Keep the existing source repos in
|
||||
the tree config file. If not enabled, all the original source repos will
|
||||
be removed from the tree config file.
|
||||
* ``config_branch`` -- (*str*) Git branch of the repo to use. Defaults to
|
||||
``main``.
|
||||
* ``arches`` -- (*[str]*) List of architectures for which to generate
|
||||
ostree native container images. There will be one task per architecture.
|
||||
By default all architectures in the variant are used.
|
||||
* ``failable`` -- (*[str]*) List of architectures for which this
|
||||
deliverable is not release blocking.
|
||||
* ``version`` -- (*str*) Version string to be added to the OCI archive name.
|
||||
If this option is set to ``!OSTREE_VERSION_FROM_LABEL_DATE_TYPE_RESPIN``,
|
||||
a value will be generated automatically as ``$VERSION.$RELEASE``.
|
||||
If this option is set to ``!VERSION_FROM_VERSION_DATE_RESPIN``,
|
||||
a value will be generated automatically as ``$VERSION.$DATE.$RESPIN``.
|
||||
:ref:`See how those values are created <auto-version>`.
|
||||
* ``tag_ref`` -- (*bool*, default ``True``) If set to ``False``, a git
|
||||
reference will not be created.
|
||||
* ``runroot_packages`` -- (*list*) A list of additional package names to be
|
||||
installed in the runroot environment in Koji.
|
||||
|
||||
Example config
|
||||
--------------
|
||||
::
|
||||
|
||||
ostree_container = {
|
||||
"^Sagano$": {
|
||||
"treefile": "fedora-tier-0-38.yaml",
|
||||
"config_url": "https://gitlab.com/CentOS/cloud/sagano.git",
|
||||
"config_branch": "main",
|
||||
"repo": [
|
||||
"Server",
|
||||
"http://example.com/repo/x86_64/os",
|
||||
{"baseurl": "Everything"},
|
||||
{"baseurl": "http://example.com/linux/repo", "exclude": "systemd-container"},
|
||||
],
|
||||
"ociarchive_path": "/mnt/koji/compose/ostree_container/",
|
||||
# Base name to use for the ociarchive file. Final name will be {name}-{version}.ociarchive
|
||||
"ociarchive_name": "sagano",
|
||||
# Automatically generate a reasonable version
|
||||
"version": "!OSTREE_VERSION_FROM_LABEL_DATE_TYPE_RESPIN",
|
||||
# Only run this for x86_64 even if Sagano has more arches
|
||||
"arches": ["x86_64"],
|
||||
}
|
||||
}
|
||||
|
||||
**ostree_container_use_koji_plugin** = False
|
||||
(*bool*) -- When set to ``True``, the Koji pungi_ostree task will be
|
||||
used to execute rpm-ostree instead of runroot. Use only if the Koji instance
|
||||
has the pungi_ostree plugin installed.
|
||||
|
||||
|
||||
Ostree Installer Settings
|
||||
=========================
|
||||
|
||||
|
||||
@ -343,6 +343,27 @@ This is a shortened configuration for Fedora Radhide compose as of 2019-10-14.
|
||||
}
|
||||
}
|
||||
|
||||
ostree_container = {
|
||||
"^Sagano$": {
|
||||
"treefile": "fedora-tier-0-38.yaml",
|
||||
"config_url": "https://gitlab.com/CentOS/cloud/sagano.git",
|
||||
"config_branch": "main",
|
||||
"repo": [
|
||||
"Server",
|
||||
"http://example.com/repo/x86_64/os",
|
||||
{"baseurl": "Everything"},
|
||||
{"baseurl": "http://example.com/linux/repo", "exclude": "systemd-container"},
|
||||
],
|
||||
"ociarchive_path": "/mnt/koji/compose/ostree_container/",
|
||||
# Base name to use for the ociarchive file. Final name will be {name}-{version}.ociarchive
|
||||
"ociarchive_name": "sagano",
|
||||
# Automatically generate a reasonable version
|
||||
"version": "!OSTREE_VERSION_FROM_LABEL_DATE_TYPE_RESPIN",
|
||||
# Only run this for x86_64 even if Sagano has more arches
|
||||
"arches": ["x86_64"],
|
||||
}
|
||||
}
|
||||
|
||||
ostree_installer = [
|
||||
("^Silverblue$", {
|
||||
"x86_64": {
|
||||
|
||||
@ -1104,6 +1104,44 @@ def make_schema():
|
||||
),
|
||||
]
|
||||
},
|
||||
"ostree_container": {
|
||||
"type": "object",
|
||||
"patternProperties": {
|
||||
# Warning: this pattern is a variant uid regex, but the
|
||||
# format does not let us validate it as there is no regular
|
||||
# expression to describe all regular expressions.
|
||||
".+": _one_or_list(
|
||||
{
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"treefile": {"type": "string"},
|
||||
"config_url": {"type": "string"},
|
||||
"ociarchive_path": {"type": "string"},
|
||||
"ociarchive_name": {"type": "string"},
|
||||
"repo": {"$ref": "#/definitions/repos"},
|
||||
"keep_original_sources": {"type": "boolean"},
|
||||
"config_branch": {"type": "string"},
|
||||
"arches": {"$ref": "#/definitions/list_of_strings"},
|
||||
"failable": {"$ref": "#/definitions/list_of_strings"},
|
||||
"version": {"type": "string"},
|
||||
"tag_ref": {"type": "boolean"},
|
||||
"runroot_packages": {
|
||||
"$ref": "#/definitions/list_of_strings",
|
||||
},
|
||||
},
|
||||
"required": [
|
||||
"treefile",
|
||||
"config_url",
|
||||
"repo",
|
||||
"ociarchive_path",
|
||||
"ociarchive_name",
|
||||
],
|
||||
"additionalProperties": False,
|
||||
}
|
||||
),
|
||||
},
|
||||
"additionalProperties": False,
|
||||
},
|
||||
"ostree_installer": _variant_arch_mapping(
|
||||
{
|
||||
"type": "object",
|
||||
@ -1128,6 +1166,7 @@ def make_schema():
|
||||
}
|
||||
),
|
||||
"ostree_use_koji_plugin": {"type": "boolean", "default": False},
|
||||
"ostree_container_use_koji_plugin": {"type": "boolean", "default": False},
|
||||
"ostree_installer_use_koji_plugin": {"type": "boolean", "default": False},
|
||||
"ostree_installer_overwrite": {"type": "boolean", "default": False},
|
||||
"live_images": _variant_arch_mapping(
|
||||
|
||||
@ -19,6 +19,7 @@ import logging
|
||||
|
||||
from .tree import Tree
|
||||
from .installer import Installer
|
||||
from .container import Container
|
||||
|
||||
|
||||
def main(args=None):
|
||||
@ -71,6 +72,42 @@ def main(args=None):
|
||||
help="use unified core mode in rpm-ostree",
|
||||
)
|
||||
|
||||
container = subparser.add_parser(
|
||||
"container", help="Compose OSTree native container"
|
||||
)
|
||||
container.set_defaults(_class=Container, func="run")
|
||||
container.add_argument(
|
||||
"--ociarchive-path",
|
||||
metavar="DIR",
|
||||
required=True,
|
||||
help="where to output the OCI archive (required)",
|
||||
)
|
||||
container.add_argument(
|
||||
"--ociarchive-name",
|
||||
required=True,
|
||||
help="the name of the the OCI archive (required)",
|
||||
)
|
||||
container.add_argument(
|
||||
"--treefile",
|
||||
metavar="FILE",
|
||||
required=True,
|
||||
help="treefile for rpm-ostree (required)",
|
||||
)
|
||||
container.add_argument(
|
||||
"--log-dir",
|
||||
metavar="DIR",
|
||||
required=True,
|
||||
help="where to log output (required).",
|
||||
)
|
||||
container.add_argument(
|
||||
"--extra-config", metavar="FILE", help="JSON file contains extra configurations"
|
||||
)
|
||||
container.add_argument(
|
||||
"--version",
|
||||
metavar="VERSION",
|
||||
help="version string to be used for OCI archive name",
|
||||
)
|
||||
|
||||
installerp = subparser.add_parser(
|
||||
"installer", help="Create an OSTree installer image"
|
||||
)
|
||||
|
||||
98
pungi/ostree/container.py
Normal file
98
pungi/ostree/container.py
Normal file
@ -0,0 +1,98 @@
|
||||
# -*- coding: utf-8 -*-
|
||||
|
||||
|
||||
# This program is free software; you can redistribute it and/or modify
|
||||
# it under the terms of the GNU General Public License as published by
|
||||
# the Free Software Foundation; version 2 of the License.
|
||||
#
|
||||
# This program is distributed in the hope that it will be useful,
|
||||
# but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
# GNU Library General Public License for more details.
|
||||
#
|
||||
# You should have received a copy of the GNU General Public License
|
||||
# along with this program; if not, see <https://gnu.org/licenses/>.
|
||||
|
||||
|
||||
import os
|
||||
import json
|
||||
from kobo import shortcuts
|
||||
|
||||
from .base import OSTree
|
||||
from .utils import (
|
||||
make_log_file,
|
||||
tweak_treeconf,
|
||||
)
|
||||
|
||||
|
||||
class Container(OSTree):
|
||||
def _make_container(self):
|
||||
"""Compose OSTree Container Native image"""
|
||||
log_file = make_log_file(self.logdir, "create-ostree-repo")
|
||||
stamp_file = os.path.join(self.logdir, "%s.stamp" % self.ociarchive_name)
|
||||
cmd = [
|
||||
"rpm-ostree",
|
||||
"compose",
|
||||
"image",
|
||||
# Always initialize for now
|
||||
"--initialize",
|
||||
# Touch the file if a new commit was created. This can help us tell
|
||||
# if the commitid file is missing because no commit was created or
|
||||
# because something went wrong.
|
||||
"--touch-if-changed=%s" % stamp_file,
|
||||
self.treefile,
|
||||
]
|
||||
if self.version is None:
|
||||
fullpath = os.path.join(
|
||||
self.ociarchive_path, "%s.ociarchive" % self.ociarchive_name
|
||||
)
|
||||
else:
|
||||
fullpath = os.path.join(
|
||||
self.ociarchive_path,
|
||||
"%s-%s.ociarchive" % (self.ociarchive_name, self.version),
|
||||
)
|
||||
cmd.append(fullpath)
|
||||
|
||||
# Set the umask to be more permissive so directories get group write
|
||||
# permissions. See https://pagure.io/releng/issue/8811#comment-629051
|
||||
oldumask = os.umask(0o0002)
|
||||
try:
|
||||
shortcuts.run(
|
||||
cmd,
|
||||
show_cmd=True,
|
||||
stdout=True,
|
||||
logfile=log_file,
|
||||
universal_newlines=True,
|
||||
)
|
||||
finally:
|
||||
os.umask(oldumask)
|
||||
|
||||
def run(self):
|
||||
self.treefile = self.args.treefile
|
||||
self.version = self.args.version
|
||||
self.logdir = self.args.log_dir
|
||||
self.extra_config = self.args.extra_config
|
||||
self.ociarchive_path = self.args.ociarchive_path
|
||||
self.ociarchive_name = self.args.ociarchive_name
|
||||
|
||||
if self.extra_config:
|
||||
self.extra_config = json.load(open(self.extra_config, "r"))
|
||||
repos = self.extra_config.get("repo", [])
|
||||
keep_original_sources = self.extra_config.get(
|
||||
"keep_original_sources", False
|
||||
)
|
||||
else:
|
||||
# missing extra_config mustn't affect tweak_treeconf call
|
||||
repos = []
|
||||
keep_original_sources = True
|
||||
|
||||
update_dict = {}
|
||||
|
||||
self.treefile = tweak_treeconf(
|
||||
self.treefile,
|
||||
source_repos=repos,
|
||||
keep_original_sources=keep_original_sources,
|
||||
update_dict=update_dict,
|
||||
)
|
||||
|
||||
self._make_container()
|
||||
197
pungi/phases/ostree_container.py
Normal file
197
pungi/phases/ostree_container.py
Normal file
@ -0,0 +1,197 @@
|
||||
# -*- coding: utf-8 -*-
|
||||
|
||||
import copy
|
||||
import json
|
||||
import os
|
||||
from kobo import shortcuts
|
||||
from kobo.threads import ThreadPool, WorkerThread
|
||||
from collections import OrderedDict
|
||||
|
||||
from pungi.runroot import Runroot
|
||||
from .base import ConfigGuardedPhase
|
||||
from .. import util
|
||||
from ..util import get_repo_dicts, translate_path
|
||||
from ..wrappers import scm
|
||||
|
||||
|
||||
class OSTreeContainerPhase(ConfigGuardedPhase):
|
||||
name = "ostree_container"
|
||||
|
||||
def __init__(self, compose, pkgset_phase=None):
|
||||
super(OSTreeContainerPhase, self).__init__(compose)
|
||||
self.pool = ThreadPool(logger=self.compose._logger)
|
||||
self.pkgset_phase = pkgset_phase
|
||||
|
||||
def get_repos(self):
|
||||
return [
|
||||
translate_path(
|
||||
self.compose,
|
||||
self.compose.paths.work.pkgset_repo(pkgset.name, "$basearch"),
|
||||
)
|
||||
for pkgset in self.pkgset_phase.package_sets
|
||||
]
|
||||
|
||||
def _enqueue(self, variant, arch, conf):
|
||||
self.pool.add(OSTreeContainerThread(self.pool, self.get_repos()))
|
||||
self.pool.queue_put((self.compose, variant, arch, conf))
|
||||
|
||||
def run(self):
|
||||
if isinstance(self.compose.conf.get(self.name), dict):
|
||||
for variant in self.compose.get_variants():
|
||||
for conf in self.get_config_block(variant):
|
||||
for arch in conf.get("arches", []) or variant.arches:
|
||||
self._enqueue(variant, arch, conf)
|
||||
else:
|
||||
# Legacy code path to support original configuration.
|
||||
for variant in self.compose.get_variants():
|
||||
for arch in variant.arches:
|
||||
for conf in self.get_config_block(variant, arch):
|
||||
self._enqueue(variant, arch, conf)
|
||||
|
||||
self.pool.start()
|
||||
|
||||
|
||||
class OSTreeContainerThread(WorkerThread):
|
||||
def __init__(self, pool, repos):
|
||||
super(OSTreeContainerThread, self).__init__(pool)
|
||||
self.repos = repos
|
||||
|
||||
def process(self, item, num):
|
||||
compose, variant, arch, config = item
|
||||
self.num = num
|
||||
failable_arches = config.get("failable", [])
|
||||
with util.failable(
|
||||
compose,
|
||||
util.can_arch_fail(failable_arches, arch),
|
||||
variant,
|
||||
arch,
|
||||
"ostree-container",
|
||||
):
|
||||
self.worker(compose, variant, arch, config)
|
||||
|
||||
def worker(self, compose, variant, arch, config):
|
||||
msg = "OSTree phase for variant %s, arch %s" % (variant.uid, arch)
|
||||
self.pool.log_info("[BEGIN] %s" % msg)
|
||||
workdir = compose.paths.work.topdir("ostree-%d" % self.num)
|
||||
self.logdir = compose.paths.log.topdir(
|
||||
"%s/%s/ostree-container-%d" % (arch, variant.uid, self.num)
|
||||
)
|
||||
repodir = os.path.join(workdir, "config_repo")
|
||||
self._clone_repo(
|
||||
compose,
|
||||
repodir,
|
||||
config["config_url"],
|
||||
config.get("config_branch", "main"),
|
||||
)
|
||||
|
||||
repos = shortcuts.force_list(config["repo"]) + self.repos
|
||||
repos = get_repo_dicts(repos, logger=self.pool)
|
||||
|
||||
# copy the original config and update before save to a json file
|
||||
new_config = copy.copy(config)
|
||||
|
||||
# repos in configuration can have repo url set to variant UID,
|
||||
# update it to have the actual url that we just translated.
|
||||
new_config.update({"repo": repos})
|
||||
|
||||
# remove unnecessary (for 'pungi-make-ostree container' script ) elements
|
||||
# from config, it doesn't hurt to have them, however remove them can
|
||||
# reduce confusion
|
||||
for k in [
|
||||
"treefile",
|
||||
"config_url",
|
||||
"config_branch",
|
||||
"failable",
|
||||
"version",
|
||||
]:
|
||||
new_config.pop(k, None)
|
||||
|
||||
# write a json file to save the configuration, so 'pungi-make-ostree tree'
|
||||
# can take use of it
|
||||
extra_config_file = os.path.join(workdir, "extra_config.json")
|
||||
with open(extra_config_file, "w") as f:
|
||||
json.dump(new_config, f, indent=4)
|
||||
|
||||
self._run_ostree_container_cmd(
|
||||
compose, variant, arch, config, repodir, extra_config_file=extra_config_file
|
||||
)
|
||||
|
||||
if compose.notifier:
|
||||
# 'pungi-make-ostree container' writes to {ociarchive_name}.stamp in
|
||||
# logdir if the compose succeeded. If the compose failed, an exception
|
||||
# will be raised.
|
||||
os.path.exists(
|
||||
os.path.join(self.logdir, "%s.stamp" % config["ociarchive_name"])
|
||||
)
|
||||
if config["version"] is None:
|
||||
filename = "%s.ociarchive" % config["ociarchive_name"]
|
||||
else:
|
||||
filename = (
|
||||
"%s-%s.ociarchive" % (config["ociarchive_name"], config["version"]),
|
||||
)
|
||||
compose.notifier.send(
|
||||
"ostree_container",
|
||||
variant=variant.uid,
|
||||
arch=arch,
|
||||
filename=filename,
|
||||
version=config["version"],
|
||||
path=translate_path(compose, config["ociarchive_path"]),
|
||||
local_path=config["ociarchive_path"],
|
||||
)
|
||||
|
||||
self.pool.log_info("[DONE ] %s" % (msg))
|
||||
|
||||
def _run_ostree_container_cmd(
|
||||
self, compose, variant, arch, config, config_repo, extra_config_file=None
|
||||
):
|
||||
args = OrderedDict(
|
||||
[
|
||||
("log-dir", self.logdir),
|
||||
("treefile", os.path.join(config_repo, config["treefile"])),
|
||||
("version", util.version_generator(compose, config.get("version"))),
|
||||
("extra-config", extra_config_file),
|
||||
("ociarchive-path", config.get("ociarchive_path")),
|
||||
("ociarchive-name", config.get("ociarchive_name")),
|
||||
]
|
||||
)
|
||||
default_packages = ["pungi", "ostree", "rpm-ostree", "selinux-policy-targeted"]
|
||||
additional_packages = config.get("runroot_packages", [])
|
||||
packages = default_packages + additional_packages
|
||||
log_file = os.path.join(self.logdir, "runroot.log")
|
||||
# TODO: Use to get previous build
|
||||
mounts = [compose.topdir, config["ostree_repo"]]
|
||||
runroot = Runroot(compose, phase="ostree_container")
|
||||
|
||||
if compose.conf["ostree_container_use_koji_plugin"]:
|
||||
runroot.run_pungi_ostree(
|
||||
dict(args),
|
||||
log_file=log_file,
|
||||
arch=arch,
|
||||
packages=packages,
|
||||
mounts=mounts,
|
||||
weight=compose.conf["runroot_weights"].get("ostree"),
|
||||
)
|
||||
else:
|
||||
cmd = ["pungi-make-ostree", "container"]
|
||||
for key, value in args.items():
|
||||
if value is True:
|
||||
cmd.append("--%s" % key)
|
||||
elif value:
|
||||
cmd.append("--%s=%s" % (key, value))
|
||||
|
||||
runroot.run(
|
||||
cmd,
|
||||
log_file=log_file,
|
||||
arch=arch,
|
||||
packages=packages,
|
||||
mounts=mounts,
|
||||
new_chroot=True,
|
||||
weight=compose.conf["runroot_weights"].get("ostree"),
|
||||
)
|
||||
|
||||
def _clone_repo(self, compose, repodir, url, branch):
|
||||
scm.get_dir_from_scm(
|
||||
{"scm": "git", "repo": url, "branch": branch, "dir": "."},
|
||||
repodir,
|
||||
compose=compose,
|
||||
)
|
||||
Loading…
Reference in New Issue
Block a user