Update from upstream #11

Closed
soksanichenko wants to merge 158 commits from a8_updated into a8
8 changed files with 502 additions and 11 deletions
Showing only changes of commit 40133074b3 - Show all commits

View File

@ -12,7 +12,7 @@
viewBox="0 0 610.46457 301.1662" viewBox="0 0 610.46457 301.1662"
id="svg2" id="svg2"
version="1.1" version="1.1"
inkscape:version="1.0.1 (3bc2e813f5, 2020-09-07)" inkscape:version="1.0.2 (e86c870879, 2021-01-15)"
sodipodi:docname="phases.svg" sodipodi:docname="phases.svg"
inkscape:export-filename="/home/lsedlar/repos/pungi/doc/_static/phases.png" inkscape:export-filename="/home/lsedlar/repos/pungi/doc/_static/phases.png"
inkscape:export-xdpi="90" inkscape:export-xdpi="90"
@ -24,9 +24,9 @@
borderopacity="1.0" borderopacity="1.0"
inkscape:pageopacity="1" inkscape:pageopacity="1"
inkscape:pageshadow="2" inkscape:pageshadow="2"
inkscape:zoom="2.1213203" inkscape:zoom="1.5"
inkscape:cx="276.65806" inkscape:cx="9.4746397"
inkscape:cy="189.24198" inkscape:cy="58.833855"
inkscape:document-units="px" inkscape:document-units="px"
inkscape:current-layer="layer1" inkscape:current-layer="layer1"
showgrid="false" showgrid="false"
@ -70,7 +70,7 @@
<dc:format>image/svg+xml</dc:format> <dc:format>image/svg+xml</dc:format>
<dc:type <dc:type
rdf:resource="http://purl.org/dc/dcmitype/StillImage" /> rdf:resource="http://purl.org/dc/dcmitype/StillImage" />
<dc:title></dc:title> <dc:title />
</cc:Work> </cc:Work>
</rdf:RDF> </rdf:RDF>
</metadata> </metadata>
@ -303,15 +303,15 @@
</g> </g>
<rect <rect
transform="matrix(0,1,1,0,0,0)" transform="matrix(0,1,1,0,0,0)"
style="fill:#e9b96e;fill-rule:evenodd;stroke:none;stroke-width:2.65937px;stroke-linecap:butt;stroke-linejoin:miter;stroke-opacity:1" style="fill:#e9b96e;fill-rule:evenodd;stroke:none;stroke-width:1.85901px;stroke-linecap:butt;stroke-linejoin:miter;stroke-opacity:1"
id="rect3338-1" id="rect3338-1"
width="185.96895" width="90.874992"
height="115.80065" height="115.80065"
x="872.67383" x="872.67383"
y="486.55563" /> y="486.55563" />
<text <text
id="text3384-0" id="text3384-0"
y="969.2854" y="921.73846"
x="489.56451" x="489.56451"
style="font-style:normal;font-variant:normal;font-weight:normal;font-stretch:normal;line-height:0%;font-family:sans-serif;-inkscape-font-specification:'sans-serif, Normal';text-align:start;letter-spacing:0px;word-spacing:0px;writing-mode:lr-tb;text-anchor:start;fill:#000000;fill-opacity:1;stroke:none;stroke-width:1px;stroke-linecap:butt;stroke-linejoin:miter;stroke-opacity:1" style="font-style:normal;font-variant:normal;font-weight:normal;font-stretch:normal;line-height:0%;font-family:sans-serif;-inkscape-font-specification:'sans-serif, Normal';text-align:start;letter-spacing:0px;word-spacing:0px;writing-mode:lr-tb;text-anchor:start;fill:#000000;fill-opacity:1;stroke:none;stroke-width:1px;stroke-linecap:butt;stroke-linejoin:miter;stroke-opacity:1"
xml:space="preserve"><tspan xml:space="preserve"><tspan
@ -319,7 +319,7 @@
id="tspan3391" id="tspan3391"
sodipodi:role="line" sodipodi:role="line"
x="489.56451" x="489.56451"
y="969.2854">ImageChecksum</tspan></text> y="921.73846">ImageChecksum</tspan></text>
<g <g
transform="translate(-42.209584,-80.817124)" transform="translate(-42.209584,-80.817124)"
id="g3458"> id="g3458">
@ -518,5 +518,24 @@
id="tspan301-5" id="tspan301-5"
style="font-size:12px;line-height:0">OSBuild</tspan></text> style="font-size:12px;line-height:0">OSBuild</tspan></text>
</g> </g>
<rect
transform="matrix(0,1,1,0,0,0)"
style="fill:#729fcf;fill-rule:evenodd;stroke:none;stroke-width:1.83502px;stroke-linecap:butt;stroke-linejoin:miter;stroke-opacity:1"
id="rect3338-1-3"
width="88.544876"
height="115.80065"
x="970.31763"
y="486.55563" />
<text
id="text3384-0-6"
y="1018.2172"
x="489.56451"
style="font-style:normal;font-variant:normal;font-weight:normal;font-stretch:normal;line-height:0%;font-family:sans-serif;-inkscape-font-specification:'sans-serif, Normal';text-align:start;letter-spacing:0px;word-spacing:0px;writing-mode:lr-tb;text-anchor:start;fill:#000000;fill-opacity:1;stroke:none;stroke-width:1px;stroke-linecap:butt;stroke-linejoin:miter;stroke-opacity:1"
xml:space="preserve"><tspan
style="font-size:13.1475px;line-height:1.25"
id="tspan3391-7"
sodipodi:role="line"
x="489.56451"
y="1018.2172">ImageContainer</tspan></text>
</g> </g>
</svg> </svg>

Before

Width:  |  Height:  |  Size: 21 KiB

After

Width:  |  Height:  |  Size: 22 KiB

View File

@ -1555,6 +1555,56 @@ OSBuild Composer for building images
arch. arch.
Image container
===============
This phase supports building containers in OSBS that embed an image created in
the same compose. This can be useful for delivering the image to users running
in containerized environments.
Pungi will start a ``buildContainer`` task in Koji with configured source
repository. The ``Dockerfile`` can expect that a repo file will be injected
into the container that defines a repo named ``image-to-include``, and its
``baseurl`` will point to the image to include. It is possible to extract the
URL with a command like ``dnf config-manager --dump image-to-include | awk
'/baseurl =/{print $3}'```
**image_container**
(*dict*) -- configuration for building containers embedding an image.
Format: ``{variant_uid_regex: [{...}]}``.
The inner object will define a single container. These keys are required:
* ``url``, ``target``, ``git_branch``. See OSBS section for definition of
these.
* ``image_spec`` -- (*object*) A string mapping of filters used to select
the image to embed. All images listed in metadata for the variant will be
processed. The keys of this filter are used to select metadata fields for
the image, and values are regular expression that need to match the
metadata value.
The filter should match exactly one image.
Example config
--------------
::
image_container = {
"^Server$": [{
"url": "git://example.com/dockerfiles.git?#HEAD",
"target": "f24-container-candidate",
"git_branch": "f24",
"image_spec": {
"format": "qcow2",
"arch": "x86_64",
"path": ".*/guest-image-.*$",
}
}]
}
OSTree Settings OSTree Settings
=============== ===============

View File

@ -115,16 +115,30 @@ ImageBuild
This phase wraps up ``koji image-build``. It also updates the metadata This phase wraps up ``koji image-build``. It also updates the metadata
ultimately responsible for ``images.json`` manifest. ultimately responsible for ``images.json`` manifest.
OSBuild
-------
Similarly to image build, this phases creates a koji `osbuild` task. In the
background it uses OSBuild Composer to create images.
OSBS OSBS
---- ----
This phase builds docker base images in `OSBS This phase builds container base images in `OSBS
<http://osbs.readthedocs.io/en/latest/index.html>`_. <http://osbs.readthedocs.io/en/latest/index.html>`_.
The finished images are available in registry provided by OSBS, but not The finished images are available in registry provided by OSBS, but not
downloaded directly into the compose. The is metadata about the created image downloaded directly into the compose. The is metadata about the created image
in ``compose/metadata/osbs.json``. in ``compose/metadata/osbs.json``.
ImageContainer
--------------
This phase builds a container image in OSBS, and stores the metadata in the
same file as OSBS phase. The container produced here wraps a different image,
created it ImageBuild or OSBuild phase. It can be useful to deliver a VM image
to containerized environments.
OSTreeInstaller OSTreeInstaller
--------------- ---------------

View File

@ -1213,6 +1213,26 @@ def make_schema():
}, },
"additionalProperties": False, "additionalProperties": False,
}, },
"image_container": {
"type": "object",
"patternProperties": {
".+": _one_or_list(
{
"type": "object",
"properties": {
"url": {"type": "url"},
"target": {"type": "string"},
"priority": {"type": "number"},
"failable": {"type": "boolean"},
"git_branch": {"type": "string"},
"image_spec": {"type": "object"},
},
"required": ["url", "target", "git_branch", "image_spec"],
}
),
},
"additionalProperties": False,
},
"extra_files": _variant_arch_mapping( "extra_files": _variant_arch_mapping(
{ {
"type": "array", "type": "array",

View File

@ -27,6 +27,7 @@ from .createiso import CreateisoPhase # noqa
from .extra_isos import ExtraIsosPhase # noqa from .extra_isos import ExtraIsosPhase # noqa
from .live_images import LiveImagesPhase # noqa from .live_images import LiveImagesPhase # noqa
from .image_build import ImageBuildPhase # noqa from .image_build import ImageBuildPhase # noqa
from .image_container import ImageContainerPhase # noqa
from .osbuild import OSBuildPhase # noqa from .osbuild import OSBuildPhase # noqa
from .repoclosure import RepoclosurePhase # noqa from .repoclosure import RepoclosurePhase # noqa
from .test import TestPhase # noqa from .test import TestPhase # noqa

View File

@ -0,0 +1,120 @@
# -*- coding: utf-8 -*-
import os
import re
from kobo.threads import ThreadPool, WorkerThread
from .base import ConfigGuardedPhase, PhaseLoggerMixin
from .. import util
from ..wrappers import kojiwrapper
from ..phases.osbs import add_metadata
class ImageContainerPhase(PhaseLoggerMixin, ConfigGuardedPhase):
name = "image_container"
def __init__(self, compose):
super(ImageContainerPhase, self).__init__(compose)
self.pool = ThreadPool(logger=self.logger)
self.pool.metadata = {}
def run(self):
for variant in self.compose.get_variants():
for conf in self.get_config_block(variant):
self.pool.add(ImageContainerThread(self.pool))
self.pool.queue_put((self.compose, variant, conf))
self.pool.start()
class ImageContainerThread(WorkerThread):
def process(self, item, num):
compose, variant, config = item
self.num = num
with util.failable(
compose,
bool(config.pop("failable", None)),
variant,
"*",
"osbs",
logger=self.pool._logger,
):
self.worker(compose, variant, config)
def worker(self, compose, variant, config):
msg = "Image container task for variant %s" % variant.uid
self.pool.log_info("[BEGIN] %s" % msg)
source = config.pop("url")
target = config.pop("target")
priority = config.pop("priority", None)
config["yum_repourls"] = [
self._get_repo(
compose,
variant,
config.get("arch_override", "").split(" "),
config.pop("image_spec"),
)
]
# Start task
koji = kojiwrapper.KojiWrapper(compose.conf["koji_profile"])
koji.login()
task_id = koji.koji_proxy.buildContainer(
source, target, config, priority=priority
)
# Wait for it to finish and capture the output into log file (even
# though there is not much there).
log_dir = os.path.join(compose.paths.log.topdir(), "image_container")
util.makedirs(log_dir)
log_file = os.path.join(
log_dir, "%s-%s-watch-task.log" % (variant.uid, self.num)
)
if koji.watch_task(task_id, log_file) != 0:
raise RuntimeError(
"ImageContainer: task %s failed: see %s for details"
% (task_id, log_file)
)
add_metadata(variant, task_id, compose, config.get("scratch", False))
self.pool.log_info("[DONE ] %s" % msg)
def _get_repo(self, compose, variant, arches, image_spec):
"""
Return a repo file that points baseurl to the image specified by
image_spec.
"""
image_paths = set()
for arch in arches or compose.im.images[variant.uid].keys():
for image in compose.im.images[variant.uid].get(arch, []):
for key, value in image_spec.items():
if not re.match(value, getattr(image, key)):
break
else:
image_paths.add(image.path)
if len(image_paths) != 1:
raise RuntimeError(
"%d images matched specification. Only one was expected."
% len(image_paths)
)
image_path = image_paths.pop()
absolute_path = os.path.join(compose.paths.compose.topdir(), image_path)
repo_file = os.path.join(
compose.paths.work.tmp_dir(None, variant),
"image-container-%s-%s.repo" % (variant, self.num),
)
with open(repo_file, "w") as f:
f.write("[image-to-include]\n")
f.write("name=Location of image to embed\n")
f.write("baseurl=%s\n" % util.translate_path(compose, absolute_path))
f.write("enabled=0\n")
f.write("gpgcheck=0\n")
return util.translate_path(compose, repo_file)

View File

@ -394,6 +394,7 @@ def run_compose(
image_build_phase = pungi.phases.ImageBuildPhase(compose) image_build_phase = pungi.phases.ImageBuildPhase(compose)
osbuild_phase = pungi.phases.OSBuildPhase(compose) osbuild_phase = pungi.phases.OSBuildPhase(compose)
osbs_phase = pungi.phases.OSBSPhase(compose) osbs_phase = pungi.phases.OSBSPhase(compose)
image_container_phase = pungi.phases.ImageContainerPhase(compose)
image_checksum_phase = pungi.phases.ImageChecksumPhase(compose) image_checksum_phase = pungi.phases.ImageChecksumPhase(compose)
repoclosure_phase = pungi.phases.RepoclosurePhase(compose) repoclosure_phase = pungi.phases.RepoclosurePhase(compose)
test_phase = pungi.phases.TestPhase(compose) test_phase = pungi.phases.TestPhase(compose)
@ -417,6 +418,7 @@ def run_compose(
extra_isos_phase, extra_isos_phase,
osbs_phase, osbs_phase,
osbuild_phase, osbuild_phase,
image_container_phase,
): ):
if phase.skip(): if phase.skip():
continue continue
@ -516,9 +518,12 @@ def run_compose(
livemedia_phase, livemedia_phase,
osbuild_phase, osbuild_phase,
) )
post_image_phase = pungi.phases.WeaverPhase(
compose, (image_checksum_phase, image_container_phase)
)
compose_images_phase = pungi.phases.WeaverPhase(compose, compose_images_schema) compose_images_phase = pungi.phases.WeaverPhase(compose, compose_images_schema)
extra_phase_schema = ( extra_phase_schema = (
(compose_images_phase, image_checksum_phase), (compose_images_phase, post_image_phase),
osbs_phase, osbs_phase,
repoclosure_phase, repoclosure_phase,
) )

View File

@ -0,0 +1,262 @@
# -*- coding: utf-8 -*-
import mock
import os
from tests import helpers
from pungi import checks
from pungi.phases import image_container
class ImageContainerPhaseTest(helpers.PungiTestCase):
@mock.patch("pungi.phases.image_container.ThreadPool")
def test_run(self, ThreadPool):
cfg = helpers.IterableMock()
compose = helpers.DummyCompose(
self.topdir, {"image_container": {"^Everything$": cfg}}
)
pool = ThreadPool.return_value
phase = image_container.ImageContainerPhase(compose)
phase.run()
self.assertEqual(len(pool.add.call_args_list), 1)
self.assertEqual(
pool.queue_put.call_args_list,
[mock.call((compose, compose.variants["Everything"], cfg))],
)
@mock.patch("pungi.phases.image_container.ThreadPool")
def test_skip_without_config(self, ThreadPool):
compose = helpers.DummyCompose(self.topdir, {})
compose.just_phases = None
compose.skip_phases = []
phase = image_container.ImageContainerPhase(compose)
self.assertTrue(phase.skip())
class ImageContainerConfigTest(helpers.PungiTestCase):
def assertConfigMissing(self, cfg, key):
conf = helpers.load_config(
helpers.PKGSET_REPOS, **{"image_container": {"^Server$": cfg}}
)
errors, warnings = checks.validate(conf, offline=True)
self.assertIn(
"Failed validation in image_container.^Server$: %r is not valid under any of the given schemas" # noqa: E501
% cfg,
errors,
)
self.assertIn(" Possible reason: %r is a required property" % key, errors)
self.assertEqual([], warnings)
def test_correct(self):
conf = helpers.load_config(
helpers.PKGSET_REPOS,
**{
"image_container": {
"^Server$": [
{
"url": "http://example.com/repo.git#HEAD",
"target": "container-candidate",
"git_branch": "main",
"image_spec": {"type": "qcow2"},
}
]
}
}
)
errors, warnings = checks.validate(conf, offline=True)
self.assertEqual([], errors)
self.assertEqual([], warnings)
def test_missing_url(self):
self.assertConfigMissing(
{
"target": "container-candidate",
"git_branch": "main",
"image_spec": {"type": "qcow2"},
},
"url",
)
def test_missing_target(self):
self.assertConfigMissing(
{
"url": "http://example.com/repo.git#HEAD",
"git_branch": "main",
"image_spec": {"type": "qcow2"},
},
"target",
)
def test_missing_git_branch(self):
self.assertConfigMissing(
{
"url": "http://example.com/repo.git#HEAD",
"target": "container-candidate",
"image_spec": {"type": "qcow2"},
},
"git_branch",
)
def test_missing_image_spec(self):
self.assertConfigMissing(
{
"url": "http://example.com/repo.git#HEAD",
"target": "container-candidate",
"git_branch": "main",
},
"image_spec",
)
class ImageContainerThreadTest(helpers.PungiTestCase):
def setUp(self):
super(ImageContainerThreadTest, self).setUp()
self.pool = mock.Mock()
self.repofile_path = "work/global/tmp-Server/image-container-Server-1.repo"
self.t = image_container.ImageContainerThread(self.pool)
self.compose = helpers.DummyCompose(
self.topdir,
{
"koji_profile": "koji",
"translate_paths": [(self.topdir, "http://root")],
},
)
self.cfg = {
"url": "git://example.com/repo?#BEEFCAFE",
"target": "f24-docker-candidate",
"git_branch": "f24-docker",
"image_spec": {"type": "qcow2"},
"arch_override": "x86_64",
}
self.compose.im.images["Server"] = {
"x86_64": [
mock.Mock(path="Server/x86_64/iso/image.iso", type="iso"),
mock.Mock(path="Server/x86_64/images/image.qcow2", type="qcow2"),
]
}
def _setupMock(self, KojiWrapper):
self.wrapper = KojiWrapper.return_value
self.wrapper.koji_proxy.buildContainer.return_value = 12345
self.wrapper.watch_task.return_value = 0
def assertRepoFile(self):
repofile = os.path.join(self.topdir, self.repofile_path)
with open(repofile) as f:
repo_content = list(f)
self.assertIn("[image-to-include]\n", repo_content)
self.assertIn(
"baseurl=http://root/compose/Server/x86_64/images/image.qcow2\n",
repo_content,
)
self.assertIn("enabled=0\n", repo_content)
def assertKojiCalls(self, cfg, scratch=False):
opts = {
"git_branch": cfg["git_branch"],
"arch_override": cfg["arch_override"],
"yum_repourls": ["http://root/" + self.repofile_path],
}
if scratch:
opts["scratch"] = True
self.assertEqual(
self.wrapper.mock_calls,
[
mock.call.login(),
mock.call.koji_proxy.buildContainer(
cfg["url"], cfg["target"], opts, priority=None,
),
mock.call.watch_task(
12345,
os.path.join(
self.topdir,
"logs/global/image_container/Server-1-watch-task.log",
),
),
],
)
@mock.patch("pungi.phases.image_container.add_metadata")
@mock.patch("pungi.phases.image_container.kojiwrapper.KojiWrapper")
def test_success(self, KojiWrapper, add_metadata):
self._setupMock(KojiWrapper)
self.t.process(
(self.compose, self.compose.variants["Server"], self.cfg.copy()), 1
)
self.assertRepoFile()
self.assertKojiCalls(self.cfg)
self.assertEqual(
add_metadata.call_args_list,
[mock.call(self.compose.variants["Server"], 12345, self.compose, False)],
)
@mock.patch("pungi.phases.image_container.add_metadata")
@mock.patch("pungi.phases.image_container.kojiwrapper.KojiWrapper")
def test_scratch_build(self, KojiWrapper, add_metadata):
self.cfg["scratch"] = True
self._setupMock(KojiWrapper)
self.t.process(
(self.compose, self.compose.variants["Server"], self.cfg.copy()), 1
)
self.assertRepoFile()
self.assertKojiCalls(self.cfg, scratch=True)
self.assertEqual(
add_metadata.call_args_list,
[mock.call(self.compose.variants["Server"], 12345, self.compose, True)],
)
@mock.patch("pungi.phases.image_container.add_metadata")
@mock.patch("pungi.phases.image_container.kojiwrapper.KojiWrapper")
def test_task_fail(self, KojiWrapper, add_metadata):
self._setupMock(KojiWrapper)
self.wrapper.watch_task.return_value = 1
with self.assertRaises(RuntimeError) as ctx:
self.t.process(
(self.compose, self.compose.variants["Server"], self.cfg.copy()), 1
)
self.assertRegex(str(ctx.exception), r"task 12345 failed: see .+ for details")
self.assertRepoFile()
self.assertKojiCalls(self.cfg)
self.assertEqual(add_metadata.call_args_list, [])
@mock.patch("pungi.phases.image_container.add_metadata")
@mock.patch("pungi.phases.image_container.kojiwrapper.KojiWrapper")
def test_task_fail_failable(self, KojiWrapper, add_metadata):
self.cfg["failable"] = "*"
self._setupMock(KojiWrapper)
self.wrapper.watch_task.return_value = 1
self.t.process(
(self.compose, self.compose.variants["Server"], self.cfg.copy()), 1
)
self.assertRepoFile()
self.assertKojiCalls(self.cfg)
self.assertEqual(add_metadata.call_args_list, [])
@mock.patch("pungi.phases.image_container.add_metadata")
@mock.patch("pungi.phases.image_container.kojiwrapper.KojiWrapper")
def test_non_unique_spec(self, KojiWrapper, add_metadata):
self.cfg["image_spec"] = {"path": ".*/image\\..*"}
self._setupMock(KojiWrapper)
with self.assertRaises(RuntimeError) as ctx:
self.t.process(
(self.compose, self.compose.variants["Server"], self.cfg.copy()), 1
)
self.assertRegex(
str(ctx.exception), "2 images matched specification. Only one was expected."
)
self.assertEqual(self.wrapper.mock_calls, [])
self.assertEqual(add_metadata.call_args_list, [])