almalinux/pungi
14
1
Fork 0
Code Issues Pull Requests Releases Activity

[atomic] Add atomic_installer phase

Browse Source 
This phase runs lorax with extra templates in Koji runroot task, links
the boot.iso to proper location in compose directory and adds the
installer iso to image manifest. This phase runs concurrently with live
media etc.

Signed-off-by: Lubomír Sedlář <lsedlar@redhat.com>
This commit is contained in:
Lubomír Sedlář 2016-03-23 10:40:16 +01:00
parent 536c6c85b7
commit 8d224b206b
10 changed files with 527 additions and 5 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
bin/pungi-koji
View File

@ -226,6 +226,7 @@ def run_compose(compose):
gather_phase = pungi.phases.GatherPhase(compose, pkgset_phase)
extrafiles_phase = pungi.phases.ExtraFilesPhase(compose, pkgset_phase)
createrepo_phase = pungi.phases.CreaterepoPhase(compose)
atomic_installer_phase = pungi.phases.AtomicInstallerPhase(compose)
ostree_phase = pungi.phases.OSTreePhase(compose)
productimg_phase = pungi.phases.ProductimgPhase(compose, pkgset_phase)
createiso_phase = pungi.phases.CreateisoPhase(compose)
@ -240,7 +241,7 @@ def run_compose(compose):
buildinstall_phase, productimg_phase, gather_phase,
extrafiles_phase, createiso_phase, liveimages_phase,
livemedia_phase, image_build_phase, image_checksum_phase,
test_phase, ostree_phase):
test_phase, ostree_phase, atomic_installer_phase):
if phase.skip():
continue
try:
@ -346,11 +347,13 @@ def run_compose(compose):
liveimages_phase.start()
image_build_phase.start()
livemedia_phase.start()
atomic_installer_phase.start()
createiso_phase.stop()
liveimages_phase.stop()
image_build_phase.stop()
livemedia_phase.stop()
atomic_installer_phase.stop()
image_checksum_phase.start()
image_checksum_phase.stop()

60
doc/configuration.rst
View File

@ -142,6 +142,7 @@ Options
* live
* image-build
* live-media
* atomic_installer
.. note::
@ -973,6 +974,65 @@ Example config
]
Atomic Installer Settings
=========================
The ``atomic_installer`` phase of *Pungi* can produce installer image bundling
an OSTree repository. This always runs in Koji as a ``runroot`` task.
**atomic**
(*dict*) -- a variant/arch mapping of configuration. The format should be
``[(variant_uid_regex, {arch|*: config_dict})]``.
The configuration dict for each variant arch pair must have this key:
* ``source_repo_from`` -- (*str*) Name of variant serving as source
repository.
These keys are optional:
* ``release`` -- (*str*) Release value to set for the installer image. Set
to ``None`` to use the date.respin format.
* ``filename`` -- (*str*) What to name the installer iso. This is a
template with options listed in Image naming section. If not specified,
global naming format will be used.
These optional keys are passed to ``lorax`` to customize the build.
* ``installpkgs`` -- (*[str]*)
* ``add_template`` -- (*[str]*)
* ``add_arch_template`` -- (*[str]*)
* ``add_template_var`` -- (*[str]*)
* ``add_arch_template_var`` -- (*[str]*)
Example config
--------------
::
atomic = [
("^Atomic$", {
"x86_64": {
"source_repo_from": "Everything",
"release": None,
"filename": "%(release_short)s-%(variant)s-%(arch)s-%(version)s-%(compose_date)s.iso",
"installpkgs": ["fedora-productimg-atomic"],
"add_template": ["/spin-kickstarts/atomic-installer/lorax-configure-repo.tmpl"],
"add_template_var": [
"ostree_osname=fedora-atomic",
"ostree_ref=fedora-atomic/Rawhide/x86_64/docker-host",
],
"add_arch_template": ["/spin-kickstarts/atomic-installer/lorax-embed-repo.tmpl"],
"add_arch_template_var": [
"ostree_repo=https://kojipkgs.fedoraproject.org/compose/atomic/Rawhide/",
"ostree_osname=fedora-atomic",
"ostree_ref=fedora-atomic/Rawhide/x86_64/docker-host",
]
}
})
]
Media Checksums Settings
========================

1
pungi/phases/__init__.py
View File

@ -30,3 +30,4 @@ from test import TestPhase # noqa
from image_checksum import ImageChecksumPhase # noqa
from livemedia_phase import LiveMediaPhase # noqa
from ostree import OSTreePhase # noqa
from atomic_installer import AtomicInstallerPhase # noqa

143
pungi/phases/atomic_installer.py Normal file
View File

@ -0,0 +1,143 @@
# -*- coding: utf-8 -*-
import os
from kobo.threads import ThreadPool, WorkerThread
import traceback
import shutil
from productmd import images
from .base import ConfigGuardedPhase
from .. import util
from ..paths import translate_path
from ..wrappers import kojiwrapper, iso, lorax
class AtomicInstallerPhase(ConfigGuardedPhase):
name = 'atomic'
config_options = (
{
"name": "atomic",
"expected_types": [dict],
"optional": True,
}
)
def __init__(self, compose):
super(AtomicInstallerPhase, self).__init__(compose)
self.pool = ThreadPool(logger=self.compose._logger)
def run(self):
for variant in self.compose.get_variants():
for arch in variant.arches:
for conf in util.get_arch_variant_data(self.compose.conf, self.name, arch, variant):
self.pool.add(AtomicInstallerThread(self.pool))
self.pool.queue_put((self.compose, variant, arch, conf))
self.pool.start()
class AtomicInstallerThread(WorkerThread):
def process(self, item, num):
compose, variant, arch, config = item
self.num = num
try:
self.worker(compose, variant, arch, config)
except Exception as exc:
if not compose.can_fail(variant, arch, 'atomic_installer'):
raise
else:
msg = ('[FAIL] Atomic for variant %s, arch %s, failed, but going on anyway.\n%s'
% (variant.uid, arch, exc))
self.pool.log_info(msg)
tb = traceback.format_exc()
self.pool.log_debug(tb)
def worker(self, compose, variant, arch, config):
msg = 'Atomic phase for variant %s, arch %s' % (variant.uid, arch)
self.pool.log_info('[BEGIN] %s' % msg)
self.logdir = compose.paths.log.topdir('{}/atomic'.format(arch))
source_variant = compose.variants[config['source_repo_from']]
source_repo = translate_path(compose, compose.paths.compose.repository(arch, source_variant))
self._run_atomic_cmd(compose, variant, arch, config, source_repo)
disc_type = compose.conf.get('disc_types', {}).get('dvd', 'dvd')
filename = compose.get_image_name(arch, variant, disc_type=disc_type,
format=config.get('filename'))
self._copy_image(compose, variant, arch, filename)
self._add_to_manifest(compose, variant, arch, filename)
self.pool.log_info('[DONE ] %s' % msg)
def _get_release(self, compose, config):
if 'release' in config and config['release'] is None:
return compose.image_release
return config.get('release', None)
def _copy_image(self, compose, variant, arch, filename):
iso_path = compose.paths.compose.iso_path(arch, variant, filename)
source_dir = compose.paths.compose.os_tree(arch, variant)
boot_iso = os.path.join(source_dir, 'images', 'boot.iso')
try:
os.link(boot_iso, iso_path)
except OSError:
shutil.copy2(boot_iso, iso_path)
def _add_to_manifest(self, compose, variant, arch, filename):
full_iso_path = compose.paths.compose.iso_path(arch, variant, filename)
iso_path = compose.paths.compose.iso_path(arch, variant, filename, relative=True)
iso_wrapper = iso.IsoWrapper()
implant_md5 = iso_wrapper.get_implanted_md5(full_iso_path)
img = images.Image(compose.im)
img.path = iso_path
img.mtime = util.get_mtime(full_iso_path)
img.size = util.get_file_size(full_iso_path)
img.arch = arch
img.type = "boot"
img.format = "iso"
img.disc_number = 1
img.disc_count = 1
img.bootable = True
img.subvariant = variant.name
img.implant_md5 = implant_md5
try:
img.volume_id = iso_wrapper.get_volume_id(full_iso_path)
except RuntimeError:
pass
compose.im.add(variant.uid, arch, img)
def _run_atomic_cmd(self, compose, variant, arch, config, source_repo):
image_dir = compose.paths.compose.os_tree(arch, variant)
lorax_wrapper = lorax.LoraxWrapper()
cmd = lorax_wrapper.get_lorax_cmd(
compose.conf['release_name'],
compose.conf["release_version"],
self._get_release(compose, config),
repo_baseurl=source_repo,
output_dir=image_dir,
variant=variant.uid,
nomacboot=True,
buildinstallpackages=config.get('installpkgs'),
add_template=config.get('add_template'),
add_arch_template=config.get('add_arch_template'),
add_template_var=config.get('add_template_var'),
add_arch_template_var=config.get('add_arch_template_var')
)
runroot_channel = compose.conf.get("runroot_channel", None)
runroot_tag = compose.conf["runroot_tag"]
packages = ['pungi', 'lorax']
log_file = os.path.join(self.logdir, 'runroot.log')
koji = kojiwrapper.KojiWrapper(compose.conf["koji_profile"])
koji_cmd = koji.get_runroot_cmd(runroot_tag, arch, cmd,
channel=runroot_channel,
use_shell=True, task_id=True,
packages=packages, mounts=[compose.topdir])
output = koji.run_runroot_cmd(koji_cmd, log_file=log_file)
if output["retcode"] != 0:
raise RuntimeError("Runroot task failed: %s. See %s for more details."
% (output["task_id"], log_file))

9
pungi/util.py
View File

@ -446,3 +446,12 @@ def find_old_compose(old_compose_dirs, release_short, release_version,
return None
return sorted(composes)[-1][1]
def process_args(fmt, args):
"""Given a list of arguments, format each value with the format string.
>>> process_args('--opt={}', ['foo', 'bar'])
['--opt=foo', '--opt=bar']
"""
return [fmt.format(val) for val in force_list(args or [])]

12
pungi/wrappers/lorax.py
View File

@ -18,12 +18,15 @@
import os
from kobo.shortcuts import force_list
from ..util import process_args
class LoraxWrapper(object):
def get_lorax_cmd(self, product, version, release, repo_baseurl, output_dir,
variant=None, bugurl=None, nomacboot=False, noupgrade=False,
is_final=False, buildarch=None, volid=None, buildinstallpackages=None):
is_final=False, buildarch=None, volid=None, buildinstallpackages=None,
add_template=None, add_arch_template=None,
add_template_var=None, add_arch_template_var=None):
cmd = ["lorax"]
cmd.append("--product=%s" % product)
cmd.append("--version=%s" % version)
@ -55,8 +58,11 @@ class LoraxWrapper(object):
if volid:
cmd.append("--volid=%s" % volid)
if buildinstallpackages:
cmd.extend(["--installpkgs=%s" % package for package in buildinstallpackages])
cmd.extend(process_args('--installpkgs={}', buildinstallpackages))
cmd.extend(process_args('--add-template={}', add_template))
cmd.extend(process_args('--add-arch-template={}', add_arch_template))
cmd.extend(process_args('--add-template-var={}', add_template_var))
cmd.extend(process_args('--add-arch-template-var={}', add_arch_template_var))
output_dir = os.path.abspath(output_dir)
cmd.append(output_dir)

4
tests/helpers.py
View File

@ -95,3 +95,7 @@ def union(*args):
for arg in args:
res.update(arg)
return res
def boom(*args, **kwargs):
raise Exception('BOOM')

279
tests/test_atomic_installer_phase.py Normal file
View File

@ -0,0 +1,279 @@
#!/usr/bin/env python
# -*- coding: utf-8 -*-
import unittest
import mock
import os
import sys
sys.path.insert(0, os.path.join(os.path.dirname(__file__), '..'))
from tests import helpers
from pungi.phases import atomic_installer as atomic
class AtomicInstallerPhaseTest(helpers.PungiTestCase):
@mock.patch('pungi.phases.atomic_installer.ThreadPool')
def test_run(self, ThreadPool):
cfg = mock.Mock()
compose = helpers.DummyCompose(self.topdir, {
'atomic': [
('^Everything$', {'x86_64': cfg})
]
})
pool = ThreadPool.return_value
phase = atomic.AtomicInstallerPhase(compose)
phase.run()
self.assertEqual(len(pool.add.call_args_list), 1)
self.assertEqual(pool.queue_put.call_args_list,
[mock.call((compose, compose.variants['Everything'], 'x86_64', cfg))])
@mock.patch('pungi.phases.atomic_installer.ThreadPool')
def test_skip_without_config(self, ThreadPool):
compose = helpers.DummyCompose(self.topdir, {})
compose.just_phases = None
compose.skip_phases = []
phase = atomic.AtomicInstallerPhase(compose)
self.assertTrue(phase.skip())
class AtomicThreadTest(helpers.PungiTestCase):
def assertImageAdded(self, compose, ImageCls, IsoWrapper):
image = ImageCls.return_value
self.assertEqual(image.path, 'Everything/x86_64/iso/image-name')
self.assertEqual(image.mtime, 13579)
self.assertEqual(image.size, 1024)
self.assertEqual(image.arch, 'x86_64')
self.assertEqual(image.type, "boot")
self.assertEqual(image.format, "iso")
self.assertEqual(image.disc_number, 1)
self.assertEqual(image.disc_count, 1)
self.assertEqual(image.bootable, True)
self.assertEqual(image.implant_md5, IsoWrapper.return_value.get_implanted_md5.return_value)
self.assertEqual(compose.im.add.mock_calls,
[mock.call('Everything', 'x86_64', image)])
@mock.patch('productmd.images.Image')
@mock.patch('pungi.util.get_mtime')
@mock.patch('pungi.util.get_file_size')
@mock.patch('pungi.wrappers.iso.IsoWrapper')
@mock.patch('os.link')
@mock.patch('pungi.wrappers.kojiwrapper.KojiWrapper')
def test_run(self, KojiWrapper, link, IsoWrapper,
get_file_size, get_mtime, ImageCls):
compose = helpers.DummyCompose(self.topdir, {
'release_name': 'Fedora',
'release_version': 'Rawhide',
'koji_profile': 'koji',
'runroot_tag': 'rrt',
})
pool = mock.Mock()
cfg = {
'source_repo_from': 'Everything',
'release': '20160321.n.0',
'filename': 'Fedora-Atomic.iso',
}
koji = KojiWrapper.return_value
koji.run_runroot_cmd.return_value = {
'task_id': 1234,
'retcode': 0,
'output': 'Foo bar\n',
}
get_file_size.return_value = 1024
get_mtime.return_value = 13579
final_iso_path = self.topdir + '/compose/Everything/x86_64/iso/image-name'
t = atomic.AtomicInstallerThread(pool)
t.process((compose, compose.variants['Everything'], 'x86_64', cfg), 1)
self.assertEqual(koji.get_runroot_cmd.call_args_list,
[mock.call('rrt', 'x86_64',
['lorax',
'--product=Fedora',
'--version=Rawhide',
'--release=20160321.n.0',
'--source=file://{}/compose/Everything/x86_64/os'.format(self.topdir),
'--variant=Everything',
'--nomacboot',
self.topdir + '/compose/Everything/x86_64/os'],
channel=None, mounts=[self.topdir],
packages=['pungi', 'lorax'],
task_id=True, use_shell=True)])
self.assertEqual(koji.run_runroot_cmd.call_args_list,
[mock.call(koji.get_runroot_cmd.return_value,
log_file=self.topdir + '/logs/x86_64/atomic/runroot.log')])
self.assertEqual(link.call_args_list,
[mock.call(self.topdir + '/compose/Everything/x86_64/os/images/boot.iso',
final_iso_path)])
self.assertEqual(get_file_size.call_args_list, [mock.call(final_iso_path)])
self.assertEqual(get_mtime.call_args_list, [mock.call(final_iso_path)])
self.assertImageAdded(compose, ImageCls, IsoWrapper)
self.assertEqual(compose.get_image_name.call_args_list,
[mock.call('x86_64', compose.variants['Everything'],
disc_type='dvd', format='Fedora-Atomic.iso')])
@mock.patch('productmd.images.Image')
@mock.patch('pungi.util.get_mtime')
@mock.patch('pungi.util.get_file_size')
@mock.patch('pungi.wrappers.iso.IsoWrapper')
@mock.patch('os.link')
@mock.patch('pungi.wrappers.kojiwrapper.KojiWrapper')
def test_run_with_implicit_release(self, KojiWrapper, link,
IsoWrapper, get_file_size, get_mtime, ImageCls):
compose = helpers.DummyCompose(self.topdir, {
'release_name': 'Fedora',
'release_version': 'Rawhide',
'koji_profile': 'koji',
'runroot_tag': 'rrt',
})
pool = mock.Mock()
cfg = {
'source_repo_from': 'Everything',
'release': None,
"installpkgs": ["fedora-productimg-atomic"],
"add_template": ["/spin-kickstarts/atomic-installer/lorax-configure-repo.tmpl"],
"add_template_var": [
"ostree_osname=fedora-atomic",
"ostree_ref=fedora-atomic/Rawhide/x86_64/docker-host",
],
"add_arch_template": ["/spin-kickstarts/atomic-installer/lorax-embed-repo.tmpl"],
"add_arch_template_var": [
"ostree_repo=https://kojipkgs.fedoraproject.org/compose/atomic/Rawhide/",
"ostree_osname=fedora-atomic",
"ostree_ref=fedora-atomic/Rawhide/x86_64/docker-host",
],
}
koji = KojiWrapper.return_value
koji.run_runroot_cmd.return_value = {
'task_id': 1234,
'retcode': 0,
'output': 'Foo bar\n',
}
get_file_size.return_value = 1024
get_mtime.return_value = 13579
final_iso_path = self.topdir + '/compose/Everything/x86_64/iso/image-name'
t = atomic.AtomicInstallerThread(pool)
t.process((compose, compose.variants['Everything'], 'x86_64', cfg), 1)
self.assertEqual(
koji.get_runroot_cmd.call_args_list,
[mock.call('rrt', 'x86_64',
['lorax',
'--product=Fedora',
'--version=Rawhide', '--release=20151203.t.0',
'--source=file://{}/compose/Everything/x86_64/os'.format(self.topdir),
'--variant=Everything',
'--nomacboot',
'--installpkgs=fedora-productimg-atomic',
'--add-template=/spin-kickstarts/atomic-installer/lorax-configure-repo.tmpl',
'--add-arch-template=/spin-kickstarts/atomic-installer/lorax-embed-repo.tmpl',
'--add-template-var=ostree_osname=fedora-atomic',
'--add-template-var=ostree_ref=fedora-atomic/Rawhide/x86_64/docker-host',
'--add-arch-template-var=ostree_repo=https://kojipkgs.fedoraproject.org/compose/atomic/Rawhide/',
'--add-arch-template-var=ostree_osname=fedora-atomic',
'--add-arch-template-var=ostree_ref=fedora-atomic/Rawhide/x86_64/docker-host',
self.topdir + '/compose/Everything/x86_64/os'],
channel=None, mounts=[self.topdir],
packages=['pungi', 'lorax'],
task_id=True, use_shell=True)])
self.assertEqual(koji.run_runroot_cmd.call_args_list,
[mock.call(koji.get_runroot_cmd.return_value,
log_file=self.topdir + '/logs/x86_64/atomic/runroot.log')])
self.assertEqual(link.call_args_list,
[mock.call(self.topdir + '/compose/Everything/x86_64/os/images/boot.iso',
final_iso_path)])
self.assertEqual(get_file_size.call_args_list, [mock.call(final_iso_path)])
self.assertEqual(get_mtime.call_args_list, [mock.call(final_iso_path)])
self.assertImageAdded(compose, ImageCls, IsoWrapper)
self.assertEqual(compose.get_image_name.call_args_list,
[mock.call('x86_64', compose.variants['Everything'],
disc_type='dvd', format=None)])
@mock.patch('productmd.images.Image')
@mock.patch('pungi.util.get_mtime')
@mock.patch('pungi.util.get_file_size')
@mock.patch('pungi.wrappers.iso.IsoWrapper')
@mock.patch('os.link')
@mock.patch('pungi.wrappers.kojiwrapper.KojiWrapper')
def test_fail_crash(self, KojiWrapper, link,
IsoWrapper, get_file_size, get_mtime, ImageCls):
compose = helpers.DummyCompose(self.topdir, {
'release_name': 'Fedora',
'release_version': 'Rawhide',
'koji_profile': 'koji',
'runroot_tag': 'rrt',
'failable_deliverables': [
('^.+$', {'*': ['atomic_installer']})
],
})
pool = mock.Mock()
cfg = {
'source_repo_from': 'Everything',
'release': None,
'filename': 'Fedora-Atomic.iso',
}
koji = KojiWrapper.return_value
koji.run_runroot_cmd.side_effect = helpers.boom
t = atomic.AtomicInstallerThread(pool)
t.process((compose, compose.variants['Everything'], 'x86_64', cfg), 1)
pool.log_info.assert_has_calls([
mock.call('[BEGIN] Atomic phase for variant Everything, arch x86_64'),
mock.call('[FAIL] Atomic for variant Everything, arch x86_64, failed, but going on anyway.\n'
'BOOM')
])
@mock.patch('productmd.images.Image')
@mock.patch('pungi.util.get_mtime')
@mock.patch('pungi.util.get_file_size')
@mock.patch('pungi.wrappers.iso.IsoWrapper')
@mock.patch('os.link')
@mock.patch('pungi.wrappers.kojiwrapper.KojiWrapper')
def test_fail_runroot_fail(self, KojiWrapper, link,
IsoWrapper, get_file_size, get_mtime, ImageCls):
compose = helpers.DummyCompose(self.topdir, {
'release_name': 'Fedora',
'release_version': 'Rawhide',
'koji_profile': 'koji',
'runroot_tag': 'rrt',
'failable_deliverables': [
('^.+$', {'*': ['atomic_installer']})
],
})
pool = mock.Mock()
cfg = {
'source_repo_from': 'Everything',
'release': None,
'filename': 'Fedora-Atomic.iso',
}
koji = KojiWrapper.return_value
koji.run_runroot_cmd.return_value = {
'output': 'Failed',
'task_id': 1234,
'retcode': 1,
}
t = atomic.AtomicInstallerThread(pool)
t.process((compose, compose.variants['Everything'], 'x86_64', cfg), 1)
pool.log_info.assert_has_calls([
mock.call('[BEGIN] Atomic phase for variant Everything, arch x86_64'),
mock.call('[FAIL] Atomic for variant Everything, arch x86_64, failed, but going on anyway.\n'
'Runroot task failed: 1234. See %s/logs/x86_64/atomic/runroot.log for more details.'
% self.topdir)
])
if __name__ == '__main__':
unittest.main()

10
tests/test_lorax_wrapper.py
View File

@ -34,7 +34,11 @@ class LoraxWrapperTest(unittest.TestCase):
variant="Server", bugurl="http://example.com/",
nomacboot=True, noupgrade=True, is_final=True,
buildarch='x86_64', volid='VOLUME_ID',
buildinstallpackages=['bash', 'vim'])
buildinstallpackages=['bash', 'vim'],
add_template=['t1', 't2'],
add_arch_template=['ta1', 'ta2'],
add_template_var=['v1', 'v2'],
add_arch_template_var=['va1', 'va2'])
self.assertEqual(cmd[0], 'lorax')
self.assertItemsEqual(cmd[1:],
@ -45,6 +49,10 @@ class LoraxWrapperTest(unittest.TestCase):
'--buildarch=x86_64', '--volid=VOLUME_ID',
'--nomacboot', '--noupgrade', '--isfinal',
'--installpkgs=bash', '--installpkgs=vim',
'--add-template=t1', '--add-template=t2',
'--add-arch-template=ta1', '--add-arch-template=ta2',
'--add-template-var=v1', '--add-template-var=v2',
'--add-arch-template-var=va1', '--add-arch-template-var=va2',
'/mnt/output_dir'])

9
tests/test_util.py
View File

@ -193,5 +193,14 @@ class TestFindOldCompose(unittest.TestCase):
self.assertEqual(old, self.tmp_dir + '/Fedora-Rawhide-Base-1-20160229.0')
class TestHelpers(unittest.TestCase):
def test_process_args(self):
self.assertEqual(util.process_args('--opt={}', None), [])
self.assertEqual(util.process_args('--opt={}', []), [])
self.assertEqual(util.process_args('--opt={}', ['foo', 'bar']),
['--opt=foo', '--opt=bar'])
self.assertEqual(util.process_args('--opt={}', 'foo'), ['--opt=foo'])
if __name__ == "__main__":
unittest.main()