almalinux/pungi
14
1
Fork 0
Code Issues Pull Requests Releases Activity

Merge #123 Live images: add repo from another variant

Browse Source
This commit is contained in:
Dennis Gilmore 2016-01-20 15:37:36 +00:00
commit c77e78c4f1
3 changed files with 280 additions and 21 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

23
doc/configuration.rst
View File

@ -624,6 +624,29 @@ Example
}),
]
Live Images Settings
====================
**live_target**
(*str*) -- Koji build target for which to build the images. This gets
passed to ``koji spin-livecd``.
**live_images**
(*list*) -- Configuration for the particular image. The elements of the
list should be tuples ``(variant_uid_regex, {arch|*: config})``. The config
should be a dict with these keys:
* ``kickstart`` (*str|dict*)
* ``name`` (*str*)
* ``version`` (*str*)
* ``additional_repos`` (*list*) -- external repos specified by URL
* ``repos_from`` (*list*) -- repos from other variants
* ``specfile`` (*str*) -- for images wrapped in RPM
* ``scratch`` (*bool*) -- only RPM-wrapped images can use scratch builds,
but by default this is turned off
Image Build Settings
====================

51
pungi/phases/live_images.py
View File

@ -67,9 +67,21 @@ class LiveImagesPhase(PhaseBase):
return True
return False
def _get_extra_repos(self, arch, variant, extras):
repo = []
for extra in extras:
v = self.compose.variants.get(extra)
if not v:
raise RuntimeError(
'There is no variant %s to get repo from when building live image for %s.'
% (extra, variant.uid))
repo.append(translate_path(
self.compose, self.compose.paths.compose.repository(arch, v, create_dir=False)))
return repo
def run(self):
symlink_isos_to = self.compose.conf.get("symlink_isos_to", None)
iso = IsoWrapper()
commands = []
for variant in self.compose.variants.values():
@ -87,22 +99,21 @@ class LiveImagesPhase(PhaseBase):
cmd = {
"name": None,
"version": None,
"arch": arch,
"variant": variant,
"iso_path": None,
"wrapped_rpms_path": iso_dir,
"build_arch": arch,
"ks_file": ks_file,
"specfile": None,
"scratch": False,
"cmd": [],
"label": "", # currently not used
}
cmd["repos"] = [translate_path(self.compose, self.compose.paths.compose.repository(arch, variant))]
cmd["repos"] = [translate_path(
self.compose, self.compose.paths.compose.repository(arch, variant, create_dir=False))]
# additional repos
data = get_arch_variant_data(self.compose.conf, "live_images", arch, variant)
cmd["repos"].extend(data[0].get("additional_repos", []))
cmd['repos'].extend(self._get_extra_repos(arch, variant, data[0].get('repos_from', [])))
# Explicit name and version
cmd["name"] = data[0].get("name", None)
@ -133,17 +144,7 @@ class LiveImagesPhase(PhaseBase):
self.compose.log_warning("Skipping creating live image, it already exists: %s" % iso_path)
continue
cmd["iso_path"] = iso_path
iso_name = os.path.basename(iso_path)
# Additional commands
chdir_cmd = "cd %s" % pipes.quote(iso_dir)
cmd["cmd"].append(chdir_cmd)
# create iso manifest
cmd["cmd"].append(iso.get_manifest_cmd(iso_name))
cmd["cmd"] = " && ".join(cmd["cmd"])
commands.append((cmd, variant, arch))
for (cmd, variant, arch) in commands:
@ -170,7 +171,7 @@ class CreateLiveImageThread(WorkerThread):
def process(self, item, num):
compose, cmd, variant, arch = item
try:
self.worker(compose, cmd, num)
self.worker(compose, cmd, variant, arch, num)
except:
if not compose.can_fail(variant, arch, 'live'):
raise
@ -179,10 +180,10 @@ class CreateLiveImageThread(WorkerThread):
% (variant.uid, arch))
self.pool.log_info(msg)
def worker(self, compose, cmd, num):
log_file = compose.paths.log.log_file(cmd["arch"], "createiso-%s" % os.path.basename(cmd["iso_path"]))
def worker(self, compose, cmd, variant, arch, num):
log_file = compose.paths.log.log_file(arch, "createiso-%s" % os.path.basename(cmd["iso_path"]))
msg = "Creating ISO (arch: %s, variant: %s): %s" % (cmd["arch"], cmd["variant"], os.path.basename(cmd["iso_path"]))
msg = "Creating ISO (arch: %s, variant: %s): %s" % (arch, variant, os.path.basename(cmd["iso_path"]))
self.pool.log_info("[BEGIN] %s" % msg)
koji_wrapper = KojiWrapper(compose.conf["koji_profile"])
@ -217,11 +218,19 @@ class CreateLiveImageThread(WorkerThread):
for rpm_path in rpm_paths:
shutil.copy2(rpm_path, cmd["wrapped_rpms_path"])
# write manifest
run(cmd["cmd"])
self._write_manifest(cmd['iso_path'])
self.pool.log_info("[DONE ] %s" % msg)
def _write_manifest(self, iso_path):
"""Generate manifest for ISO at given path.
:param iso_path: (str) absolute path to the ISO
"""
dir, filename = os.path.split(iso_path)
iso = IsoWrapper()
run("cd %s && %s" % (pipes.quote(dir), iso.get_manifest_cmd(filename)))
def get_ks_in(compose, arch, variant):
data = get_arch_variant_data(compose.conf, "live_images", arch, variant)

227
tests/test_liveimagesphase.py Executable file
View File

@ -0,0 +1,227 @@
#!/usr/bin/env python2
# -*- coding: utf-8 -*-
import unittest
import mock
import os
import sys
sys.path.insert(0, os.path.join(os.path.dirname(__file__), ".."))
from pungi.phases.live_images import LiveImagesPhase, CreateLiveImageThread
from pungi.util import get_arch_variant_data
class _DummyCompose(object):
def __init__(self, config):
self.compose_id = 'Test-20151203.0.t'
self.conf = config
self.paths = mock.Mock(
compose=mock.Mock(
repository=mock.Mock(
side_effect=lambda arch, variant, create_dir=False: os.path.join('/repo', arch, variant.uid)
),
iso_dir=mock.Mock(
side_effect=lambda arch, variant, symlink_to: os.path.join(
'/iso_dir', arch, variant.uid
)
),
iso_path=mock.Mock(
side_effect=lambda arch, variant, filename, symlink_to: os.path.join(
'/iso_dir', arch, variant.uid, filename
)
)
),
log=mock.Mock(
log_file=mock.Mock(return_value='/a/b/log/log_file')
)
)
self._logger = mock.Mock()
self.variants = {
'Server': mock.Mock(uid='Server', arches=['x86_64', 'amd64']),
'Client': mock.Mock(uid='Client', arches=['amd64']),
'Everything': mock.Mock(uid='Everything', arches=['x86_64', 'amd64']),
}
self.log_error = mock.Mock()
self.get_image_name = mock.Mock(return_value='image-name')
def get_arches(self):
return ['x86_64', 'amd64']
def get_variants(self, arch=None, types=None):
return [v for v in self.variants.values() if not arch or arch in v.arches]
def can_fail(self, variant, arch, deliverable):
failable = get_arch_variant_data(self.conf, 'failable_deliverables', arch, variant)
return deliverable in failable
class TestLiveImagesPhase(unittest.TestCase):
@mock.patch('pungi.phases.live_images.ThreadPool')
@mock.patch('pungi.phases.live_images.get_ks_in')
@mock.patch('pungi.phases.live_images.tweak_ks')
def test_image_build(self, tweak_ks, get_ks_in, ThreadPool):
compose = _DummyCompose({
'live_images': [
('^Client$', {
'amd64': {
'additional_repos': ['http://example.com/repo/'],
'repos_from': ['Everything'],
}
})
],
})
get_ks_in.side_effect = (lambda compose, arch, variant:
None if variant.uid != 'Client' or arch != 'amd64' else '/path/to/ks_in')
tweak_ks.return_value = '/path/to/ks_file'
phase = LiveImagesPhase(compose)
phase.run()
# assert at least one thread was started
self.assertTrue(phase.pool.add.called)
self.maxDiff = None
self.assertItemsEqual(phase.pool.queue_put.mock_calls,
[mock.call((compose,
{'ks_file': '/path/to/ks_file',
'build_arch': 'amd64',
'wrapped_rpms_path': '/iso_dir/amd64/Client',
'scratch': False,
'repos': ['/repo/amd64/Client',
'http://example.com/repo/',
'/repo/amd64/Everything'],
'label': '',
'name': None,
'iso_path': '/iso_dir/amd64/Client/image-name',
'version': None,
'specfile': None},
compose.variants['Client'],
'amd64'))])
class TestCreateLiveImageThread(unittest.TestCase):
@mock.patch('shutil.copy2')
@mock.patch('pungi.phases.live_images.run')
@mock.patch('pungi.phases.live_images.KojiWrapper')
def test_process(self, KojiWrapper, run, copy2):
compose = _DummyCompose({'koji_profile': 'koji'})
pool = mock.Mock()
cmd = {
'ks_file': '/path/to/ks_file',
'build_arch': 'amd64',
'wrapped_rpms_path': '/iso_dir/amd64/Client',
'scratch': False,
'repos': ['/repo/amd64/Client',
'http://example.com/repo/',
'/repo/amd64/Everything'],
'label': '',
'name': None,
'iso_path': '/iso_dir/amd64/Client/image-name',
'version': None,
'specfile': None
}
koji_wrapper = KojiWrapper.return_value
koji_wrapper.get_create_image_cmd.return_value = 'koji spin-livecd ...'
koji_wrapper.run_create_image_cmd.return_value = {
'retcode': 0,
'output': 'some output',
'task_id': 123
}
koji_wrapper.get_image_path.return_value = ['/path/to/image']
t = CreateLiveImageThread(pool)
with mock.patch('time.sleep'):
t.process((compose, cmd, compose.variants['Client'], 'amd64'), 1)
self.assertEqual(koji_wrapper.run_create_image_cmd.mock_calls,
[mock.call('koji spin-livecd ...', log_file='/a/b/log/log_file')])
self.assertEqual(koji_wrapper.get_image_path.mock_calls, [mock.call(123)])
self.assertEqual(copy2.mock_calls,
[mock.call('/path/to/image', '/iso_dir/amd64/Client/image-name')])
write_manifest_cmd = ' && '.join([
'cd /iso_dir/amd64/Client',
'isoinfo -R -f -i image-name | grep -v \'/TRANS.TBL$\' | sort >> image-name.manifest'
])
self.assertEqual(run.mock_calls, [mock.call(write_manifest_cmd)])
@mock.patch('shutil.copy2')
@mock.patch('pungi.phases.live_images.run')
@mock.patch('pungi.phases.live_images.KojiWrapper')
def test_process_handles_fail(self, KojiWrapper, run, copy2):
compose = _DummyCompose({
'koji_profile': 'koji',
'failable_deliverables': [('^.+$', {'*': ['live']})],
})
pool = mock.Mock()
cmd = {
'ks_file': '/path/to/ks_file',
'build_arch': 'amd64',
'wrapped_rpms_path': '/iso_dir/amd64/Client',
'scratch': False,
'repos': ['/repo/amd64/Client',
'http://example.com/repo/',
'/repo/amd64/Everything'],
'label': '',
'name': None,
'iso_path': '/iso_dir/amd64/Client/image-name',
'version': None,
'specfile': None
}
koji_wrapper = KojiWrapper.return_value
koji_wrapper.get_create_image_cmd.return_value = 'koji spin-livecd ...'
koji_wrapper.run_create_image_cmd.return_value = {
'retcode': 1,
'output': 'some output',
'task_id': 123
}
t = CreateLiveImageThread(pool)
with mock.patch('time.sleep'):
t.process((compose, cmd, compose.variants['Client'], 'amd64'), 1)
@mock.patch('shutil.copy2')
@mock.patch('pungi.phases.live_images.run')
@mock.patch('pungi.phases.live_images.KojiWrapper')
def test_process_handles_exception(self, KojiWrapper, run, copy2):
compose = _DummyCompose({
'koji_profile': 'koji',
'failable_deliverables': [('^.+$', {'*': ['live']})],
})
pool = mock.Mock()
cmd = {
'ks_file': '/path/to/ks_file',
'build_arch': 'amd64',
'wrapped_rpms_path': '/iso_dir/amd64/Client',
'scratch': False,
'repos': ['/repo/amd64/Client',
'http://example.com/repo/',
'/repo/amd64/Everything'],
'label': '',
'name': None,
'iso_path': '/iso_dir/amd64/Client/image-name',
'version': None,
'specfile': None
}
def boom(*args, **kwargs):
raise RuntimeError('BOOM')
koji_wrapper = KojiWrapper.return_value
koji_wrapper.get_create_image_cmd.side_effect = boom
t = CreateLiveImageThread(pool)
with mock.patch('time.sleep'):
t.process((compose, cmd, compose.variants['Client'], 'amd64'), 1)
if __name__ == "__main__":
unittest.main()