diff --git a/bin/pungi-config-validate b/bin/pungi-config-validate index 815d14fe..a2bc5814 100755 --- a/bin/pungi-config-validate +++ b/bin/pungi-config-validate @@ -107,6 +107,7 @@ def run(config, topdir, has_old): pungi.phases.OSTreePhase(compose), pungi.phases.ProductimgPhase(compose, pkgset_phase), pungi.phases.CreateisoPhase(compose, buildinstall_phase), + pungi.phases.ExtraIsosPhase(compose), pungi.phases.LiveImagesPhase(compose), pungi.phases.LiveMediaPhase(compose), pungi.phases.ImageBuildPhase(compose), diff --git a/bin/pungi-koji b/bin/pungi-koji index 0d1b892e..b3b49de5 100755 --- a/bin/pungi-koji +++ b/bin/pungi-koji @@ -300,6 +300,7 @@ def run_compose(compose, create_latest_link=True, latest_link_status=None): ostree_phase = pungi.phases.OSTreePhase(compose) productimg_phase = pungi.phases.ProductimgPhase(compose, pkgset_phase) createiso_phase = pungi.phases.CreateisoPhase(compose, buildinstall_phase) + extra_isos_phase = pungi.phases.ExtraIsosPhase(compose) liveimages_phase = pungi.phases.LiveImagesPhase(compose) livemedia_phase = pungi.phases.LiveMediaPhase(compose) image_build_phase = pungi.phases.ImageBuildPhase(compose) @@ -313,7 +314,7 @@ def run_compose(compose, create_latest_link=True, latest_link_status=None): extrafiles_phase, createiso_phase, liveimages_phase, livemedia_phase, image_build_phase, image_checksum_phase, test_phase, ostree_phase, ostree_installer_phase, - osbs_phase): + extra_isos_phase, osbs_phase): if phase.skip(): continue try: @@ -402,6 +403,7 @@ def run_compose(compose, create_latest_link=True, latest_link_status=None): # Start all phases for image artifacts compose_images_schema = ( createiso_phase, + extra_isos_phase, liveimages_phase, image_build_phase, livemedia_phase, diff --git a/doc/_static/phases.png b/doc/_static/phases.png index a6d40383..bca203e0 100644 Binary files a/doc/_static/phases.png and b/doc/_static/phases.png differ diff --git a/doc/_static/phases.svg b/doc/_static/phases.svg index 66184b15..7bbc5e1f 100644 --- a/doc/_static/phases.svg +++ b/doc/_static/phases.svg @@ -9,12 +9,12 @@ xmlns="http://www.w3.org/2000/svg" xmlns:sodipodi="http://sodipodi.sourceforge.net/DTD/sodipodi-0.dtd" xmlns:inkscape="http://www.inkscape.org/namespaces/inkscape" - width="839.33331" - height="220.33334" - viewBox="0 0 839.33334 220.33335" + width="771.66455" + height="221.50018" + viewBox="0 0 771.66458 221.50019" id="svg2" version="1.1" - inkscape:version="0.91 r13725" + inkscape:version="0.92.3 (2405546, 2018-03-11)" sodipodi:docname="phases.svg" inkscape:export-filename="/home/lsedlar/repos/pungi/doc/_static/phases.png" inkscape:export-xdpi="90" @@ -26,21 +26,25 @@ borderopacity="1.0" inkscape:pageopacity="1" inkscape:pageshadow="2" - inkscape:zoom="1.6532468" - inkscape:cx="337.4932" - inkscape:cy="70.825454" + inkscape:zoom="1.169022" + inkscape:cx="396.63448" + inkscape:cy="97.894202" inkscape:document-units="px" inkscape:current-layer="layer1" showgrid="false" inkscape:window-width="1920" - inkscape:window-height="1020" + inkscape:window-height="1016" inkscape:window-x="1920" - inkscape:window-y="31" + inkscape:window-y="27" inkscape:window-maximized="1" units="px" inkscape:document-rotation="0" showguides="true" - inkscape:guide-bbox="true" /> + inkscape:guide-bbox="true" + fit-margin-top="0" + fit-margin-left="0" + fit-margin-right="0" + fit-margin-bottom="0" /> image/svg+xml - + @@ -115,8 +119,7 @@ style="font-style:normal;font-variant:normal;font-weight:normal;font-stretch:normal;line-height:0%;font-family:sans-serif;-inkscape-font-specification:'sans-serif, Normal';text-align:start;letter-spacing:0px;word-spacing:0px;writing-mode:lr-tb;text-anchor:start;fill:#000000;fill-opacity:1;stroke:none;stroke-width:1px;stroke-linecap:butt;stroke-linejoin:miter;stroke-opacity:1" x="556.95709" y="971.54041" - id="text3384-0" - sodipodi:linespacing="0%"> OSTreeInstaller + y="1039.4121" + style="font-size:11.99999714px;line-height:0">OSTreeInstaller - - - + + Createiso - - - - Createiso + + + + LiveImages - - - - LiveImages + + + + ImageBuild - - - - ImageBuild + + + + LiveMedia - - - - LiveMedia + + + + OSBS - + y="1065.7078">OSBS + + + + ExtraIsos diff --git a/doc/configuration.rst b/doc/configuration.rst index d28ffdca..c8135093 100644 --- a/doc/configuration.rst +++ b/doc/configuration.rst @@ -1450,6 +1450,85 @@ Example config } +Extra ISOs +========== + +Create an ISO image that contains packages from multiple variants. Such ISO +always belongs to one variant, and will be stored in ISO directory of that +variant. + +The ISO will be bootable if buildinstall phase runs for the parent variant. It +will reuse boot configuration from that variant. + +**extra_isos** + (*dict*) -- a mapping from variant UID regex to a list of configuration + blocks. + + * ``include_variants`` -- (*list*) list of variant UIDs from which content + should be added to the ISO; the variant of this image is added + automatically. + + Rest of configuration keys is optional. + + * ``filename`` -- (*str*) template for naming the image. In addition to the + regular placeholders ``filename`` is available with the name generated + using ``image_name_format`` option. + + * ``volid`` -- (*str*) template for generating volume ID. Again ``volid`` + placeholder can be used similarly as for file name. This can also be a + list of templates that will be tried sequentially until one generates a + volume ID that fits into 32 character limit. + + * ``extra_files`` -- (*list*) a list of :ref:`scm_dict ` + objects. These files will be put in the top level directory of the image. + + * ``arches`` -- (*list*) a list of architectures for which to build this + image. By default all arches from the variant will be used. This option + can be used to limit them. + + * ``failable_arches`` -- (*list*) a list of architectures for which the + image can fail to be generated and not fail the entire compose. + + * ``skip_src`` -- (*bool*) allows to disable creating an image with source + packages. + +Example config +-------------- +:: + + extra_isos = { + 'Server': [{ + # Will generate foo-DP-1.0-20180510.t.43-Server-x86_64-dvd1.iso + 'filename': 'foo-{filename}', + 'volid': 'foo-{arch}', + + 'extra_files': [{ + 'scm': 'git', + 'repo': 'https://pagure.io/pungi.git', + 'file': 'setup.py' + }], + + 'include_variants': ['Client'] + }] + } + # This should create image with the following layout: + # . + # ├── Client + # │   ├── Packages + # │   │ ├── a + # │   │ └── b + # │   └── repodata + # ├── Server + # │   ├── extra_files.json # extra file from Server + # │   ├── LICENSE # extra file from Server + # │   ├── Packages + # │   │ ├── a + # │   │ └── b + # │   └── repodata + # └── setup.py + + + Media Checksums Settings ======================== diff --git a/doc/phases.rst b/doc/phases.rst index 61704406..6b04932a 100644 --- a/doc/phases.rst +++ b/doc/phases.rst @@ -99,6 +99,15 @@ packages fit on a single image. There can also be images with source repositories. These are never bootable. +ExtraIsos +--------- + +This phase is very similar to ``createiso``, except it combines content from +multiple variants onto a single image. Packages, repodata and extra files from +each configured variant are put into a subdirectory. Additional extra files can +be put into top level of the image. The image will be bootable if the main +variant is bootable. + LiveImages, LiveMedia --------------------- diff --git a/pungi/checks.py b/pungi/checks.py index 335034f6..7d5e01c4 100644 --- a/pungi/checks.py +++ b/pungi/checks.py @@ -863,6 +863,47 @@ def make_schema(): "$ref": "#/definitions/list_of_strings" }), + "extra_isos": { + "type": "object", + "patternProperties": { + # Warning: this pattern is a variant uid regex, but the + # format does not let us validate it as there is no regular + # expression to describe all regular expressions. + ".+": { + "type": "array", + "items": { + "type": "object", + "properties": { + "include_variants": {"$ref": "#/definitions/strings"}, + "extra_files": _one_or_list({ + "type": "object", + "properties": { + "scm": {"type": "string"}, + "repo": {"type": "string"}, + "branch": {"$ref": "#/definitions/optional_string"}, + "file": {"$ref": "#/definitions/strings"}, + "target": {"type": "string"}, + }, + "additionalProperties": False, + }), + "filename": {"type": "string"}, + "volid": {"$ref": "#/definitions/strings"}, + "arches": {"$ref": "#/definitions/list_of_strings"}, + "failable_arches": { + "$ref": "#/definitions/list_of_strings" + }, + "skip_src": { + "type": "boolean", + "default": False, + }, + }, + "required": ["include_variants"], + "additionalProperties": False + } + } + } + }, + "live_media": { "type": "object", "patternProperties": { diff --git a/pungi/paths.py b/pungi/paths.py index cfed2903..bb07005b 100644 --- a/pungi/paths.py +++ b/pungi/paths.py @@ -264,6 +264,18 @@ class WorkPaths(object): makedirs(path) return path + def extra_iso_extra_files_dir(self, arch, variant, create_dir=True): + """ + Examples: + work/x86_64/Server/extra-iso-extra-files + """ + if arch == "global": + raise RuntimeError("Global extra files dir makes no sense.") + path = os.path.join(self.topdir(arch, create_dir=create_dir), variant.uid, "extra-iso-extra-files") + if create_dir: + makedirs(path) + return path + def repo_package_list(self, arch, variant, pkg_type=None, create_dir=True): """ Examples: diff --git a/pungi/phases/__init__.py b/pungi/phases/__init__.py index 095a3136..0ed3e564 100644 --- a/pungi/phases/__init__.py +++ b/pungi/phases/__init__.py @@ -25,6 +25,7 @@ from .product_img import ProductimgPhase # noqa from .buildinstall import BuildinstallPhase # noqa from .extra_files import ExtraFilesPhase # noqa from .createiso import CreateisoPhase # noqa +from .extra_isos import ExtraIsosPhase # noqa from .live_images import LiveImagesPhase # noqa from .image_build import ImageBuildPhase # noqa from .test import TestPhase # noqa diff --git a/pungi/phases/createiso.py b/pungi/phases/createiso.py index 6e7a053c..29e10e6f 100644 --- a/pungi/phases/createiso.py +++ b/pungi/phases/createiso.py @@ -395,13 +395,7 @@ def prepare_iso(compose, arch, variant, disc_num=1, disc_count=None, split_iso_d if i in ti.checksums.checksums.keys(): del ti.checksums.checksums[i] - # make a copy of isolinux/isolinux.bin, images/boot.img - they get modified when mkisofs is called - for i in ("isolinux/isolinux.bin", "images/boot.img"): - src_path = os.path.join(tree_dir, i) - dst_path = os.path.join(iso_dir, i) - if os.path.exists(src_path): - makedirs(os.path.dirname(dst_path)) - shutil.copy2(src_path, dst_path) + copy_boot_images(tree_dir, iso_dir) if disc_count > 1: # remove repodata/repomd.xml from checksums, create a new one later @@ -454,3 +448,15 @@ def prepare_iso(compose, arch, variant, disc_num=1, disc_count=None, split_iso_d gp = "%s-graft-points" % iso_dir iso.write_graft_points(gp, data, exclude=["*/lost+found", "*/boot.iso"]) return gp + + +def copy_boot_images(src, dest): + """When mkisofs is called it tries to modify isolinux/isolinux.bin and + images/boot.img. Therefore we need to make copies of them. + """ + for i in ("isolinux/isolinux.bin", "images/boot.img"): + src_path = os.path.join(src, i) + dst_path = os.path.join(dest, i) + if os.path.exists(src_path): + makedirs(os.path.dirname(dst_path)) + shutil.copy2(src_path, dst_path) diff --git a/pungi/phases/extra_isos.py b/pungi/phases/extra_isos.py new file mode 100644 index 00000000..574fe9cf --- /dev/null +++ b/pungi/phases/extra_isos.py @@ -0,0 +1,201 @@ +# -*- coding: utf-8 -*- + + +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; version 2 of the License. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Library General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, see . + +import os + +from kobo.shortcuts import force_list +from kobo.threads import ThreadPool, WorkerThread + +from pungi import createiso +from pungi.phases.base import ConfigGuardedPhase, PhaseBase, PhaseLoggerMixin +from pungi.phases.createiso import (add_iso_to_metadata, copy_boot_images, + run_createiso_command) +from pungi.util import failable, get_format_substs, get_variant_data, get_volid +from pungi.wrappers import iso +from pungi.wrappers.scm import get_dir_from_scm, get_file_from_scm + + +class ExtraIsosPhase(PhaseLoggerMixin, ConfigGuardedPhase, PhaseBase): + name = "extra_isos" + + def __init__(self, compose): + super(ExtraIsosPhase, self).__init__(compose) + self.pool = ThreadPool(logger=self.logger) + + def validate(self): + for variant in self.compose.get_variants(types=['variant']): + for config in get_variant_data(self.compose.conf, self.name, variant): + extra_arches = set(config.get('arches', [])) - set(variant.arches) + if extra_arches: + self.compose.log_warning( + 'Extra iso config for %s mentions non-existing arches: %s' + % (variant, ', '.join(sorted(extra_arches)))) + + def run(self): + commands = [] + + for variant in self.compose.get_variants(types=['variant']): + for config in get_variant_data(self.compose.conf, self.name, variant): + arches = set(variant.arches) + if config.get('arches'): + arches &= set(config['arches']) + if not config['skip_src']: + arches.add('src') + for arch in sorted(arches): + commands.append((config, variant, arch)) + + for (config, variant, arch) in commands: + self.pool.add(ExtraIsosThread(self.pool)) + self.pool.queue_put((self.compose, config, variant, arch)) + + self.pool.start() + + +class ExtraIsosThread(WorkerThread): + def process(self, item, num): + self.num = num + compose, config, variant, arch = item + can_fail = arch in config.get('failable_arches', []) + with failable(compose, can_fail, variant, arch, 'extra_iso', logger=self.pool._logger): + self.worker(compose, config, variant, arch) + + def worker(self, compose, config, variant, arch): + filename = get_filename(compose, variant, arch, config.get('filename')) + volid = get_volume_id(compose, variant, arch, config.get('volid', [])) + iso_dir = compose.paths.compose.iso_dir(arch, variant) + iso_path = os.path.join(iso_dir, filename) + + msg = "Creating ISO (arch: %s, variant: %s): %s" % (arch, variant, filename) + self.pool.log_info("[BEGIN] %s" % msg) + + get_extra_files(compose, variant, arch, config.get('extra_files', [])) + + bootable = arch != "src" and compose.conf['bootable'] + + graft_points = get_iso_contents(compose, variant, arch, + config['include_variants'], + filename, bootable) + + opts = createiso.CreateIsoOpts( + output_dir=iso_dir, + iso_name=filename, + volid=volid, + graft_points=graft_points, + arch=arch, + supported=compose.supported, + ) + + if bootable: + opts = opts._replace(buildinstall_method=compose.conf['buildinstall_method']) + + script_file = os.path.join(compose.paths.work.tmp_dir(arch, variant), + 'extraiso-%s.sh' % filename) + with open(script_file, 'w') as f: + createiso.write_script(opts, f) + + run_createiso_command(compose.conf["runroot"], self.num, compose, bootable, arch, + ['bash', script_file], [compose.topdir], + log_file=compose.paths.log.log_file( + arch, "extraiso-%s" % os.path.basename(iso_path)), + with_jigdo=False) + + add_iso_to_metadata(compose, variant, arch, iso_path, bootable, 1, 1) + + self.pool.log_info("[DONE ] %s" % msg) + + +def get_extra_files(compose, variant, arch, extra_files): + """Clone the configured files into a directory from where they can be + included in the ISO. + """ + extra_files_dir = compose.paths.work.extra_iso_extra_files_dir(arch, variant) + for scm_dict in extra_files: + getter = get_file_from_scm if 'file' in scm_dict else get_dir_from_scm + target_path = os.path.join(extra_files_dir, scm_dict.get('target', '').lstrip('/')) + getter(scm_dict, target_path, logger=compose._logger) + + +def get_iso_contents(compose, variant, arch, include_variants, filename, bootable): + """Find all files that should be on the ISO. For bootable image we start + with the boot configuration. Then for each variant we add packages, + repodata and extra files. Finally we add top-level extra files. + """ + iso_dir = compose.paths.work.iso_dir(arch, filename) + + files = {} + if bootable: + buildinstall_dir = compose.paths.work.buildinstall_dir(arch, create_dir=False) + if compose.conf['buildinstall_method'] == 'lorax': + buildinstall_dir = os.path.join(buildinstall_dir, variant.uid) + + copy_boot_images(buildinstall_dir, iso_dir) + files = iso.get_graft_points([buildinstall_dir, iso_dir]) + + variants = [variant.uid] + include_variants + for variant_uid in variants: + var = compose.all_variants[variant_uid] + + # Get packages... + package_dir = compose.paths.compose.packages(arch, var) + for k, v in iso.get_graft_points([package_dir]).items(): + files[os.path.join(var.uid, 'Packages', k)] = v + + # Get repodata... + tree_dir = compose.paths.compose.repository(arch, var) + repo_dir = os.path.join(tree_dir, 'repodata') + for k, v in iso.get_graft_points([repo_dir]).items(): + files[os.path.join(var.uid, 'repodata', k)] = v + + # Get extra files... + extra_files_dir = compose.paths.work.extra_files_dir(arch, var) + for k, v in iso.get_graft_points([extra_files_dir]).items(): + files[os.path.join(var.uid, k)] = v + + # Add extra files specific for the ISO + extra_files_dir = compose.paths.work.extra_iso_extra_files_dir(arch, variant) + files.update(iso.get_graft_points([extra_files_dir])) + + gp = "%s-graft-points" % iso_dir + iso.write_graft_points(gp, files, exclude=["*/lost+found", "*/boot.iso"]) + return gp + + +def get_filename(compose, variant, arch, format): + disc_type = compose.conf['disc_types'].get('dvd', 'dvd') + base_filename = compose.get_image_name( + arch, variant, disc_type=disc_type, disc_num=1) + if not format: + return base_filename + kwargs = { + 'arch': arch, + 'disc_type': disc_type, + 'disc_num': 1, + 'suffix': '.iso', + 'filename': base_filename, + 'variant': variant, + } + args = get_format_substs(compose, **kwargs) + try: + return (format % args).format(**args) + except KeyError as err: + raise RuntimeError('Failed to create image name: unknown format element: %s' % err) + + +def get_volume_id(compose, variant, arch, formats): + disc_type = compose.conf['disc_types'].get('dvd', 'dvd') + # Get volume ID for regular ISO so that we can substitute it in. + volid = get_volid(compose, arch, variant, disc_type=disc_type) + return get_volid(compose, arch, variant, disc_type=disc_type, + formats=force_list(formats), volid=volid) diff --git a/pungi/util.py b/pungi/util.py index e2b07de6..ea0f6e48 100644 --- a/pungi/util.py +++ b/pungi/util.py @@ -330,7 +330,8 @@ def _apply_substitutions(compose, volid): return volid -def get_volid(compose, arch, variant=None, escape_spaces=False, disc_type=False): +def get_volid(compose, arch, variant=None, escape_spaces=False, disc_type=False, + formats=None, **kwargs): """Get ISO volume ID for arch and variant""" if variant and variant.type == "addon": # addons are part of parent variant media @@ -359,9 +360,10 @@ def get_volid(compose, arch, variant=None, escape_spaces=False, disc_type=False) all_products = layered_products + products else: all_products = products + formats = formats or all_products tried = set() - for i in all_products: + for i in formats: if not variant_uid and "%(variant)s" in i: continue try: @@ -372,7 +374,8 @@ def get_volid(compose, arch, variant=None, escape_spaces=False, disc_type=False) arch=arch, disc_type=disc_type or '', base_product_short=base_product_short, - base_product_version=base_product_version) + base_product_version=base_product_version, + **kwargs) volid = (i % args).format(**args) except KeyError as err: raise RuntimeError('Failed to create volume id: unknown format element: %s' % err) diff --git a/tests/data/dummy-pungi.conf b/tests/data/dummy-pungi.conf index 2d1f21f1..120c44f1 100644 --- a/tests/data/dummy-pungi.conf +++ b/tests/data/dummy-pungi.conf @@ -114,4 +114,11 @@ createiso_skip = [ }), ] +extra_isos = { + '^Server$': [{ + 'include_variants': ['Client'] + 'filename': 'extra-{filename}', + }] +} + create_jigdo = False diff --git a/tests/test_extra_isos_phase.py b/tests/test_extra_isos_phase.py new file mode 100644 index 00000000..420d0ffe --- /dev/null +++ b/tests/test_extra_isos_phase.py @@ -0,0 +1,529 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- + + +try: + import unittest2 as unittest +except ImportError: + import unittest +import mock + +import os +import sys + +sys.path.insert(0, os.path.join(os.path.dirname(__file__), '..')) + +from tests import helpers +from pungi.phases import extra_isos + + +@mock.patch('pungi.phases.extra_isos.ThreadPool') +class ExtraIsosPhaseTest(helpers.PungiTestCase): + + def test_logs_extra_arches(self, ThreadPool): + cfg = { + 'include_variants': ['Client'], + 'arches': ['x86_64', 'ppc64le', 'aarch64'], + } + compose = helpers.DummyCompose(self.topdir, { + 'extra_isos': { + '^Server$': [cfg] + } + }) + + phase = extra_isos.ExtraIsosPhase(compose) + phase.validate() + + self.assertEqual(len(compose.log_warning.call_args_list), 1) + + def test_one_task_for_each_arch(self, ThreadPool): + cfg = { + 'include_variants': ['Client'], + } + compose = helpers.DummyCompose(self.topdir, { + 'extra_isos': { + '^Server$': [cfg] + } + }) + + phase = extra_isos.ExtraIsosPhase(compose) + phase.run() + + self.assertEqual(len(ThreadPool.return_value.add.call_args_list), 3) + self.assertItemsEqual( + ThreadPool.return_value.queue_put.call_args_list, + [mock.call((compose, cfg, compose.variants['Server'], 'x86_64')), + mock.call((compose, cfg, compose.variants['Server'], 'amd64')), + mock.call((compose, cfg, compose.variants['Server'], 'src'))] + ) + + def test_filter_arches(self, ThreadPool): + cfg = { + 'include_variants': ['Client'], + 'arches': ['x86_64'], + } + compose = helpers.DummyCompose(self.topdir, { + 'extra_isos': { + '^Server$': [cfg] + } + }) + + phase = extra_isos.ExtraIsosPhase(compose) + phase.run() + + self.assertEqual(len(ThreadPool.return_value.add.call_args_list), 2) + self.assertItemsEqual( + ThreadPool.return_value.queue_put.call_args_list, + [mock.call((compose, cfg, compose.variants['Server'], 'x86_64')), + mock.call((compose, cfg, compose.variants['Server'], 'src'))] + ) + + def test_skip_source(self, ThreadPool): + cfg = { + 'include_variants': ['Client'], + 'skip_src': True, + } + compose = helpers.DummyCompose(self.topdir, { + 'extra_isos': { + '^Server$': [cfg] + } + }) + + phase = extra_isos.ExtraIsosPhase(compose) + phase.run() + + self.assertEqual(len(ThreadPool.return_value.add.call_args_list), 2) + self.assertItemsEqual( + ThreadPool.return_value.queue_put.call_args_list, + [mock.call((compose, cfg, compose.variants['Server'], 'x86_64')), + mock.call((compose, cfg, compose.variants['Server'], 'amd64'))] + ) + + +@mock.patch('pungi.phases.extra_isos.get_volume_id') +@mock.patch('pungi.phases.extra_isos.get_filename') +@mock.patch('pungi.phases.extra_isos.get_iso_contents') +@mock.patch('pungi.phases.extra_isos.get_extra_files') +@mock.patch('pungi.phases.extra_isos.run_createiso_command') +@mock.patch('pungi.phases.extra_isos.add_iso_to_metadata') +class ExtraIsosThreadTest(helpers.PungiTestCase): + + def test_binary_bootable_image(self, aitm, rcc, gef, gic, gfn, gvi): + compose = helpers.DummyCompose(self.topdir, { + 'bootable': True, + 'buildinstall_method': 'lorax' + }) + server = compose.variants['Server'] + cfg = { + 'include_variants': ['Client'], + } + + gfn.return_value = 'my.iso' + gvi.return_value = 'my volume id' + gic.return_value = '/tmp/iso-graft-points' + + t = extra_isos.ExtraIsosThread(mock.Mock()) + with mock.patch('time.sleep'): + t.process((compose, cfg, server, 'x86_64'), 1) + + self.assertEqual(gfn.call_args_list, + [mock.call(compose, server, 'x86_64', None)]) + self.assertEqual(gvi.call_args_list, + [mock.call(compose, server, 'x86_64', [])]) + self.assertEqual(gef.call_args_list, + [mock.call(compose, server, 'x86_64', [])]) + self.assertEqual(gic.call_args_list, + [mock.call(compose, server, 'x86_64', ['Client'], 'my.iso', True)]) + self.assertEqual( + rcc.call_args_list, + [mock.call(False, 1, compose, True, 'x86_64', + ['bash', os.path.join(self.topdir, 'work/x86_64/tmp-Server/extraiso-my.iso.sh')], + [self.topdir], + log_file=os.path.join(self.topdir, 'logs/x86_64/extraiso-my.iso.x86_64.log'), + with_jigdo=False)] + + ) + self.assertEqual( + aitm.call_args_list, + [mock.call(compose, server, 'x86_64', + os.path.join(self.topdir, 'compose/Server/x86_64/iso/my.iso'), + True, 1, 1)] + ) + + def test_binary_image_custom_naming(self, aitm, rcc, gef, gic, gfn, gvi): + compose = helpers.DummyCompose(self.topdir, {}) + server = compose.variants['Server'] + cfg = { + 'include_variants': ['Client'], + 'filename': 'fn', + 'volid': ['v1', 'v2'], + } + + gfn.return_value = 'my.iso' + gvi.return_value = 'my volume id' + gic.return_value = '/tmp/iso-graft-points' + + t = extra_isos.ExtraIsosThread(mock.Mock()) + with mock.patch('time.sleep'): + t.process((compose, cfg, server, 'x86_64'), 1) + + self.assertEqual(gfn.call_args_list, + [mock.call(compose, server, 'x86_64', 'fn')]) + self.assertEqual(gvi.call_args_list, + [mock.call(compose, server, 'x86_64', ['v1', 'v2'])]) + self.assertEqual(gef.call_args_list, + [mock.call(compose, server, 'x86_64', [])]) + self.assertEqual(gic.call_args_list, + [mock.call(compose, server, 'x86_64', ['Client'], 'my.iso', False)]) + self.assertEqual( + rcc.call_args_list, + [mock.call(False, 1, compose, False, 'x86_64', + ['bash', os.path.join(self.topdir, 'work/x86_64/tmp-Server/extraiso-my.iso.sh')], + [self.topdir], + log_file=os.path.join(self.topdir, 'logs/x86_64/extraiso-my.iso.x86_64.log'), + with_jigdo=False)] + + ) + self.assertEqual( + aitm.call_args_list, + [mock.call(compose, server, 'x86_64', + os.path.join(self.topdir, 'compose/Server/x86_64/iso/my.iso'), + False, 1, 1)] + ) + + def test_source_is_not_bootable(self, aitm, rcc, gef, gic, gfn, gvi): + compose = helpers.DummyCompose(self.topdir, { + 'bootable': True, + 'buildinstall_method': 'lorax' + }) + server = compose.variants['Server'] + cfg = { + 'include_variants': ['Client'], + } + + gfn.return_value = 'my.iso' + gvi.return_value = 'my volume id' + gic.return_value = '/tmp/iso-graft-points' + + t = extra_isos.ExtraIsosThread(mock.Mock()) + with mock.patch('time.sleep'): + t.process((compose, cfg, server, 'src'), 1) + + self.assertEqual(gfn.call_args_list, + [mock.call(compose, server, 'src', None)]) + self.assertEqual(gvi.call_args_list, + [mock.call(compose, server, 'src', [])]) + self.assertEqual(gef.call_args_list, + [mock.call(compose, server, 'src', [])]) + self.assertEqual(gic.call_args_list, + [mock.call(compose, server, 'src', ['Client'], 'my.iso', False)]) + self.assertEqual( + rcc.call_args_list, + [mock.call(False, 1, compose, False, 'src', + ['bash', os.path.join(self.topdir, 'work/src/tmp-Server/extraiso-my.iso.sh')], + [self.topdir], + log_file=os.path.join(self.topdir, 'logs/src/extraiso-my.iso.src.log'), + with_jigdo=False)] + + ) + self.assertEqual( + aitm.call_args_list, + [mock.call(compose, server, 'src', + os.path.join(self.topdir, 'compose/Server/source/iso/my.iso'), + False, 1, 1)] + ) + + def test_failable_failed(self, aitm, rcc, gef, gic, gfn, gvi): + compose = helpers.DummyCompose(self.topdir, {}) + server = compose.variants['Server'] + cfg = { + 'include_variants': ['Client'], + 'failable_arches': ['x86_64'], + } + + gfn.return_value = 'my.iso' + gvi.return_value = 'my volume id' + gic.return_value = '/tmp/iso-graft-points' + rcc.side_effect = helpers.mk_boom() + + t = extra_isos.ExtraIsosThread(mock.Mock()) + with mock.patch('time.sleep'): + t.process((compose, cfg, server, 'x86_64'), 1) + + self.assertEqual(aitm.call_args_list, []) + + def test_non_failable_failed(self, aitm, rcc, gef, gic, gfn, gvi): + compose = helpers.DummyCompose(self.topdir, {}) + server = compose.variants['Server'] + cfg = { + 'include_variants': ['Client'], + } + + gfn.return_value = 'my.iso' + gvi.return_value = 'my volume id' + gic.return_value = '/tmp/iso-graft-points' + rcc.side_effect = helpers.mk_boom(RuntimeError) + + t = extra_isos.ExtraIsosThread(mock.Mock()) + with self.assertRaises(RuntimeError): + with mock.patch('time.sleep'): + t.process((compose, cfg, server, 'x86_64'), 1) + + self.assertEqual(aitm.call_args_list, []) + + +@mock.patch('pungi.phases.extra_isos.get_file_from_scm') +@mock.patch('pungi.phases.extra_isos.get_dir_from_scm') +class GetExtraFilesTest(helpers.PungiTestCase): + + def setUp(self): + super(GetExtraFilesTest, self).setUp() + self.compose = helpers.DummyCompose(self.topdir, {}) + self.variant = self.compose.variants['Server'] + self.arch = 'x86_64' + + def test_no_config(self, get_dir, get_file): + extra_isos.get_extra_files(self.compose, self.variant, self.arch, []) + + self.assertEqual(get_dir.call_args_list, []) + self.assertEqual(get_file.call_args_list, []) + + def test_get_file(self, get_dir, get_file): + cfg = { + 'scm': 'git', + 'repo': 'https://pagure.io/pungi.git', + 'file': 'GPL', + 'target': 'legalese', + } + extra_isos.get_extra_files(self.compose, self.variant, self.arch, [cfg]) + + self.assertEqual(get_dir.call_args_list, []) + self.assertEqual(get_file.call_args_list, + [mock.call(cfg, + os.path.join(self.topdir, 'work', + self.arch, self.variant.uid, + 'extra-iso-extra-files/legalese'), + logger=self.compose._logger)]) + + def test_get_dir(self, get_dir, get_file): + cfg = { + 'scm': 'git', + 'repo': 'https://pagure.io/pungi.git', + 'dir': 'docs', + 'target': 'foo', + } + extra_isos.get_extra_files(self.compose, self.variant, self.arch, [cfg]) + + self.assertEqual(get_file.call_args_list, []) + self.assertEqual(get_dir.call_args_list, + [mock.call(cfg, + os.path.join(self.topdir, 'work', + self.arch, self.variant.uid, + 'extra-iso-extra-files/foo'), + logger=self.compose._logger)]) + + +@mock.patch('pungi.wrappers.iso.write_graft_points') +@mock.patch('pungi.wrappers.iso.get_graft_points') +class GetIsoContentsTest(helpers.PungiTestCase): + + def setUp(self): + super(GetIsoContentsTest, self).setUp() + self.compose = helpers.DummyCompose(self.topdir, {}) + self.variant = self.compose.variants['Server'] + + def test_non_bootable_binary(self, ggp, wgp): + gp = { + 'compose/Client/x86_64/os/Packages': {'f/foo.rpm': '/mnt/f/foo.rpm'}, + 'compose/Client/x86_64/os/repodata': {'primary.xml': '/mnt/repodata/primary.xml'}, + 'compose/Server/x86_64/os/Packages': {'b/bar.rpm': '/mnt/b/bar.rpm'}, + 'compose/Server/x86_64/os/repodata': {'repomd.xml': '/mnt/repodata/repomd.xml'}, + 'work/x86_64/Client/extra-files': {'GPL': '/mnt/GPL'}, + 'work/x86_64/Server/extra-files': {'AUTHORS': '/mnt/AUTHORS'}, + 'work/x86_64/Server/extra-iso-extra-files': {'EULA': '/mnt/EULA'}, + } + + ggp.side_effect = lambda x: gp[x[0][len(self.topdir) + 1:]] + gp_file = os.path.join(self.topdir, 'work/x86_64/iso/my.iso-graft-points') + + self.assertEqual( + extra_isos.get_iso_contents(self.compose, self.variant, 'x86_64', + ['Client'], 'my.iso', False), + gp_file + ) + + expected = { + 'Client/GPL': '/mnt/GPL', + 'Client/Packages/f/foo.rpm': '/mnt/f/foo.rpm', + 'Client/repodata/primary.xml': '/mnt/repodata/primary.xml', + 'EULA': '/mnt/EULA', + 'Server/AUTHORS': '/mnt/AUTHORS', + 'Server/Packages/b/bar.rpm': '/mnt/b/bar.rpm', + 'Server/repodata/repomd.xml': '/mnt/repodata/repomd.xml', + } + + self.assertItemsEqual( + ggp.call_args_list, + [mock.call([os.path.join(self.topdir, x)]) for x in gp] + ) + self.assertEqual(len(wgp.call_args_list), 1) + self.assertEqual(wgp.call_args_list[0][0][0], gp_file) + self.assertDictEqual(dict(wgp.call_args_list[0][0][1]), expected) + self.assertEqual(wgp.call_args_list[0][1], {'exclude': ["*/lost+found", "*/boot.iso"]}) + + def test_source(self, ggp, wgp): + gp = { + 'compose/Client/source/tree/Packages': {'f/foo.rpm': '/mnt/f/foo.rpm'}, + 'compose/Client/source/tree/repodata': {'primary.xml': '/mnt/repodata/primary.xml'}, + 'compose/Server/source/tree/Packages': {'b/bar.rpm': '/mnt/b/bar.rpm'}, + 'compose/Server/source/tree/repodata': {'repomd.xml': '/mnt/repodata/repomd.xml'}, + 'work/src/Client/extra-files': {'GPL': '/mnt/GPL'}, + 'work/src/Server/extra-files': {'AUTHORS': '/mnt/AUTHORS'}, + 'work/src/Server/extra-iso-extra-files': {'EULA': '/mnt/EULA'}, + } + + ggp.side_effect = lambda x: gp[x[0][len(self.topdir) + 1:]] + gp_file = os.path.join(self.topdir, 'work/src/iso/my.iso-graft-points') + + self.assertEqual( + extra_isos.get_iso_contents(self.compose, self.variant, 'src', + ['Client'], 'my.iso', False), + gp_file + ) + + expected = { + 'Client/GPL': '/mnt/GPL', + 'Client/Packages/f/foo.rpm': '/mnt/f/foo.rpm', + 'Client/repodata/primary.xml': '/mnt/repodata/primary.xml', + 'EULA': '/mnt/EULA', + 'Server/AUTHORS': '/mnt/AUTHORS', + 'Server/Packages/b/bar.rpm': '/mnt/b/bar.rpm', + 'Server/repodata/repomd.xml': '/mnt/repodata/repomd.xml', + } + + self.assertItemsEqual( + ggp.call_args_list, + [mock.call([os.path.join(self.topdir, x)]) for x in gp] + ) + self.assertEqual(len(wgp.call_args_list), 1) + self.assertEqual(wgp.call_args_list[0][0][0], gp_file) + self.assertDictEqual(dict(wgp.call_args_list[0][0][1]), expected) + self.assertEqual(wgp.call_args_list[0][1], {'exclude': ["*/lost+found", "*/boot.iso"]}) + + def test_bootable(self, ggp, wgp): + self.compose.conf['buildinstall_method'] = 'lorax' + + bi_dir = os.path.join(self.topdir, 'work/x86_64/buildinstall/Server') + iso_dir = os.path.join(self.topdir, 'work/x86_64/iso/my.iso') + helpers.touch(os.path.join(bi_dir, 'isolinux/isolinux.bin')) + helpers.touch(os.path.join(bi_dir, 'images/boot.img')) + + gp = { + 'compose/Client/x86_64/os/Packages': {'f/foo.rpm': '/mnt/f/foo.rpm'}, + 'compose/Client/x86_64/os/repodata': {'primary.xml': '/mnt/repodata/primary.xml'}, + 'compose/Server/x86_64/os/Packages': {'b/bar.rpm': '/mnt/b/bar.rpm'}, + 'compose/Server/x86_64/os/repodata': {'repomd.xml': '/mnt/repodata/repomd.xml'}, + 'work/x86_64/Client/extra-files': {'GPL': '/mnt/GPL'}, + 'work/x86_64/Server/extra-files': {'AUTHORS': '/mnt/AUTHORS'}, + 'work/x86_64/Server/extra-iso-extra-files': {'EULA': '/mnt/EULA'}, + } + bi_gp = { + 'isolinux/isolinux.bin': os.path.join(iso_dir, 'isolinux/isolinux.bin'), + 'images/boot.img': os.path.join(iso_dir, 'images/boot.img'), + } + + ggp.side_effect = lambda x: gp[x[0][len(self.topdir) + 1:]] if len(x) == 1 else bi_gp + gp_file = os.path.join(self.topdir, 'work/x86_64/iso/my.iso-graft-points') + + self.assertEqual( + extra_isos.get_iso_contents(self.compose, self.variant, 'x86_64', + ['Client'], 'my.iso', True), + gp_file + ) + + self.maxDiff = None + + expected = { + 'Client/GPL': '/mnt/GPL', + 'Client/Packages/f/foo.rpm': '/mnt/f/foo.rpm', + 'Client/repodata/primary.xml': '/mnt/repodata/primary.xml', + 'EULA': '/mnt/EULA', + 'Server/AUTHORS': '/mnt/AUTHORS', + 'Server/Packages/b/bar.rpm': '/mnt/b/bar.rpm', + 'Server/repodata/repomd.xml': '/mnt/repodata/repomd.xml', + 'isolinux/isolinux.bin': os.path.join(iso_dir, 'isolinux/isolinux.bin'), + 'images/boot.img': os.path.join(iso_dir, 'images/boot.img'), + } + + self.assertItemsEqual( + ggp.call_args_list, + [mock.call([os.path.join(self.topdir, x)]) for x in gp] + [mock.call([bi_dir, iso_dir])] + ) + self.assertEqual(len(wgp.call_args_list), 1) + self.assertEqual(wgp.call_args_list[0][0][0], gp_file) + self.assertDictEqual(dict(wgp.call_args_list[0][0][1]), expected) + self.assertEqual(wgp.call_args_list[0][1], {'exclude': ["*/lost+found", "*/boot.iso"]}) + + # Check files were copied to temp directory + self.assertTrue(os.path.exists(os.path.join(iso_dir, 'isolinux/isolinux.bin'))) + self.assertTrue(os.path.exists(os.path.join(iso_dir, 'images/boot.img'))) + + +class GetFilenameTest(helpers.PungiTestCase): + def test_use_original_name(self): + compose = helpers.DummyCompose(self.topdir, {}) + + fn = extra_isos.get_filename(compose, compose.variants['Server'], 'x86_64', + 'foo-{variant}-{arch}-{filename}') + + self.assertEqual(fn, 'foo-Server-x86_64-image-name') + + def test_use_default_without_format(self): + compose = helpers.DummyCompose(self.topdir, {}) + + fn = extra_isos.get_filename(compose, compose.variants['Server'], 'x86_64', + None) + + self.assertEqual(fn, 'image-name') + + def test_reports_unknown_placeholder(self): + compose = helpers.DummyCompose(self.topdir, {}) + + with self.assertRaises(RuntimeError) as ctx: + extra_isos.get_filename(compose, compose.variants['Server'], 'x86_64', + 'foo-{boom}') + + self.assertIn('boom', str(ctx.exception)) + + +class GetVolumeIDTest(helpers.PungiTestCase): + def test_use_original_volume_id(self): + compose = helpers.DummyCompose(self.topdir, {}) + + volid = extra_isos.get_volume_id(compose, compose.variants['Server'], + 'x86_64', + 'f-{volid}') + + self.assertEqual(volid, 'f-test-1.0 Server.x86_64') + + def test_falls_back_to_shorter(self): + compose = helpers.DummyCompose(self.topdir, {}) + + volid = extra_isos.get_volume_id(compose, compose.variants['Server'], + 'x86_64', + ['long-foobar-{volid}', 'f-{volid}']) + + self.assertEqual(volid, 'f-test-1.0 Server.x86_64') + + def test_reports_unknown_placeholder(self): + compose = helpers.DummyCompose(self.topdir, {}) + + with self.assertRaises(RuntimeError) as ctx: + extra_isos.get_volume_id(compose, compose.variants['Server'], + 'x86_64', 'f-{boom}') + + self.assertIn('boom', str(ctx.exception)) + + +if __name__ == '__main__': + unittest.main()