diff --git a/bin/pungi-createiso b/bin/pungi-createiso new file mode 100755 index 00000000..46003bd4 --- /dev/null +++ b/bin/pungi-createiso @@ -0,0 +1,15 @@ +#!/usr/bin/env python2 +# -*- coding: utf-8 -*- + +import os +import sys + +here = sys.path[0] +if here != '/usr/bin': + # Git checkout + sys.path[0] = os.path.dirname(here) + +import pungi.createiso + +if __name__ == '__main__': + pungi.createiso.main() diff --git a/bin/pungi-pylorax-find-templates b/bin/pungi-pylorax-find-templates new file mode 100755 index 00000000..0fad5d33 --- /dev/null +++ b/bin/pungi-pylorax-find-templates @@ -0,0 +1,20 @@ +#!/usr/bin/env python3 + +# This needs to work with Python 3 as pylorax only provides the find_templates +# function in recent builds that are not provided for Python 2.7. +# +# This script will print a location of lorax templates. If it fails to import +# pylorax, or the find_templates function does not exist, the first command +# line argument will be printed instead. + +import sys + +if len(sys.argv) != 2: + print('Usage: {} FALLBACK'.format(sys.argv[0]), file=sys.stderr) + sys.exit(1) + +try: + import pylorax + print(pylorax.find_templates()) +except (ImportError, AttributeError): + print(sys.argv[1]) diff --git a/pungi/createiso.py b/pungi/createiso.py new file mode 100644 index 00000000..4178f3dd --- /dev/null +++ b/pungi/createiso.py @@ -0,0 +1,115 @@ +# -*- coding: utf-8 -*- + +import argparse +import os +import contextlib +from kobo import shortcuts + +from .wrappers.iso import IsoWrapper +from .wrappers.jigdo import JigdoWrapper +from .util import makedirs + + +def find_templates(fallback): + """ + Helper for finding lorax templates. The called program needs to run with + Python 3, while the rest of this script only supports Python 2. + """ + _, output = shortcuts.run(['pungi-pylorax-find-templates', fallback], + stdout=True, show_cmd=True) + return output.strip() + + +@contextlib.contextmanager +def in_dir(dir): + """Temporarily switch to another directory.""" + old_cwd = os.getcwd() + makedirs(dir) + os.chdir(dir) + yield + os.chdir(old_cwd) + + +def make_image(iso, opts): + mkisofs_kwargs = {} + + if opts.buildinstall_method: + if opts.buildinstall_method == 'lorax': + dir = find_templates('/usr/share/lorax') + mkisofs_kwargs["boot_args"] = iso.get_boot_options( + opts.arch, os.path.join(dir, 'config_files/ppc')) + elif opts.buildinstall_method == 'buildinstall': + mkisofs_kwargs["boot_args"] = iso.get_boot_options( + opts.arch, "/usr/lib/anaconda-runtime/boot") + + # ppc(64) doesn't seem to support utf-8 + if opts.arch in ("ppc", "ppc64", "ppc64le"): + mkisofs_kwargs["input_charset"] = None + + cmd = iso.get_mkisofs_cmd(opts.iso_name, None, volid=opts.volid, + exclude=["./lost+found"], + graft_points=opts.graft_points, **mkisofs_kwargs) + shortcuts.run(cmd, stdout=True, show_cmd=True) + + +def implant_md5(iso, opts): + cmd = iso.get_implantisomd5_cmd(opts.iso_name, opts.supported) + shortcuts.run(cmd, stdout=True, show_cmd=True) + + +def make_manifest(iso, opts): + shortcuts.run(iso.get_manifest_cmd(opts.iso_name), stdout=True, show_cmd=True) + + +def make_jigdo(opts): + jigdo = JigdoWrapper() + files = [ + { + "path": opts.os_tree, + "label": None, + "uri": None, + } + ] + cmd = jigdo.get_jigdo_cmd(os.path.join(opts.output_dir, opts.iso_name), + files, output_dir=opts.jigdo_dir, + no_servers=True, report="noprogress") + shortcuts.run(cmd, stdout=True, show_cmd=True) + + +def run(opts): + iso = IsoWrapper() + make_image(iso, opts) + implant_md5(iso, opts) + make_manifest(iso, opts) + if opts.jigdo_dir: + make_jigdo(opts) + + +def main(args=None): + parser = argparse.ArgumentParser() + parser.add_argument('--output-dir', required=True, + help='where to put the final image') + parser.add_argument('--iso-name', required=True, + help='filename for the created ISO image') + parser.add_argument('--volid', required=True, + help='volume id for the image') + parser.add_argument('--graft-points', required=True, + help='') + parser.add_argument('--buildinstall-method', + choices=['lorax', 'buildinstall'], + help='how was the boot.iso created for bootable products') + parser.add_argument('--arch', required=True, + help='what arch are we building the ISO for') + parser.add_argument('--supported', action='store_true', + help='supported flag for implantisomd5') + parser.add_argument('--jigdo-dir', + help='where to put jigdo files') + parser.add_argument('--os-tree', + help='where to put jigdo files') + + opts = parser.parse_args(args) + + if bool(opts.jigdo_dir) != bool(opts.os_tree): + parser.error('--jigdo-dir must be used together with --os-tree') + with in_dir(opts.output_dir): + run(opts) diff --git a/pungi/phases/createiso.py b/pungi/phases/createiso.py index 6af784a6..df0322d2 100644 --- a/pungi/phases/createiso.py +++ b/pungi/phases/createiso.py @@ -29,7 +29,6 @@ from kobo.shortcuts import run, relative_path from pungi.wrappers.iso import IsoWrapper from pungi.wrappers.createrepo import CreaterepoWrapper from pungi.wrappers.kojiwrapper import KojiWrapper -from pungi.wrappers.jigdo import JigdoWrapper from pungi.phases.base import PhaseBase from pungi.util import makedirs, get_volid, get_arch_variant_data, failable from pungi.media_split import MediaSplitter @@ -51,8 +50,22 @@ class CreateisoPhase(PhaseBase): PhaseBase.__init__(self, compose) self.pool = ThreadPool(logger=self.compose._logger) + def _find_rpms(self, path): + """Check if there are some RPMs in the path.""" + for _, _, files in os.walk(path): + for fn in files: + if fn.endswith(".rpm"): + return True + return False + + def _is_bootable(self, variant, arch): + if arch == "src": + return False + if variant.type != "variant": + return False + return self.compose.conf.get("bootable", False) + def run(self): - iso = IsoWrapper(logger=self.compose._logger) symlink_isos_to = self.compose.conf.get("symlink_isos_to", None) disc_type = self.compose.conf.get('disc_types', {}).get('dvd', 'dvd') deliverables = [] @@ -72,17 +85,9 @@ class CreateisoPhase(PhaseBase): if not iso_dir: continue - found = False - for root, dirs, files in os.walk(os_tree): - if found: - break - for fn in files: - if fn.endswith(".rpm"): - found = True - break - - if not found: - self.compose.log_warning("No RPMs found for %s.%s, skipping ISO" % (variant, arch)) + if not self._find_rpms(os_tree): + self.compose.log_warning("No RPMs found for %s.%s, skipping ISO" + % (variant, arch)) continue split_iso_data = split_iso(self.compose, arch, variant) @@ -91,31 +96,22 @@ class CreateisoPhase(PhaseBase): for disc_num, iso_data in enumerate(split_iso_data): disc_num += 1 - filename = self.compose.get_image_name(arch, variant, - disc_type=disc_type, - disc_num=disc_num) - iso_path = self.compose.paths.compose.iso_path(arch, - variant, - filename, - symlink_to=symlink_isos_to) - relative_iso_path = self.compose.paths.compose.iso_path(arch, - variant, - filename, - create_dir=False, - relative=True) + filename = self.compose.get_image_name( + arch, variant, disc_type=disc_type, disc_num=disc_num) + iso_path = self.compose.paths.compose.iso_path( + arch, variant, filename, symlink_to=symlink_isos_to) + relative_iso_path = self.compose.paths.compose.iso_path( + arch, variant, filename, create_dir=False, relative=True) if os.path.isfile(iso_path): self.compose.log_warning("Skipping mkisofs, image already exists: %s" % iso_path) continue - iso_name = os.path.basename(iso_path) deliverables.append(iso_path) - graft_points = prepare_iso(self.compose, arch, variant, disc_num=disc_num, disc_count=disc_count, split_iso_data=iso_data) + graft_points = prepare_iso(self.compose, arch, variant, + disc_num=disc_num, disc_count=disc_count, + split_iso_data=iso_data) - bootable = self.compose.conf.get("bootable", False) - if arch == "src": - bootable = False - if variant.type != "variant": - bootable = False + bootable = self._is_bootable(variant, arch) cmd = { "arch": arch, @@ -131,61 +127,34 @@ class CreateisoPhase(PhaseBase): } if os.path.islink(iso_dir): - cmd["mount"] = os.path.abspath(os.path.join(os.path.dirname(iso_dir), os.readlink(iso_dir))) + cmd["mount"] = os.path.abspath(os.path.join(os.path.dirname(iso_dir), + os.readlink(iso_dir))) - chdir_cmd = "cd %s" % pipes.quote(iso_dir) - cmd["cmd"].append(chdir_cmd) - - mkisofs_kwargs = {} + cmd['cmd'] = [ + 'pungi-createiso', + '--output-dir={}'.format(iso_dir), + '--iso-name={}'.format(filename), + '--volid={}'.format(volid), + '--graft-points={}'.format(graft_points), + '--arch={}'.format(arch), + ] if bootable: - buildinstall_method = self.compose.conf["buildinstall_method"] - if buildinstall_method == "lorax": - # TODO: $arch instead of ppc - mkisofs_kwargs["boot_args"] = iso.get_boot_options(arch, "/usr/share/lorax/config_files/ppc") - elif buildinstall_method == "buildinstall": - mkisofs_kwargs["boot_args"] = iso.get_boot_options(arch, "/usr/lib/anaconda-runtime/boot") + cmd['cmd'].extend([ + '--bootable', + '--buildinstall-method={}'.format(self.compose.conf['buildinstall_method']), + ]) - # ppc(64) doesn't seem to support utf-8 - if arch in ("ppc", "ppc64", "ppc64le"): - mkisofs_kwargs["input_charset"] = None + if self.compose.supported: + cmd['cmd'].append('--supported') - mkisofs_cmd = iso.get_mkisofs_cmd(iso_name, None, volid=volid, exclude=["./lost+found"], graft_points=graft_points, **mkisofs_kwargs) - mkisofs_cmd = " ".join([pipes.quote(i) for i in mkisofs_cmd]) - cmd["cmd"].append(mkisofs_cmd) - - if bootable and arch == "x86_64": - isohybrid_cmd = "isohybrid --uefi %s" % pipes.quote(iso_name) - cmd["cmd"].append(isohybrid_cmd) - elif bootable and arch == "i386": - isohybrid_cmd = "isohybrid %s" % pipes.quote(iso_name) - cmd["cmd"].append(isohybrid_cmd) - - # implant MD5SUM to iso - isomd5sum_cmd = iso.get_implantisomd5_cmd(iso_name, self.compose.supported) - isomd5sum_cmd = " ".join([pipes.quote(i) for i in isomd5sum_cmd]) - cmd["cmd"].append(isomd5sum_cmd) - - # create iso manifest - cmd["cmd"].append(iso.get_manifest_cmd(iso_name)) - - # create jigdo - create_jigdo = self.compose.conf.get("create_jigdo", True) - if create_jigdo: - jigdo = JigdoWrapper(logger=self.compose._logger) + if self.compose.conf.get('create_jigdo', True): jigdo_dir = self.compose.paths.compose.jigdo_dir(arch, variant) - files = [ - { - "path": os_tree, - "label": None, - "uri": None, - } - ] - jigdo_cmd = jigdo.get_jigdo_cmd(iso_path, files, output_dir=jigdo_dir, no_servers=True, report="noprogress") - jigdo_cmd = " ".join([pipes.quote(i) for i in jigdo_cmd]) - cmd["cmd"].append(jigdo_cmd) + cmd['cmd'].extend([ + '--jigdo-dir={}'.format(jigdo_dir), + '--os-tree={}'.format(os_tree), + ]) - cmd["cmd"] = " && ".join(cmd["cmd"]) commands.append((cmd, variant, arch)) self.compose.notifier.send('createiso-targets', deliverables=deliverables) @@ -228,20 +197,22 @@ class CreateIsoThread(WorkerThread): runroot = compose.conf.get("runroot", False) bootable = compose.conf.get("bootable", False) - log_file = compose.paths.log.log_file(cmd["arch"], "createiso-%s" % os.path.basename(cmd["iso_path"])) + log_file = compose.paths.log.log_file( + cmd["arch"], "createiso-%s" % os.path.basename(cmd["iso_path"])) - msg = "Creating ISO (arch: %s, variant: %s): %s" % (cmd["arch"], cmd["variant"], os.path.basename(cmd["iso_path"])) + msg = "Creating ISO (arch: %s, variant: %s): %s" % ( + cmd["arch"], cmd["variant"], os.path.basename(cmd["iso_path"])) self.pool.log_info("[BEGIN] %s" % msg) if runroot: # run in a koji build root packages = ["coreutils", "genisoimage", "isomd5sum", "jigdo", "strace", "lsof"] + extra_packages = { + 'lorax': ['lorax', 'pungi'], + 'buildinstall': ['anaconda'], + } if bootable: - buildinstall_method = compose.conf["buildinstall_method"] - if buildinstall_method == "lorax": - packages += ["lorax"] - elif buildinstall_method == "buildinstall": - packages += ["anaconda"] + packages.extend(extra_packages[compose.conf["buildinstall_method"]]) runroot_channel = compose.conf.get("runroot_channel", None) runroot_tag = compose.conf["runroot_tag"] @@ -260,7 +231,10 @@ class CreateIsoThread(WorkerThread): # pick random arch from available runroot tag arches cmd["build_arch"] = random.choice(tag_arches) - koji_cmd = koji_wrapper.get_runroot_cmd(runroot_tag, cmd["build_arch"], cmd["cmd"], channel=runroot_channel, use_shell=True, task_id=True, packages=packages, mounts=mounts) + koji_cmd = koji_wrapper.get_runroot_cmd( + runroot_tag, cmd["build_arch"], cmd["cmd"], + channel=runroot_channel, use_shell=True, task_id=True, + packages=packages, mounts=mounts) # avoid race conditions? # Kerberos authentication failed: Permission denied in replay cache code (-1765328215) @@ -269,7 +243,8 @@ class CreateIsoThread(WorkerThread): output = koji_wrapper.run_runroot_cmd(koji_cmd, log_file=log_file) if output["retcode"] != 0: self.fail(compose, cmd) - raise RuntimeError("Runroot task failed: %s. See %s for more details." % (output["task_id"], log_file)) + raise RuntimeError("Runroot task failed: %s. See %s for more details." + % (output["task_id"], log_file)) else: # run locally diff --git a/tests/test_createiso_script.py b/tests/test_createiso_script.py new file mode 100755 index 00000000..4532d891 --- /dev/null +++ b/tests/test_createiso_script.py @@ -0,0 +1,197 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- + +import unittest +import mock + +import os +import sys + +sys.path.insert(0, os.path.join(os.path.dirname(__file__), '..')) + +from tests import helpers +from pungi import createiso + + +class OstreeScriptTest(helpers.PungiTestCase): + + def assertEqualCalls(self, actual, expected): + self.assertEqual(len(actual), len(expected)) + for x, y in zip(actual, expected): + self.assertEqual(x, y) + + @mock.patch('kobo.shortcuts.run') + def test_minimal_run(self, run): + createiso.main([ + '--output-dir={}/isos'.format(self.topdir), + '--iso-name=DP-1.0-20160405.t.3-x86_64.iso', + '--volid=DP-1.0-20160405.t.3', + '--graft-points=graft-list', + '--arch=x86_64', + ]) + self.maxDiff = None + self.assertEqual( + run.call_args_list, + [mock.call(['/usr/bin/genisoimage', '-untranslated-filenames', + '-volid', 'DP-1.0-20160405.t.3', '-J', '-joliet-long', + '-rational-rock', '-translation-table', + '-input-charset', 'utf-8', '-x', './lost+found', + '-o', 'DP-1.0-20160405.t.3-x86_64.iso', + '-graft-points', '-path-list', 'graft-list'], + show_cmd=True, stdout=True), + mock.call(['/usr/bin/implantisomd5', 'DP-1.0-20160405.t.3-x86_64.iso'], + show_cmd=True, stdout=True), + mock.call('isoinfo -R -f -i DP-1.0-20160405.t.3-x86_64.iso | grep -v \'/TRANS.TBL$\' | sort >> DP-1.0-20160405.t.3-x86_64.iso.manifest', + show_cmd=True, stdout=True)] + ) + + @mock.patch('kobo.shortcuts.run') + def test_bootable_run(self, run): + run.return_value = (0, '/usr/share/lorax') + + createiso.main([ + '--output-dir={}/isos'.format(self.topdir), + '--iso-name=DP-1.0-20160405.t.3-x86_64.iso', + '--volid=DP-1.0-20160405.t.3', + '--graft-points=graft-list', + '--arch=x86_64', + '--buildinstall-method=lorax', + ]) + + self.maxDiff = None + self.assertItemsEqual( + run.call_args_list, + [mock.call(['/usr/bin/genisoimage', '-untranslated-filenames', + '-volid', 'DP-1.0-20160405.t.3', '-J', '-joliet-long', + '-rational-rock', '-translation-table', + '-input-charset', 'utf-8', '-x', './lost+found', + '-b', 'isolinux/isolinux.bin', '-c', 'isolinux/boot.cat', + '-no-emul-boot', + '-boot-load-size', '4', '-boot-info-table', + '-eltorito-alt-boot', '-e', 'images/efiboot.img', + '-no-emul-boot', + '-o', 'DP-1.0-20160405.t.3-x86_64.iso', + '-graft-points', '-path-list', 'graft-list'], + show_cmd=True, stdout=True), + mock.call(['pungi-pylorax-find-templates', '/usr/share/lorax'], + show_cmd=True, stdout=True), + mock.call(['/usr/bin/implantisomd5', 'DP-1.0-20160405.t.3-x86_64.iso'], + show_cmd=True, stdout=True), + mock.call('isoinfo -R -f -i DP-1.0-20160405.t.3-x86_64.iso | grep -v \'/TRANS.TBL$\' | sort >> DP-1.0-20160405.t.3-x86_64.iso.manifest', + show_cmd=True, stdout=True)] + ) + + @mock.patch('kobo.shortcuts.run') + def test_bootable_run_ppc64(self, run): + run.return_value = (0, '/usr/share/lorax') + + createiso.main([ + '--output-dir={}/isos'.format(self.topdir), + '--iso-name=DP-1.0-20160405.t.3-ppc64.iso', + '--volid=DP-1.0-20160405.t.3', + '--graft-points=graft-list', + '--arch=ppc64', + '--buildinstall-method=lorax', + ]) + + self.maxDiff = None + self.assertItemsEqual( + run.call_args_list, + [mock.call(['/usr/bin/genisoimage', '-untranslated-filenames', + '-volid', 'DP-1.0-20160405.t.3', '-J', '-joliet-long', + '-rational-rock', '-translation-table', + '-x', './lost+found', + '-part', '-hfs', '-r', '-l', '-sysid', 'PPC', '-no-desktop', + '-allow-multidot', '-chrp-boot', '-map', '/usr/share/lorax/config_files/ppc/mapping', + '-hfs-bless', '/ppc/mac', + '-o', 'DP-1.0-20160405.t.3-ppc64.iso', + '-graft-points', '-path-list', 'graft-list'], + show_cmd=True, stdout=True), + mock.call(['pungi-pylorax-find-templates', '/usr/share/lorax'], + show_cmd=True, stdout=True), + mock.call(['/usr/bin/implantisomd5', 'DP-1.0-20160405.t.3-ppc64.iso'], + show_cmd=True, stdout=True), + mock.call('isoinfo -R -f -i DP-1.0-20160405.t.3-ppc64.iso | grep -v \'/TRANS.TBL$\' | sort >> DP-1.0-20160405.t.3-ppc64.iso.manifest', + show_cmd=True, stdout=True)] + ) + + @mock.patch('kobo.shortcuts.run') + def test_bootable_run_buildinstall(self, run): + createiso.main([ + '--output-dir={}/isos'.format(self.topdir), + '--iso-name=DP-1.0-20160405.t.3-ppc64.iso', + '--volid=DP-1.0-20160405.t.3', + '--graft-points=graft-list', + '--arch=ppc64', + '--buildinstall-method=buildinstall', + ]) + + self.maxDiff = None + self.assertItemsEqual( + run.call_args_list, + [mock.call(['/usr/bin/genisoimage', '-untranslated-filenames', + '-volid', 'DP-1.0-20160405.t.3', '-J', '-joliet-long', + '-rational-rock', '-translation-table', + '-x', './lost+found', + '-part', '-hfs', '-r', '-l', '-sysid', 'PPC', '-no-desktop', + '-allow-multidot', '-chrp-boot', + '-map', '/usr/lib/anaconda-runtime/boot/mapping', + '-hfs-bless', '/ppc/mac', + '-o', 'DP-1.0-20160405.t.3-ppc64.iso', + '-graft-points', '-path-list', 'graft-list'], + show_cmd=True, stdout=True), + mock.call(['/usr/bin/implantisomd5', 'DP-1.0-20160405.t.3-ppc64.iso'], + show_cmd=True, stdout=True), + mock.call('isoinfo -R -f -i DP-1.0-20160405.t.3-ppc64.iso | grep -v \'/TRANS.TBL$\' | sort >> DP-1.0-20160405.t.3-ppc64.iso.manifest', + show_cmd=True, stdout=True)] + ) + + @mock.patch('sys.stderr') + @mock.patch('kobo.shortcuts.run') + def test_run_with_jigdo_bad_args(self, run, stderr): + with self.assertRaises(SystemExit): + createiso.main([ + '--output-dir={}/isos'.format(self.topdir), + '--iso-name=DP-1.0-20160405.t.3-x86_64.iso', + '--volid=DP-1.0-20160405.t.3', + '--graft-points=graft-list', + '--arch=x86_64', + '--jigdo-dir={}/jigdo'.format(self.topdir), + ]) + + @mock.patch('kobo.shortcuts.run') + def test_run_with_jigdo(self, run): + createiso.main([ + '--output-dir={}/isos'.format(self.topdir), + '--iso-name=DP-1.0-20160405.t.3-x86_64.iso', + '--volid=DP-1.0-20160405.t.3', + '--graft-points=graft-list', + '--arch=x86_64', + '--jigdo-dir={}/jigdo'.format(self.topdir), + '--os-tree={}/os'.format(self.topdir), + ]) + self.maxDiff = None + self.assertItemsEqual( + run.call_args_list, + [mock.call(['/usr/bin/genisoimage', '-untranslated-filenames', + '-volid', 'DP-1.0-20160405.t.3', '-J', '-joliet-long', + '-rational-rock', '-translation-table', + '-input-charset', 'utf-8', '-x', './lost+found', + '-o', 'DP-1.0-20160405.t.3-x86_64.iso', + '-graft-points', '-path-list', 'graft-list'], + show_cmd=True, stdout=True), + mock.call(['/usr/bin/implantisomd5', 'DP-1.0-20160405.t.3-x86_64.iso'], + show_cmd=True, stdout=True), + mock.call('isoinfo -R -f -i DP-1.0-20160405.t.3-x86_64.iso | grep -v \'/TRANS.TBL$\' | sort >> DP-1.0-20160405.t.3-x86_64.iso.manifest', + show_cmd=True, stdout=True), + mock.call(['jigdo-file', 'make-template', '--force', + '--image={}/isos/DP-1.0-20160405.t.3-x86_64.iso'.format(self.topdir), + '--jigdo={}/jigdo/DP-1.0-20160405.t.3-x86_64.iso.jigdo'.format(self.topdir), + '--template={}/jigdo/DP-1.0-20160405.t.3-x86_64.iso.template'.format(self.topdir), + '--no-servers-section', '--report=noprogress', self.topdir + '/os//'], + show_cmd=True, stdout=True)] + ) + + +if __name__ == '__main__': + unittest.main()