almalinux/pungi
14
1
Fork 0
Code Issues Pull Requests Releases Activity

[createiso] Move runroot work to separate script

Browse Source 
Instead of running a long command line in the runroot (or locally), move
all that work into a separate script that will be installed. This means
chroot will need to install pungi.

Everything should work as it did before. The only exception to this is
that there is logic to find lorax templates instead of harcoding the
location. This is done using a separate script.

Related: #230
Fixes: #231
Signed-off-by: Lubomír Sedlář <lsedlar@redhat.com>
This commit is contained in:
Lubomír Sedlář 2016-04-04 15:49:30 +02:00
parent 5b1b6c1c4f
commit df400002d8
5 changed files with 411 additions and 89 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

15
bin/pungi-createiso Executable file
View File

@ -0,0 +1,15 @@
#!/usr/bin/env python2
# -*- coding: utf-8 -*-
import os
import sys
here = sys.path[0]
if here != '/usr/bin':
# Git checkout
sys.path[0] = os.path.dirname(here)
import pungi.createiso
if __name__ == '__main__':
pungi.createiso.main()

20
bin/pungi-pylorax-find-templates Executable file
View File

@ -0,0 +1,20 @@
#!/usr/bin/env python3
# This needs to work with Python 3 as pylorax only provides the find_templates
# function in recent builds that are not provided for Python 2.7.
#
# This script will print a location of lorax templates. If it fails to import
# pylorax, or the find_templates function does not exist, the first command
# line argument will be printed instead.
import sys
if len(sys.argv) != 2:
print('Usage: {} FALLBACK'.format(sys.argv[0]), file=sys.stderr)
sys.exit(1)
try:
import pylorax
print(pylorax.find_templates())
except (ImportError, AttributeError):
print(sys.argv[1])

115
pungi/createiso.py Normal file
View File

@ -0,0 +1,115 @@
# -*- coding: utf-8 -*-
import argparse
import os
import contextlib
from kobo import shortcuts
from .wrappers.iso import IsoWrapper
from .wrappers.jigdo import JigdoWrapper
from .util import makedirs
def find_templates(fallback):
"""
Helper for finding lorax templates. The called program needs to run with
Python 3, while the rest of this script only supports Python 2.
"""
_, output = shortcuts.run(['pungi-pylorax-find-templates', fallback],
stdout=True, show_cmd=True)
return output.strip()
@contextlib.contextmanager
def in_dir(dir):
"""Temporarily switch to another directory."""
old_cwd = os.getcwd()
makedirs(dir)
os.chdir(dir)
yield
os.chdir(old_cwd)
def make_image(iso, opts):
mkisofs_kwargs = {}
if opts.buildinstall_method:
if opts.buildinstall_method == 'lorax':
dir = find_templates('/usr/share/lorax')
mkisofs_kwargs["boot_args"] = iso.get_boot_options(
opts.arch, os.path.join(dir, 'config_files/ppc'))
elif opts.buildinstall_method == 'buildinstall':
mkisofs_kwargs["boot_args"] = iso.get_boot_options(
opts.arch, "/usr/lib/anaconda-runtime/boot")
# ppc(64) doesn't seem to support utf-8
if opts.arch in ("ppc", "ppc64", "ppc64le"):
mkisofs_kwargs["input_charset"] = None
cmd = iso.get_mkisofs_cmd(opts.iso_name, None, volid=opts.volid,
exclude=["./lost+found"],
graft_points=opts.graft_points, **mkisofs_kwargs)
shortcuts.run(cmd, stdout=True, show_cmd=True)
def implant_md5(iso, opts):
cmd = iso.get_implantisomd5_cmd(opts.iso_name, opts.supported)
shortcuts.run(cmd, stdout=True, show_cmd=True)
def make_manifest(iso, opts):
shortcuts.run(iso.get_manifest_cmd(opts.iso_name), stdout=True, show_cmd=True)
def make_jigdo(opts):
jigdo = JigdoWrapper()
files = [
{
"path": opts.os_tree,
"label": None,
"uri": None,
}
]
cmd = jigdo.get_jigdo_cmd(os.path.join(opts.output_dir, opts.iso_name),
files, output_dir=opts.jigdo_dir,
no_servers=True, report="noprogress")
shortcuts.run(cmd, stdout=True, show_cmd=True)
def run(opts):
iso = IsoWrapper()
make_image(iso, opts)
implant_md5(iso, opts)
make_manifest(iso, opts)
if opts.jigdo_dir:
make_jigdo(opts)
def main(args=None):
parser = argparse.ArgumentParser()
parser.add_argument('--output-dir', required=True,
help='where to put the final image')
parser.add_argument('--iso-name', required=True,
help='filename for the created ISO image')
parser.add_argument('--volid', required=True,
help='volume id for the image')
parser.add_argument('--graft-points', required=True,
help='')
parser.add_argument('--buildinstall-method',
choices=['lorax', 'buildinstall'],
help='how was the boot.iso created for bootable products')
parser.add_argument('--arch', required=True,
help='what arch are we building the ISO for')
parser.add_argument('--supported', action='store_true',
help='supported flag for implantisomd5')
parser.add_argument('--jigdo-dir',
help='where to put jigdo files')
parser.add_argument('--os-tree',
help='where to put jigdo files')
opts = parser.parse_args(args)
if bool(opts.jigdo_dir) != bool(opts.os_tree):
parser.error('--jigdo-dir must be used together with --os-tree')
with in_dir(opts.output_dir):
run(opts)

153
pungi/phases/createiso.py
View File

@ -29,7 +29,6 @@ from kobo.shortcuts import run, relative_path
from pungi.wrappers.iso import IsoWrapper
from pungi.wrappers.createrepo import CreaterepoWrapper
from pungi.wrappers.kojiwrapper import KojiWrapper
from pungi.wrappers.jigdo import JigdoWrapper
from pungi.phases.base import PhaseBase
from pungi.util import makedirs, get_volid, get_arch_variant_data, failable
from pungi.media_split import MediaSplitter
@ -51,8 +50,22 @@ class CreateisoPhase(PhaseBase):
PhaseBase.__init__(self, compose)
self.pool = ThreadPool(logger=self.compose._logger)
def _find_rpms(self, path):
"""Check if there are some RPMs in the path."""
for _, _, files in os.walk(path):
for fn in files:
if fn.endswith(".rpm"):
return True
return False
def _is_bootable(self, variant, arch):
if arch == "src":
return False
if variant.type != "variant":
return False
return self.compose.conf.get("bootable", False)
def run(self):
iso = IsoWrapper(logger=self.compose._logger)
symlink_isos_to = self.compose.conf.get("symlink_isos_to", None)
disc_type = self.compose.conf.get('disc_types', {}).get('dvd', 'dvd')
deliverables = []
@ -72,17 +85,9 @@ class CreateisoPhase(PhaseBase):
if not iso_dir:
continue
found = False
for root, dirs, files in os.walk(os_tree):
if found:
break
for fn in files:
if fn.endswith(".rpm"):
found = True
break
if not found:
self.compose.log_warning("No RPMs found for %s.%s, skipping ISO" % (variant, arch))
if not self._find_rpms(os_tree):
self.compose.log_warning("No RPMs found for %s.%s, skipping ISO"
% (variant, arch))
continue
split_iso_data = split_iso(self.compose, arch, variant)
@ -91,31 +96,22 @@ class CreateisoPhase(PhaseBase):
for disc_num, iso_data in enumerate(split_iso_data):
disc_num += 1
filename = self.compose.get_image_name(arch, variant,
disc_type=disc_type,
disc_num=disc_num)
iso_path = self.compose.paths.compose.iso_path(arch,
variant,
filename,
symlink_to=symlink_isos_to)
relative_iso_path = self.compose.paths.compose.iso_path(arch,
variant,
filename,
create_dir=False,
relative=True)
filename = self.compose.get_image_name(
arch, variant, disc_type=disc_type, disc_num=disc_num)
iso_path = self.compose.paths.compose.iso_path(
arch, variant, filename, symlink_to=symlink_isos_to)
relative_iso_path = self.compose.paths.compose.iso_path(
arch, variant, filename, create_dir=False, relative=True)
if os.path.isfile(iso_path):
self.compose.log_warning("Skipping mkisofs, image already exists: %s" % iso_path)
continue
iso_name = os.path.basename(iso_path)
deliverables.append(iso_path)
graft_points = prepare_iso(self.compose, arch, variant, disc_num=disc_num, disc_count=disc_count, split_iso_data=iso_data)
graft_points = prepare_iso(self.compose, arch, variant,
disc_num=disc_num, disc_count=disc_count,
split_iso_data=iso_data)
bootable = self.compose.conf.get("bootable", False)
if arch == "src":
bootable = False
if variant.type != "variant":
bootable = False
bootable = self._is_bootable(variant, arch)
cmd = {
"arch": arch,
@ -131,61 +127,34 @@ class CreateisoPhase(PhaseBase):
}
if os.path.islink(iso_dir):
cmd["mount"] = os.path.abspath(os.path.join(os.path.dirname(iso_dir), os.readlink(iso_dir)))
cmd["mount"] = os.path.abspath(os.path.join(os.path.dirname(iso_dir),
os.readlink(iso_dir)))
chdir_cmd = "cd %s" % pipes.quote(iso_dir)
cmd["cmd"].append(chdir_cmd)
mkisofs_kwargs = {}
cmd['cmd'] = [
'pungi-createiso',
'--output-dir={}'.format(iso_dir),
'--iso-name={}'.format(filename),
'--volid={}'.format(volid),
'--graft-points={}'.format(graft_points),
'--arch={}'.format(arch),
]
if bootable:
buildinstall_method = self.compose.conf["buildinstall_method"]
if buildinstall_method == "lorax":
# TODO: $arch instead of ppc
mkisofs_kwargs["boot_args"] = iso.get_boot_options(arch, "/usr/share/lorax/config_files/ppc")
elif buildinstall_method == "buildinstall":
mkisofs_kwargs["boot_args"] = iso.get_boot_options(arch, "/usr/lib/anaconda-runtime/boot")
cmd['cmd'].extend([
'--bootable',
'--buildinstall-method={}'.format(self.compose.conf['buildinstall_method']),
])
# ppc(64) doesn't seem to support utf-8
if arch in ("ppc", "ppc64", "ppc64le"):
mkisofs_kwargs["input_charset"] = None
if self.compose.supported:
cmd['cmd'].append('--supported')
mkisofs_cmd = iso.get_mkisofs_cmd(iso_name, None, volid=volid, exclude=["./lost+found"], graft_points=graft_points, **mkisofs_kwargs)
mkisofs_cmd = " ".join([pipes.quote(i) for i in mkisofs_cmd])
cmd["cmd"].append(mkisofs_cmd)
if bootable and arch == "x86_64":
isohybrid_cmd = "isohybrid --uefi %s" % pipes.quote(iso_name)
cmd["cmd"].append(isohybrid_cmd)
elif bootable and arch == "i386":
isohybrid_cmd = "isohybrid %s" % pipes.quote(iso_name)
cmd["cmd"].append(isohybrid_cmd)
# implant MD5SUM to iso
isomd5sum_cmd = iso.get_implantisomd5_cmd(iso_name, self.compose.supported)
isomd5sum_cmd = " ".join([pipes.quote(i) for i in isomd5sum_cmd])
cmd["cmd"].append(isomd5sum_cmd)
# create iso manifest
cmd["cmd"].append(iso.get_manifest_cmd(iso_name))
# create jigdo
create_jigdo = self.compose.conf.get("create_jigdo", True)
if create_jigdo:
jigdo = JigdoWrapper(logger=self.compose._logger)
if self.compose.conf.get('create_jigdo', True):
jigdo_dir = self.compose.paths.compose.jigdo_dir(arch, variant)
files = [
{
"path": os_tree,
"label": None,
"uri": None,
}
]
jigdo_cmd = jigdo.get_jigdo_cmd(iso_path, files, output_dir=jigdo_dir, no_servers=True, report="noprogress")
jigdo_cmd = " ".join([pipes.quote(i) for i in jigdo_cmd])
cmd["cmd"].append(jigdo_cmd)
cmd['cmd'].extend([
'--jigdo-dir={}'.format(jigdo_dir),
'--os-tree={}'.format(os_tree),
])
cmd["cmd"] = " && ".join(cmd["cmd"])
commands.append((cmd, variant, arch))
self.compose.notifier.send('createiso-targets', deliverables=deliverables)
@ -228,20 +197,22 @@ class CreateIsoThread(WorkerThread):
runroot = compose.conf.get("runroot", False)
bootable = compose.conf.get("bootable", False)
log_file = compose.paths.log.log_file(cmd["arch"], "createiso-%s" % os.path.basename(cmd["iso_path"]))
log_file = compose.paths.log.log_file(
cmd["arch"], "createiso-%s" % os.path.basename(cmd["iso_path"]))
msg = "Creating ISO (arch: %s, variant: %s): %s" % (cmd["arch"], cmd["variant"], os.path.basename(cmd["iso_path"]))
msg = "Creating ISO (arch: %s, variant: %s): %s" % (
cmd["arch"], cmd["variant"], os.path.basename(cmd["iso_path"]))
self.pool.log_info("[BEGIN] %s" % msg)
if runroot:
# run in a koji build root
packages = ["coreutils", "genisoimage", "isomd5sum", "jigdo", "strace", "lsof"]
extra_packages = {
'lorax': ['lorax', 'pungi'],
'buildinstall': ['anaconda'],
}
if bootable:
buildinstall_method = compose.conf["buildinstall_method"]
if buildinstall_method == "lorax":
packages += ["lorax"]
elif buildinstall_method == "buildinstall":
packages += ["anaconda"]
packages.extend(extra_packages[compose.conf["buildinstall_method"]])
runroot_channel = compose.conf.get("runroot_channel", None)
runroot_tag = compose.conf["runroot_tag"]
@ -260,7 +231,10 @@ class CreateIsoThread(WorkerThread):
# pick random arch from available runroot tag arches
cmd["build_arch"] = random.choice(tag_arches)
koji_cmd = koji_wrapper.get_runroot_cmd(runroot_tag, cmd["build_arch"], cmd["cmd"], channel=runroot_channel, use_shell=True, task_id=True, packages=packages, mounts=mounts)
koji_cmd = koji_wrapper.get_runroot_cmd(
runroot_tag, cmd["build_arch"], cmd["cmd"],
channel=runroot_channel, use_shell=True, task_id=True,
packages=packages, mounts=mounts)
# avoid race conditions?
# Kerberos authentication failed: Permission denied in replay cache code (-1765328215)
@ -269,7 +243,8 @@ class CreateIsoThread(WorkerThread):
output = koji_wrapper.run_runroot_cmd(koji_cmd, log_file=log_file)
if output["retcode"] != 0:
self.fail(compose, cmd)
raise RuntimeError("Runroot task failed: %s. See %s for more details." % (output["task_id"], log_file))
raise RuntimeError("Runroot task failed: %s. See %s for more details."
% (output["task_id"], log_file))
else:
# run locally

197
tests/test_createiso_script.py Executable file
View File

@ -0,0 +1,197 @@
#!/usr/bin/env python
# -*- coding: utf-8 -*-
import unittest
import mock
import os
import sys
sys.path.insert(0, os.path.join(os.path.dirname(__file__), '..'))
from tests import helpers
from pungi import createiso
class OstreeScriptTest(helpers.PungiTestCase):
def assertEqualCalls(self, actual, expected):
self.assertEqual(len(actual), len(expected))
for x, y in zip(actual, expected):
self.assertEqual(x, y)
@mock.patch('kobo.shortcuts.run')
def test_minimal_run(self, run):
createiso.main([
'--output-dir={}/isos'.format(self.topdir),
'--iso-name=DP-1.0-20160405.t.3-x86_64.iso',
'--volid=DP-1.0-20160405.t.3',
'--graft-points=graft-list',
'--arch=x86_64',
])
self.maxDiff = None
self.assertEqual(
run.call_args_list,
[mock.call(['/usr/bin/genisoimage', '-untranslated-filenames',
'-volid', 'DP-1.0-20160405.t.3', '-J', '-joliet-long',
'-rational-rock', '-translation-table',
'-input-charset', 'utf-8', '-x', './lost+found',
'-o', 'DP-1.0-20160405.t.3-x86_64.iso',
'-graft-points', '-path-list', 'graft-list'],
show_cmd=True, stdout=True),
mock.call(['/usr/bin/implantisomd5', 'DP-1.0-20160405.t.3-x86_64.iso'],
show_cmd=True, stdout=True),
mock.call('isoinfo -R -f -i DP-1.0-20160405.t.3-x86_64.iso | grep -v \'/TRANS.TBL$\' | sort >> DP-1.0-20160405.t.3-x86_64.iso.manifest',
show_cmd=True, stdout=True)]
)
@mock.patch('kobo.shortcuts.run')
def test_bootable_run(self, run):
run.return_value = (0, '/usr/share/lorax')
createiso.main([
'--output-dir={}/isos'.format(self.topdir),
'--iso-name=DP-1.0-20160405.t.3-x86_64.iso',
'--volid=DP-1.0-20160405.t.3',
'--graft-points=graft-list',
'--arch=x86_64',
'--buildinstall-method=lorax',
])
self.maxDiff = None
self.assertItemsEqual(
run.call_args_list,
[mock.call(['/usr/bin/genisoimage', '-untranslated-filenames',
'-volid', 'DP-1.0-20160405.t.3', '-J', '-joliet-long',
'-rational-rock', '-translation-table',
'-input-charset', 'utf-8', '-x', './lost+found',
'-b', 'isolinux/isolinux.bin', '-c', 'isolinux/boot.cat',
'-no-emul-boot',
'-boot-load-size', '4', '-boot-info-table',
'-eltorito-alt-boot', '-e', 'images/efiboot.img',
'-no-emul-boot',
'-o', 'DP-1.0-20160405.t.3-x86_64.iso',
'-graft-points', '-path-list', 'graft-list'],
show_cmd=True, stdout=True),
mock.call(['pungi-pylorax-find-templates', '/usr/share/lorax'],
show_cmd=True, stdout=True),
mock.call(['/usr/bin/implantisomd5', 'DP-1.0-20160405.t.3-x86_64.iso'],
show_cmd=True, stdout=True),
mock.call('isoinfo -R -f -i DP-1.0-20160405.t.3-x86_64.iso | grep -v \'/TRANS.TBL$\' | sort >> DP-1.0-20160405.t.3-x86_64.iso.manifest',
show_cmd=True, stdout=True)]
)
@mock.patch('kobo.shortcuts.run')
def test_bootable_run_ppc64(self, run):
run.return_value = (0, '/usr/share/lorax')
createiso.main([
'--output-dir={}/isos'.format(self.topdir),
'--iso-name=DP-1.0-20160405.t.3-ppc64.iso',
'--volid=DP-1.0-20160405.t.3',
'--graft-points=graft-list',
'--arch=ppc64',
'--buildinstall-method=lorax',
])
self.maxDiff = None
self.assertItemsEqual(
run.call_args_list,
[mock.call(['/usr/bin/genisoimage', '-untranslated-filenames',
'-volid', 'DP-1.0-20160405.t.3', '-J', '-joliet-long',
'-rational-rock', '-translation-table',
'-x', './lost+found',
'-part', '-hfs', '-r', '-l', '-sysid', 'PPC', '-no-desktop',
'-allow-multidot', '-chrp-boot', '-map', '/usr/share/lorax/config_files/ppc/mapping',
'-hfs-bless', '/ppc/mac',
'-o', 'DP-1.0-20160405.t.3-ppc64.iso',
'-graft-points', '-path-list', 'graft-list'],
show_cmd=True, stdout=True),
mock.call(['pungi-pylorax-find-templates', '/usr/share/lorax'],
show_cmd=True, stdout=True),
mock.call(['/usr/bin/implantisomd5', 'DP-1.0-20160405.t.3-ppc64.iso'],
show_cmd=True, stdout=True),
mock.call('isoinfo -R -f -i DP-1.0-20160405.t.3-ppc64.iso | grep -v \'/TRANS.TBL$\' | sort >> DP-1.0-20160405.t.3-ppc64.iso.manifest',
show_cmd=True, stdout=True)]
)
@mock.patch('kobo.shortcuts.run')
def test_bootable_run_buildinstall(self, run):
createiso.main([
'--output-dir={}/isos'.format(self.topdir),
'--iso-name=DP-1.0-20160405.t.3-ppc64.iso',
'--volid=DP-1.0-20160405.t.3',
'--graft-points=graft-list',
'--arch=ppc64',
'--buildinstall-method=buildinstall',
])
self.maxDiff = None
self.assertItemsEqual(
run.call_args_list,
[mock.call(['/usr/bin/genisoimage', '-untranslated-filenames',
'-volid', 'DP-1.0-20160405.t.3', '-J', '-joliet-long',
'-rational-rock', '-translation-table',
'-x', './lost+found',
'-part', '-hfs', '-r', '-l', '-sysid', 'PPC', '-no-desktop',
'-allow-multidot', '-chrp-boot',
'-map', '/usr/lib/anaconda-runtime/boot/mapping',
'-hfs-bless', '/ppc/mac',
'-o', 'DP-1.0-20160405.t.3-ppc64.iso',
'-graft-points', '-path-list', 'graft-list'],
show_cmd=True, stdout=True),
mock.call(['/usr/bin/implantisomd5', 'DP-1.0-20160405.t.3-ppc64.iso'],
show_cmd=True, stdout=True),
mock.call('isoinfo -R -f -i DP-1.0-20160405.t.3-ppc64.iso | grep -v \'/TRANS.TBL$\' | sort >> DP-1.0-20160405.t.3-ppc64.iso.manifest',
show_cmd=True, stdout=True)]
)
@mock.patch('sys.stderr')
@mock.patch('kobo.shortcuts.run')
def test_run_with_jigdo_bad_args(self, run, stderr):
with self.assertRaises(SystemExit):
createiso.main([
'--output-dir={}/isos'.format(self.topdir),
'--iso-name=DP-1.0-20160405.t.3-x86_64.iso',
'--volid=DP-1.0-20160405.t.3',
'--graft-points=graft-list',
'--arch=x86_64',
'--jigdo-dir={}/jigdo'.format(self.topdir),
])
@mock.patch('kobo.shortcuts.run')
def test_run_with_jigdo(self, run):
createiso.main([
'--output-dir={}/isos'.format(self.topdir),
'--iso-name=DP-1.0-20160405.t.3-x86_64.iso',
'--volid=DP-1.0-20160405.t.3',
'--graft-points=graft-list',
'--arch=x86_64',
'--jigdo-dir={}/jigdo'.format(self.topdir),
'--os-tree={}/os'.format(self.topdir),
])
self.maxDiff = None
self.assertItemsEqual(
run.call_args_list,
[mock.call(['/usr/bin/genisoimage', '-untranslated-filenames',
'-volid', 'DP-1.0-20160405.t.3', '-J', '-joliet-long',
'-rational-rock', '-translation-table',
'-input-charset', 'utf-8', '-x', './lost+found',
'-o', 'DP-1.0-20160405.t.3-x86_64.iso',
'-graft-points', '-path-list', 'graft-list'],
show_cmd=True, stdout=True),
mock.call(['/usr/bin/implantisomd5', 'DP-1.0-20160405.t.3-x86_64.iso'],
show_cmd=True, stdout=True),
mock.call('isoinfo -R -f -i DP-1.0-20160405.t.3-x86_64.iso | grep -v \'/TRANS.TBL$\' | sort >> DP-1.0-20160405.t.3-x86_64.iso.manifest',
show_cmd=True, stdout=True),
mock.call(['jigdo-file', 'make-template', '--force',
'--image={}/isos/DP-1.0-20160405.t.3-x86_64.iso'.format(self.topdir),
'--jigdo={}/jigdo/DP-1.0-20160405.t.3-x86_64.iso.jigdo'.format(self.topdir),
'--template={}/jigdo/DP-1.0-20160405.t.3-x86_64.iso.template'.format(self.topdir),
'--no-servers-section', '--report=noprogress', self.topdir + '/os//'],
show_cmd=True, stdout=True)]
)
if __name__ == '__main__':
unittest.main()