fcf1442f71
When `link_type = "symlink"` is used, the packages are in fact symlinks to /mnt/koji. When graft points file is generated, the paths in this graft points file point to symlinks and therefore symlinks are copied into the generated ISO file instead of real files. In this commit, the code to generate the graft points file is changed so it resolves the symlink to real file stored on /mnt/koji. To make this code safer, it does such resolving only in case the symlink points outside of `compose.paths.compose.topdir()`. Therefore you can still generate ISO file with symlink pointing to file stored within the ISO file itself, although this is not done currently afaik. The main reason for this is to be able to generate ISO files even without hardlinks (which would need read-write access on /mnt/koji) and without copying all the packages from /mnt/koji to local storage. Signed-off-by: Jan Kaluza <jkaluza@redhat.com>
295 lines
11 KiB
Python
295 lines
11 KiB
Python
# -*- coding: utf-8 -*-
|
|
|
|
|
|
# This program is free software; you can redistribute it and/or modify
|
|
# it under the terms of the GNU General Public License as published by
|
|
# the Free Software Foundation; version 2 of the License.
|
|
#
|
|
# This program is distributed in the hope that it will be useful,
|
|
# but WITHOUT ANY WARRANTY; without even the implied warranty of
|
|
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
|
# GNU Library General Public License for more details.
|
|
#
|
|
# You should have received a copy of the GNU General Public License
|
|
# along with this program; if not, see <https://gnu.org/licenses/>.
|
|
|
|
import os
|
|
|
|
from kobo.shortcuts import force_list
|
|
from kobo.threads import ThreadPool, WorkerThread
|
|
import productmd.treeinfo
|
|
from productmd.extra_files import ExtraFiles
|
|
|
|
from pungi import createiso
|
|
from pungi import metadata
|
|
from pungi.phases.base import ConfigGuardedPhase, PhaseBase, PhaseLoggerMixin
|
|
from pungi.phases.createiso import (add_iso_to_metadata, copy_boot_images,
|
|
run_createiso_command, load_and_tweak_treeinfo)
|
|
from pungi.util import failable, get_format_substs, get_variant_data, get_volid
|
|
from pungi.wrappers import iso
|
|
from pungi.wrappers.scm import get_dir_from_scm, get_file_from_scm
|
|
|
|
|
|
class ExtraIsosPhase(PhaseLoggerMixin, ConfigGuardedPhase, PhaseBase):
|
|
name = "extra_isos"
|
|
|
|
def __init__(self, compose):
|
|
super(ExtraIsosPhase, self).__init__(compose)
|
|
self.pool = ThreadPool(logger=self.logger)
|
|
|
|
def validate(self):
|
|
for variant in self.compose.get_variants(types=['variant']):
|
|
for config in get_variant_data(self.compose.conf, self.name, variant):
|
|
extra_arches = set(config.get('arches', [])) - set(variant.arches)
|
|
if extra_arches:
|
|
self.compose.log_warning(
|
|
'Extra iso config for %s mentions non-existing arches: %s'
|
|
% (variant, ', '.join(sorted(extra_arches))))
|
|
|
|
def run(self):
|
|
commands = []
|
|
|
|
for variant in self.compose.get_variants(types=['variant']):
|
|
for config in get_variant_data(self.compose.conf, self.name, variant):
|
|
arches = set(variant.arches)
|
|
if config.get('arches'):
|
|
arches &= set(config['arches'])
|
|
if not config['skip_src']:
|
|
arches.add('src')
|
|
for arch in sorted(arches):
|
|
commands.append((config, variant, arch))
|
|
|
|
for (config, variant, arch) in commands:
|
|
self.pool.add(ExtraIsosThread(self.pool))
|
|
self.pool.queue_put((self.compose, config, variant, arch))
|
|
|
|
self.pool.start()
|
|
|
|
|
|
class ExtraIsosThread(WorkerThread):
|
|
def process(self, item, num):
|
|
self.num = num
|
|
compose, config, variant, arch = item
|
|
can_fail = arch in config.get('failable_arches', [])
|
|
with failable(compose, can_fail, variant, arch, 'extra_iso', logger=self.pool._logger):
|
|
self.worker(compose, config, variant, arch)
|
|
|
|
def worker(self, compose, config, variant, arch):
|
|
filename = get_filename(compose, variant, arch, config.get('filename'))
|
|
volid = get_volume_id(compose, variant, arch, config.get('volid', []))
|
|
iso_dir = compose.paths.compose.iso_dir(arch, variant)
|
|
iso_path = os.path.join(iso_dir, filename)
|
|
|
|
prepare_media_metadata(compose, variant, arch)
|
|
|
|
msg = "Creating ISO (arch: %s, variant: %s): %s" % (arch, variant, filename)
|
|
self.pool.log_info("[BEGIN] %s" % msg)
|
|
|
|
get_extra_files(compose, variant, arch, config.get('extra_files', []))
|
|
|
|
bootable = arch != "src" and bool(compose.conf.get('buildinstall_method'))
|
|
|
|
graft_points = get_iso_contents(
|
|
compose,
|
|
variant,
|
|
arch,
|
|
config['include_variants'],
|
|
filename,
|
|
bootable=bootable,
|
|
inherit_extra_files=config.get("inherit_extra_files", False),
|
|
)
|
|
|
|
opts = createiso.CreateIsoOpts(
|
|
output_dir=iso_dir,
|
|
iso_name=filename,
|
|
volid=volid,
|
|
graft_points=graft_points,
|
|
arch=arch,
|
|
supported=compose.supported,
|
|
hfs_compat=compose.conf["iso_hfs_ppc64le_compatible"],
|
|
)
|
|
if compose.conf['create_jigdo']:
|
|
jigdo_dir = compose.paths.compose.jigdo_dir(arch, variant)
|
|
os_tree = compose.paths.compose.os_tree(arch, variant)
|
|
opts = opts._replace(jigdo_dir=jigdo_dir, os_tree=os_tree)
|
|
|
|
if bootable:
|
|
opts = opts._replace(buildinstall_method=compose.conf['buildinstall_method'])
|
|
|
|
script_file = os.path.join(compose.paths.work.tmp_dir(arch, variant),
|
|
'extraiso-%s.sh' % filename)
|
|
with open(script_file, 'w') as f:
|
|
createiso.write_script(opts, f)
|
|
|
|
run_createiso_command(self.num, compose, bootable, arch,
|
|
['bash', script_file], [compose.topdir],
|
|
log_file=compose.paths.log.log_file(
|
|
arch, "extraiso-%s" % os.path.basename(iso_path)),
|
|
with_jigdo=compose.conf['create_jigdo'])
|
|
|
|
img = add_iso_to_metadata(
|
|
compose,
|
|
variant,
|
|
arch,
|
|
iso_path,
|
|
bootable,
|
|
additional_variants=config["include_variants"],
|
|
)
|
|
img._max_size = config.get("max_size")
|
|
|
|
self.pool.log_info("[DONE ] %s" % msg)
|
|
|
|
|
|
def get_extra_files(compose, variant, arch, extra_files):
|
|
"""Clone the configured files into a directory from where they can be
|
|
included in the ISO.
|
|
"""
|
|
extra_files_dir = compose.paths.work.extra_iso_extra_files_dir(arch, variant)
|
|
filelist = []
|
|
for scm_dict in extra_files:
|
|
getter = get_file_from_scm if 'file' in scm_dict else get_dir_from_scm
|
|
target = scm_dict.get("target", "").lstrip("/")
|
|
target_path = os.path.join(extra_files_dir, target).rstrip("/")
|
|
filelist.extend(
|
|
os.path.join(target, f)
|
|
for f in getter(scm_dict, target_path, compose=compose)
|
|
)
|
|
|
|
if filelist:
|
|
metadata.populate_extra_files_metadata(
|
|
ExtraFiles(),
|
|
variant,
|
|
arch,
|
|
extra_files_dir,
|
|
filelist,
|
|
compose.conf["media_checksums"],
|
|
)
|
|
|
|
|
|
def get_iso_contents(
|
|
compose, variant, arch, include_variants, filename, bootable, inherit_extra_files
|
|
):
|
|
"""Find all files that should be on the ISO. For bootable image we start
|
|
with the boot configuration. Then for each variant we add packages,
|
|
repodata and extra files. Finally we add top-level extra files.
|
|
"""
|
|
iso_dir = compose.paths.work.iso_dir(arch, filename)
|
|
|
|
files = {}
|
|
if bootable:
|
|
buildinstall_dir = compose.paths.work.buildinstall_dir(arch, create_dir=False)
|
|
if compose.conf['buildinstall_method'] == 'lorax':
|
|
buildinstall_dir = os.path.join(buildinstall_dir, variant.uid)
|
|
|
|
copy_boot_images(buildinstall_dir, iso_dir)
|
|
files = iso.get_graft_points(compose, [buildinstall_dir, iso_dir])
|
|
|
|
# We need to point efiboot.img to compose/ tree, because it was
|
|
# modified in buildinstall phase and the file in work/ has different
|
|
# checksum to what is in the .treeinfo.
|
|
if "images/efiboot.img" in files:
|
|
files["images/efiboot.img"] = os.path.join(
|
|
compose.paths.compose.os_tree(arch, variant), "images/efiboot.img"
|
|
)
|
|
|
|
variants = [variant.uid] + include_variants
|
|
for variant_uid in variants:
|
|
var = compose.all_variants[variant_uid]
|
|
|
|
# Get packages...
|
|
package_dir = compose.paths.compose.packages(arch, var)
|
|
for k, v in iso.get_graft_points(compose, [package_dir]).items():
|
|
files[os.path.join(var.uid, 'Packages', k)] = v
|
|
|
|
# Get repodata...
|
|
tree_dir = compose.paths.compose.repository(arch, var)
|
|
repo_dir = os.path.join(tree_dir, 'repodata')
|
|
for k, v in iso.get_graft_points(compose, [repo_dir]).items():
|
|
files[os.path.join(var.uid, 'repodata', k)] = v
|
|
|
|
if inherit_extra_files:
|
|
# Get extra files...
|
|
extra_files_dir = compose.paths.work.extra_files_dir(arch, var)
|
|
for k, v in iso.get_graft_points(compose, [extra_files_dir]).items():
|
|
files[os.path.join(var.uid, k)] = v
|
|
|
|
extra_files_dir = compose.paths.work.extra_iso_extra_files_dir(arch, variant)
|
|
|
|
original_treeinfo = os.path.join(
|
|
compose.paths.compose.os_tree(arch=arch, variant=variant), ".treeinfo"
|
|
)
|
|
tweak_treeinfo(
|
|
compose,
|
|
include_variants,
|
|
original_treeinfo,
|
|
os.path.join(extra_files_dir, ".treeinfo"),
|
|
)
|
|
|
|
# Add extra files specific for the ISO
|
|
files.update(iso.get_graft_points(compose, [extra_files_dir]))
|
|
|
|
gp = "%s-graft-points" % iso_dir
|
|
iso.write_graft_points(gp, files, exclude=["*/lost+found", "*/boot.iso"])
|
|
return gp
|
|
|
|
|
|
def tweak_treeinfo(compose, include_variants, source_file, dest_file):
|
|
ti = load_and_tweak_treeinfo(source_file)
|
|
for variant_uid in include_variants:
|
|
variant = compose.all_variants[variant_uid]
|
|
var = productmd.treeinfo.Variant(ti)
|
|
var.id = variant.id
|
|
var.uid = variant.uid
|
|
var.name = variant.name
|
|
var.type = variant.type
|
|
ti.variants.add(var)
|
|
|
|
for variant_id in ti.variants:
|
|
var = ti.variants[variant_id]
|
|
var.paths.packages = os.path.join(var.uid, "Packages")
|
|
var.paths.repository = var.uid
|
|
|
|
ti.dump(dest_file)
|
|
|
|
|
|
def get_filename(compose, variant, arch, format):
|
|
disc_type = compose.conf['disc_types'].get('dvd', 'dvd')
|
|
base_filename = compose.get_image_name(
|
|
arch, variant, disc_type=disc_type, disc_num=1)
|
|
if not format:
|
|
return base_filename
|
|
kwargs = {
|
|
'arch': arch,
|
|
'disc_type': disc_type,
|
|
'disc_num': 1,
|
|
'suffix': '.iso',
|
|
'filename': base_filename,
|
|
'variant': variant,
|
|
}
|
|
args = get_format_substs(compose, **kwargs)
|
|
try:
|
|
return (format % args).format(**args)
|
|
except KeyError as err:
|
|
raise RuntimeError('Failed to create image name: unknown format element: %s' % err)
|
|
|
|
|
|
def get_volume_id(compose, variant, arch, formats):
|
|
disc_type = compose.conf['disc_types'].get('dvd', 'dvd')
|
|
# Get volume ID for regular ISO so that we can substitute it in.
|
|
volid = get_volid(compose, arch, variant, disc_type=disc_type)
|
|
return get_volid(compose, arch, variant, disc_type=disc_type,
|
|
formats=force_list(formats), volid=volid)
|
|
|
|
|
|
def prepare_media_metadata(compose, variant, arch):
|
|
"""Write a .discinfo and media.repo files to a directory that will be
|
|
included on the ISO. It's possible to overwrite the files by using extra
|
|
files.
|
|
"""
|
|
md_dir = compose.paths.work.extra_iso_extra_files_dir(arch, variant)
|
|
description = metadata.get_description(compose, variant, arch)
|
|
metadata.create_media_repo(
|
|
os.path.join(md_dir, "media.repo"), description, timestamp=None
|
|
)
|
|
metadata.create_discinfo(os.path.join(md_dir, ".discinfo"), description, arch)
|