321 lines
11 KiB
Python
321 lines
11 KiB
Python
# -*- coding: utf-8 -*-
|
|
|
|
|
|
# This program is free software; you can redistribute it and/or modify
|
|
# it under the terms of the GNU General Public License as published by
|
|
# the Free Software Foundation; version 2 of the License.
|
|
#
|
|
# This program is distributed in the hope that it will be useful,
|
|
# but WITHOUT ANY WARRANTY; without even the implied warranty of
|
|
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
|
# GNU Library General Public License for more details.
|
|
#
|
|
# You should have received a copy of the GNU General Public License
|
|
# along with this program; if not, see <https://gnu.org/licenses/>.
|
|
|
|
import os
|
|
|
|
from kobo.shortcuts import force_list
|
|
from kobo.threads import ThreadPool, WorkerThread
|
|
import productmd.treeinfo
|
|
from productmd.extra_files import ExtraFiles
|
|
|
|
from pungi import createiso
|
|
from pungi import metadata
|
|
from pungi.phases.base import ConfigGuardedPhase, PhaseBase, PhaseLoggerMixin
|
|
from pungi.phases.createiso import (
|
|
add_iso_to_metadata,
|
|
copy_boot_images,
|
|
run_createiso_command,
|
|
load_and_tweak_treeinfo,
|
|
)
|
|
from pungi.util import failable, get_format_substs, get_variant_data, get_volid
|
|
from pungi.wrappers import iso
|
|
from pungi.wrappers.scm import get_dir_from_scm, get_file_from_scm
|
|
|
|
|
|
class ExtraIsosPhase(PhaseLoggerMixin, ConfigGuardedPhase, PhaseBase):
|
|
name = "extra_isos"
|
|
|
|
def __init__(self, compose):
|
|
super(ExtraIsosPhase, self).__init__(compose)
|
|
self.pool = ThreadPool(logger=self.logger)
|
|
|
|
def validate(self):
|
|
for variant in self.compose.get_variants(types=["variant"]):
|
|
for config in get_variant_data(self.compose.conf, self.name, variant):
|
|
extra_arches = set(config.get("arches", [])) - set(variant.arches)
|
|
if extra_arches:
|
|
self.compose.log_warning(
|
|
"Extra iso config for %s mentions non-existing arches: %s"
|
|
% (variant, ", ".join(sorted(extra_arches)))
|
|
)
|
|
|
|
def run(self):
|
|
commands = []
|
|
|
|
for variant in self.compose.get_variants(types=["variant"]):
|
|
for config in get_variant_data(self.compose.conf, self.name, variant):
|
|
arches = set(variant.arches)
|
|
if config.get("arches"):
|
|
arches &= set(config["arches"])
|
|
if not config["skip_src"]:
|
|
arches.add("src")
|
|
for arch in sorted(arches):
|
|
commands.append((config, variant, arch))
|
|
|
|
for (config, variant, arch) in commands:
|
|
self.pool.add(ExtraIsosThread(self.pool))
|
|
self.pool.queue_put((self.compose, config, variant, arch))
|
|
|
|
self.pool.start()
|
|
|
|
|
|
class ExtraIsosThread(WorkerThread):
|
|
def process(self, item, num):
|
|
self.num = num
|
|
compose, config, variant, arch = item
|
|
can_fail = arch in config.get("failable_arches", [])
|
|
with failable(
|
|
compose, can_fail, variant, arch, "extra_iso", logger=self.pool._logger
|
|
):
|
|
self.worker(compose, config, variant, arch)
|
|
|
|
def worker(self, compose, config, variant, arch):
|
|
filename = get_filename(compose, variant, arch, config.get("filename"))
|
|
volid = get_volume_id(compose, variant, arch, config.get("volid", []))
|
|
iso_dir = compose.paths.compose.iso_dir(arch, variant)
|
|
iso_path = os.path.join(iso_dir, filename)
|
|
|
|
prepare_media_metadata(compose, variant, arch)
|
|
|
|
msg = "Creating ISO (arch: %s, variant: %s): %s" % (arch, variant, filename)
|
|
self.pool.log_info("[BEGIN] %s" % msg)
|
|
|
|
get_extra_files(compose, variant, arch, config.get("extra_files", []))
|
|
|
|
bootable = arch != "src" and bool(compose.conf.get("buildinstall_method"))
|
|
|
|
graft_points = get_iso_contents(
|
|
compose,
|
|
variant,
|
|
arch,
|
|
config["include_variants"],
|
|
filename,
|
|
bootable=bootable,
|
|
inherit_extra_files=config.get("inherit_extra_files", False),
|
|
)
|
|
|
|
opts = createiso.CreateIsoOpts(
|
|
output_dir=iso_dir,
|
|
iso_name=filename,
|
|
volid=volid,
|
|
graft_points=graft_points,
|
|
arch=arch,
|
|
supported=compose.supported,
|
|
hfs_compat=compose.conf["iso_hfs_ppc64le_compatible"],
|
|
)
|
|
if compose.conf["create_jigdo"]:
|
|
jigdo_dir = compose.paths.compose.jigdo_dir(arch, variant)
|
|
os_tree = compose.paths.compose.os_tree(arch, variant)
|
|
opts = opts._replace(jigdo_dir=jigdo_dir, os_tree=os_tree)
|
|
|
|
if bootable:
|
|
opts = opts._replace(
|
|
buildinstall_method=compose.conf["buildinstall_method"]
|
|
)
|
|
|
|
script_file = os.path.join(
|
|
compose.paths.work.tmp_dir(arch, variant), "extraiso-%s.sh" % filename
|
|
)
|
|
with open(script_file, "w") as f:
|
|
createiso.write_script(opts, f)
|
|
|
|
run_createiso_command(
|
|
self.num,
|
|
compose,
|
|
bootable,
|
|
arch,
|
|
["bash", script_file],
|
|
[compose.topdir],
|
|
log_file=compose.paths.log.log_file(
|
|
arch, "extraiso-%s" % os.path.basename(iso_path)
|
|
),
|
|
with_jigdo=compose.conf["create_jigdo"],
|
|
)
|
|
|
|
img = add_iso_to_metadata(
|
|
compose,
|
|
variant,
|
|
arch,
|
|
iso_path,
|
|
bootable,
|
|
additional_variants=config["include_variants"],
|
|
)
|
|
img._max_size = config.get("max_size")
|
|
|
|
self.pool.log_info("[DONE ] %s" % msg)
|
|
|
|
|
|
def get_extra_files(compose, variant, arch, extra_files):
|
|
"""Clone the configured files into a directory from where they can be
|
|
included in the ISO.
|
|
"""
|
|
extra_files_dir = compose.paths.work.extra_iso_extra_files_dir(arch, variant)
|
|
filelist = []
|
|
for scm_dict in extra_files:
|
|
getter = get_file_from_scm if "file" in scm_dict else get_dir_from_scm
|
|
target = scm_dict.get("target", "").lstrip("/")
|
|
target_path = os.path.join(extra_files_dir, target).rstrip("/")
|
|
filelist.extend(
|
|
os.path.join(target, f)
|
|
for f in getter(scm_dict, target_path, compose=compose)
|
|
)
|
|
|
|
if filelist:
|
|
metadata.populate_extra_files_metadata(
|
|
ExtraFiles(),
|
|
variant,
|
|
arch,
|
|
extra_files_dir,
|
|
filelist,
|
|
compose.conf["media_checksums"],
|
|
)
|
|
|
|
|
|
def get_iso_contents(
|
|
compose, variant, arch, include_variants, filename, bootable, inherit_extra_files
|
|
):
|
|
"""Find all files that should be on the ISO. For bootable image we start
|
|
with the boot configuration. Then for each variant we add packages,
|
|
repodata and extra files. Finally we add top-level extra files.
|
|
"""
|
|
iso_dir = compose.paths.work.iso_dir(arch, filename)
|
|
|
|
files = {}
|
|
if bootable:
|
|
buildinstall_dir = compose.paths.work.buildinstall_dir(arch, create_dir=False)
|
|
if compose.conf["buildinstall_method"] == "lorax":
|
|
buildinstall_dir = os.path.join(buildinstall_dir, variant.uid)
|
|
|
|
copy_boot_images(buildinstall_dir, iso_dir)
|
|
files = iso.get_graft_points(compose, [buildinstall_dir, iso_dir])
|
|
|
|
# We need to point efiboot.img to compose/ tree, because it was
|
|
# modified in buildinstall phase and the file in work/ has different
|
|
# checksum to what is in the .treeinfo.
|
|
if "images/efiboot.img" in files:
|
|
files["images/efiboot.img"] = os.path.join(
|
|
compose.paths.compose.os_tree(arch, variant), "images/efiboot.img"
|
|
)
|
|
|
|
variants = [variant.uid] + include_variants
|
|
for variant_uid in variants:
|
|
var = compose.all_variants[variant_uid]
|
|
|
|
# Get packages...
|
|
package_dir = compose.paths.compose.packages(arch, var)
|
|
for k, v in iso.get_graft_points(compose, [package_dir]).items():
|
|
files[os.path.join(var.uid, "Packages", k)] = v
|
|
|
|
# Get repodata...
|
|
tree_dir = compose.paths.compose.repository(arch, var)
|
|
repo_dir = os.path.join(tree_dir, "repodata")
|
|
for k, v in iso.get_graft_points(compose, [repo_dir]).items():
|
|
files[os.path.join(var.uid, "repodata", k)] = v
|
|
|
|
if inherit_extra_files:
|
|
# Get extra files...
|
|
extra_files_dir = compose.paths.work.extra_files_dir(arch, var)
|
|
for k, v in iso.get_graft_points(compose, [extra_files_dir]).items():
|
|
files[os.path.join(var.uid, k)] = v
|
|
|
|
extra_files_dir = compose.paths.work.extra_iso_extra_files_dir(arch, variant)
|
|
|
|
original_treeinfo = os.path.join(
|
|
compose.paths.compose.os_tree(arch=arch, variant=variant), ".treeinfo"
|
|
)
|
|
tweak_treeinfo(
|
|
compose,
|
|
include_variants,
|
|
original_treeinfo,
|
|
os.path.join(extra_files_dir, ".treeinfo"),
|
|
)
|
|
|
|
# Add extra files specific for the ISO
|
|
files.update(iso.get_graft_points(compose, [extra_files_dir]))
|
|
|
|
gp = "%s-graft-points" % iso_dir
|
|
iso.write_graft_points(gp, files, exclude=["*/lost+found", "*/boot.iso"])
|
|
return gp
|
|
|
|
|
|
def tweak_treeinfo(compose, include_variants, source_file, dest_file):
|
|
ti = load_and_tweak_treeinfo(source_file)
|
|
for variant_uid in include_variants:
|
|
variant = compose.all_variants[variant_uid]
|
|
var = productmd.treeinfo.Variant(ti)
|
|
var.id = variant.id
|
|
var.uid = variant.uid
|
|
var.name = variant.name
|
|
var.type = variant.type
|
|
ti.variants.add(var)
|
|
|
|
for variant_id in ti.variants:
|
|
var = ti.variants[variant_id]
|
|
var.paths.packages = os.path.join(var.uid, "Packages")
|
|
var.paths.repository = var.uid
|
|
|
|
ti.dump(dest_file)
|
|
|
|
|
|
def get_filename(compose, variant, arch, format):
|
|
disc_type = compose.conf["disc_types"].get("dvd", "dvd")
|
|
base_filename = compose.get_image_name(
|
|
arch, variant, disc_type=disc_type, disc_num=1
|
|
)
|
|
if not format:
|
|
return base_filename
|
|
kwargs = {
|
|
"arch": arch,
|
|
"disc_type": disc_type,
|
|
"disc_num": 1,
|
|
"suffix": ".iso",
|
|
"filename": base_filename,
|
|
"variant": variant,
|
|
}
|
|
args = get_format_substs(compose, **kwargs)
|
|
try:
|
|
return (format % args).format(**args)
|
|
except KeyError as err:
|
|
raise RuntimeError(
|
|
"Failed to create image name: unknown format element: %s" % err
|
|
)
|
|
|
|
|
|
def get_volume_id(compose, variant, arch, formats):
|
|
disc_type = compose.conf["disc_types"].get("dvd", "dvd")
|
|
# Get volume ID for regular ISO so that we can substitute it in.
|
|
volid = get_volid(compose, arch, variant, disc_type=disc_type)
|
|
return get_volid(
|
|
compose,
|
|
arch,
|
|
variant,
|
|
disc_type=disc_type,
|
|
formats=force_list(formats),
|
|
volid=volid,
|
|
)
|
|
|
|
|
|
def prepare_media_metadata(compose, variant, arch):
|
|
"""Write a .discinfo and media.repo files to a directory that will be
|
|
included on the ISO. It's possible to overwrite the files by using extra
|
|
files.
|
|
"""
|
|
md_dir = compose.paths.work.extra_iso_extra_files_dir(arch, variant)
|
|
description = metadata.get_description(compose, variant, arch)
|
|
metadata.create_media_repo(
|
|
os.path.join(md_dir, "media.repo"), description, timestamp=None
|
|
)
|
|
metadata.create_discinfo(os.path.join(md_dir, ".discinfo"), description, arch)
|