2016-03-23 09:40:16 +00:00
|
|
|
# -*- coding: utf-8 -*-
|
|
|
|
|
|
|
|
import os
|
|
|
|
from kobo.threads import ThreadPool, WorkerThread
|
|
|
|
import shutil
|
|
|
|
from productmd import images
|
2017-09-21 07:29:53 +00:00
|
|
|
from six.moves import shlex_quote
|
2016-04-08 10:48:42 +00:00
|
|
|
from kobo import shortcuts
|
2016-03-23 09:40:16 +00:00
|
|
|
|
2016-11-18 20:33:03 +00:00
|
|
|
from .base import ConfigGuardedPhase, PhaseLoggerMixin
|
2016-03-23 09:40:16 +00:00
|
|
|
from .. import util
|
2019-01-28 12:21:31 +00:00
|
|
|
from ..arch import get_valid_arches
|
2018-02-20 13:29:49 +00:00
|
|
|
from ..util import get_volid, get_repo_urls, version_generator, translate_path
|
2019-03-21 09:58:13 +00:00
|
|
|
from ..wrappers import iso, lorax, scm
|
|
|
|
from ..runroot import Runroot
|
2016-03-23 09:40:16 +00:00
|
|
|
|
|
|
|
|
2016-11-18 20:33:03 +00:00
|
|
|
class OstreeInstallerPhase(PhaseLoggerMixin, ConfigGuardedPhase):
|
2016-04-05 07:13:01 +00:00
|
|
|
name = 'ostree_installer'
|
2016-03-23 09:40:16 +00:00
|
|
|
|
2019-08-01 14:36:54 +00:00
|
|
|
def __init__(self, compose, buildinstall_phase, pkgset_phase=None):
|
2016-04-05 07:13:01 +00:00
|
|
|
super(OstreeInstallerPhase, self).__init__(compose)
|
2016-11-18 20:33:03 +00:00
|
|
|
self.pool = ThreadPool(logger=self.logger)
|
2018-05-07 08:47:48 +00:00
|
|
|
self.bi = buildinstall_phase
|
2019-08-01 14:36:54 +00:00
|
|
|
self.pkgset_phase = pkgset_phase
|
2016-03-23 09:40:16 +00:00
|
|
|
|
2017-09-07 13:54:56 +00:00
|
|
|
def validate(self):
|
|
|
|
errors = []
|
|
|
|
|
2018-05-24 11:18:57 +00:00
|
|
|
if not self.compose.conf['ostree_installer_overwrite'] and not self.bi.skip():
|
2018-05-07 08:47:48 +00:00
|
|
|
for variant in self.compose.get_variants():
|
|
|
|
for arch in variant.arches:
|
|
|
|
conf = util.get_arch_variant_data(self.compose.conf, self.name,
|
|
|
|
arch, variant)
|
|
|
|
if conf and not variant.is_empty:
|
|
|
|
errors.append('Can not generate ostree installer for %s.%s: '
|
|
|
|
'it has buildinstall running already and the '
|
|
|
|
'files would clash.' % (variant.uid, arch))
|
2017-09-07 13:54:56 +00:00
|
|
|
|
|
|
|
if errors:
|
|
|
|
raise ValueError('\n'.join(errors))
|
|
|
|
|
2019-08-01 14:36:54 +00:00
|
|
|
def get_repos(self):
|
|
|
|
return [
|
|
|
|
translate_path(
|
|
|
|
self.compose,
|
|
|
|
self.compose.paths.work.pkgset_repo(pkgset.name, "$basearch"),
|
|
|
|
)
|
|
|
|
for pkgset in self.pkgset_phase.package_sets
|
|
|
|
]
|
|
|
|
|
2016-03-23 09:40:16 +00:00
|
|
|
def run(self):
|
|
|
|
for variant in self.compose.get_variants():
|
|
|
|
for arch in variant.arches:
|
2017-08-18 07:33:51 +00:00
|
|
|
for conf in self.get_config_block(variant, arch):
|
2019-08-01 14:36:54 +00:00
|
|
|
self.pool.add(OstreeInstallerThread(self.pool, self.get_repos()))
|
2016-03-23 09:40:16 +00:00
|
|
|
self.pool.queue_put((self.compose, variant, arch, conf))
|
|
|
|
|
|
|
|
self.pool.start()
|
|
|
|
|
|
|
|
|
2016-04-05 07:13:01 +00:00
|
|
|
class OstreeInstallerThread(WorkerThread):
|
2019-08-01 14:36:54 +00:00
|
|
|
def __init__(self, pool, baseurls):
|
|
|
|
super(OstreeInstallerThread, self).__init__(pool)
|
|
|
|
self.baseurls = baseurls
|
|
|
|
|
2016-03-23 09:40:16 +00:00
|
|
|
def process(self, item, num):
|
|
|
|
compose, variant, arch, config = item
|
|
|
|
self.num = num
|
2016-06-24 07:44:40 +00:00
|
|
|
failable_arches = config.get('failable', [])
|
|
|
|
self.can_fail = util.can_arch_fail(failable_arches, arch)
|
2016-11-18 20:33:03 +00:00
|
|
|
with util.failable(compose, self.can_fail, variant, arch, 'ostree-installer',
|
|
|
|
logger=self.pool._logger):
|
2016-03-23 09:40:16 +00:00
|
|
|
self.worker(compose, variant, arch, config)
|
|
|
|
|
|
|
|
def worker(self, compose, variant, arch, config):
|
2016-04-05 07:13:01 +00:00
|
|
|
msg = 'Ostree phase for variant %s, arch %s' % (variant.uid, arch)
|
2016-03-23 09:40:16 +00:00
|
|
|
self.pool.log_info('[BEGIN] %s' % msg)
|
2017-03-15 14:54:45 +00:00
|
|
|
self.logdir = compose.paths.log.topdir('%s/%s/ostree_installer-%s' % (arch, variant, self.num))
|
2016-03-23 09:40:16 +00:00
|
|
|
|
2018-02-20 13:29:49 +00:00
|
|
|
repos = get_repo_urls(None, # compose==None. Special value says that method should ignore deprecated variant-type repo
|
2018-04-09 13:32:33 +00:00
|
|
|
shortcuts.force_list(config['repo'])
|
2019-08-01 14:36:54 +00:00
|
|
|
+ self.baseurls,
|
2018-02-20 13:29:49 +00:00
|
|
|
arch=arch,
|
|
|
|
logger=self.pool)
|
2018-06-06 13:44:11 +00:00
|
|
|
if compose.has_comps:
|
|
|
|
repos.append(
|
|
|
|
translate_path(
|
|
|
|
compose,
|
|
|
|
compose.paths.work.comps_repo(
|
|
|
|
'$basearch', variant=variant, create_dir=False
|
|
|
|
),
|
|
|
|
)
|
|
|
|
)
|
2017-03-27 21:53:08 +00:00
|
|
|
repos = [url.replace('$arch', arch) for url in repos]
|
2016-04-08 07:29:05 +00:00
|
|
|
output_dir = os.path.join(compose.paths.work.topdir(arch), variant.uid, 'ostree_installer')
|
|
|
|
util.makedirs(os.path.dirname(output_dir))
|
2016-03-23 09:40:16 +00:00
|
|
|
|
2016-04-11 08:25:10 +00:00
|
|
|
self.template_dir = os.path.join(compose.paths.work.topdir(arch), variant.uid, 'lorax_templates')
|
2020-01-02 02:16:50 +00:00
|
|
|
self._clone_templates(compose, config.get('template_repo'), config.get('template_branch'))
|
2016-10-06 07:17:09 +00:00
|
|
|
disc_type = compose.conf['disc_types'].get('ostree', 'ostree')
|
2016-04-11 08:25:10 +00:00
|
|
|
|
2016-10-06 07:17:09 +00:00
|
|
|
volid = get_volid(compose, arch, variant, disc_type=disc_type)
|
2019-03-21 09:58:13 +00:00
|
|
|
self._run_ostree_cmd(compose, variant, arch, config, repos, output_dir, volid)
|
2016-03-23 09:40:16 +00:00
|
|
|
|
2016-04-14 08:56:12 +00:00
|
|
|
filename = compose.get_image_name(arch, variant, disc_type=disc_type)
|
2016-04-08 07:29:05 +00:00
|
|
|
self._copy_image(compose, variant, arch, filename, output_dir)
|
2016-03-23 09:40:16 +00:00
|
|
|
self._add_to_manifest(compose, variant, arch, filename)
|
2019-03-21 09:58:13 +00:00
|
|
|
self.pool.log_info('[DONE ] %s' % (msg))
|
2016-03-23 09:40:16 +00:00
|
|
|
|
2020-01-02 02:16:50 +00:00
|
|
|
def _clone_templates(self, compose, url, branch='master'):
|
2016-04-11 08:25:10 +00:00
|
|
|
if not url:
|
|
|
|
self.template_dir = None
|
|
|
|
return
|
|
|
|
scm.get_dir_from_scm({'scm': 'git', 'repo': url, 'branch': branch, 'dir': '.'},
|
2020-01-02 02:16:50 +00:00
|
|
|
self.template_dir, compose=compose)
|
2016-04-11 08:25:10 +00:00
|
|
|
|
2016-03-23 09:40:16 +00:00
|
|
|
def _get_release(self, compose, config):
|
2017-04-07 13:33:43 +00:00
|
|
|
if 'release' in config:
|
|
|
|
return version_generator(compose, config['release']) or compose.image_release
|
2016-03-23 09:40:16 +00:00
|
|
|
return config.get('release', None)
|
|
|
|
|
2016-04-08 07:29:05 +00:00
|
|
|
def _copy_image(self, compose, variant, arch, filename, output_dir):
|
2016-03-23 09:40:16 +00:00
|
|
|
iso_path = compose.paths.compose.iso_path(arch, variant, filename)
|
2016-05-26 07:58:56 +00:00
|
|
|
os_path = compose.paths.compose.os_tree(arch, variant)
|
2016-04-08 07:29:05 +00:00
|
|
|
boot_iso = os.path.join(output_dir, 'images', 'boot.iso')
|
2016-03-23 09:40:16 +00:00
|
|
|
|
2018-05-15 07:26:54 +00:00
|
|
|
util.copy_all(output_dir, os_path)
|
2016-03-23 09:40:16 +00:00
|
|
|
try:
|
|
|
|
os.link(boot_iso, iso_path)
|
|
|
|
except OSError:
|
|
|
|
shutil.copy2(boot_iso, iso_path)
|
|
|
|
|
|
|
|
def _add_to_manifest(self, compose, variant, arch, filename):
|
|
|
|
full_iso_path = compose.paths.compose.iso_path(arch, variant, filename)
|
|
|
|
iso_path = compose.paths.compose.iso_path(arch, variant, filename, relative=True)
|
2016-10-12 11:56:19 +00:00
|
|
|
implant_md5 = iso.get_implanted_md5(full_iso_path)
|
2016-03-23 09:40:16 +00:00
|
|
|
|
|
|
|
img = images.Image(compose.im)
|
|
|
|
img.path = iso_path
|
|
|
|
img.mtime = util.get_mtime(full_iso_path)
|
|
|
|
img.size = util.get_file_size(full_iso_path)
|
|
|
|
img.arch = arch
|
2016-10-06 11:12:34 +00:00
|
|
|
img.type = "dvd-ostree"
|
2016-03-23 09:40:16 +00:00
|
|
|
img.format = "iso"
|
|
|
|
img.disc_number = 1
|
|
|
|
img.disc_count = 1
|
|
|
|
img.bootable = True
|
2017-10-11 06:17:17 +00:00
|
|
|
img.subvariant = variant.uid
|
2016-03-23 09:40:16 +00:00
|
|
|
img.implant_md5 = implant_md5
|
2016-06-24 07:44:40 +00:00
|
|
|
setattr(img, 'can_fail', self.can_fail)
|
2016-05-03 14:31:20 +00:00
|
|
|
setattr(img, 'deliverable', 'ostree-installer')
|
2016-03-23 09:40:16 +00:00
|
|
|
try:
|
2016-10-12 11:56:19 +00:00
|
|
|
img.volume_id = iso.get_volume_id(full_iso_path)
|
2016-03-23 09:40:16 +00:00
|
|
|
except RuntimeError:
|
|
|
|
pass
|
|
|
|
compose.im.add(variant.uid, arch, img)
|
|
|
|
|
2016-04-11 08:25:10 +00:00
|
|
|
def _get_templates(self, config, key):
|
|
|
|
"""Retrieve all templates from configuration and make sure the paths
|
|
|
|
are absolute. Raises RuntimeError if template repo is needed but not
|
|
|
|
configured.
|
|
|
|
"""
|
|
|
|
templates = []
|
|
|
|
for template in config.get(key, []):
|
|
|
|
if template[0] != '/':
|
|
|
|
if not self.template_dir:
|
|
|
|
raise RuntimeError('Relative path to template without setting template_repo.')
|
|
|
|
template = os.path.join(self.template_dir, template)
|
|
|
|
templates.append(template)
|
|
|
|
return templates
|
|
|
|
|
2016-10-06 07:17:09 +00:00
|
|
|
def _run_ostree_cmd(self, compose, variant, arch, config, source_repo, output_dir, volid):
|
2016-03-23 09:40:16 +00:00
|
|
|
lorax_wrapper = lorax.LoraxWrapper()
|
2017-06-15 06:10:56 +00:00
|
|
|
lorax_cmd = lorax_wrapper.get_lorax_cmd(
|
2016-03-23 09:40:16 +00:00
|
|
|
compose.conf['release_name'],
|
|
|
|
compose.conf["release_version"],
|
|
|
|
self._get_release(compose, config),
|
|
|
|
repo_baseurl=source_repo,
|
2016-04-08 07:29:05 +00:00
|
|
|
output_dir=output_dir,
|
2016-03-23 09:40:16 +00:00
|
|
|
variant=variant.uid,
|
|
|
|
nomacboot=True,
|
2016-10-06 07:17:09 +00:00
|
|
|
volid=volid,
|
2019-01-28 12:21:31 +00:00
|
|
|
buildarch=get_valid_arches(arch)[0],
|
2016-03-23 09:40:16 +00:00
|
|
|
buildinstallpackages=config.get('installpkgs'),
|
2016-04-11 08:25:10 +00:00
|
|
|
add_template=self._get_templates(config, 'add_template'),
|
|
|
|
add_arch_template=self._get_templates(config, 'add_arch_template'),
|
2016-03-23 09:40:16 +00:00
|
|
|
add_template_var=config.get('add_template_var'),
|
2016-09-21 11:48:51 +00:00
|
|
|
add_arch_template_var=config.get('add_arch_template_var'),
|
2016-11-17 15:15:03 +00:00
|
|
|
rootfs_size=config.get('rootfs_size'),
|
2016-09-21 11:48:51 +00:00
|
|
|
is_final=compose.supported,
|
2016-11-09 08:00:33 +00:00
|
|
|
log_dir=self.logdir,
|
2016-03-23 09:40:16 +00:00
|
|
|
)
|
2017-09-21 07:29:53 +00:00
|
|
|
cmd = 'rm -rf %s && %s' % (shlex_quote(output_dir),
|
|
|
|
' '.join([shlex_quote(x) for x in lorax_cmd]))
|
2016-03-23 09:40:16 +00:00
|
|
|
|
2016-04-12 10:45:28 +00:00
|
|
|
packages = ['pungi', 'lorax', 'ostree']
|
2019-10-10 19:11:38 +00:00
|
|
|
packages += config.get('extra_runroot_pkgs', [])
|
|
|
|
|
2016-03-23 09:40:16 +00:00
|
|
|
log_file = os.path.join(self.logdir, 'runroot.log')
|
2019-03-21 09:58:13 +00:00
|
|
|
|
2020-01-02 09:34:03 +00:00
|
|
|
runroot = Runroot(compose, phase="ostree_installer")
|
2019-03-21 09:58:13 +00:00
|
|
|
runroot.run(
|
|
|
|
cmd, log_file=log_file, arch=arch, packages=packages,
|
2019-05-22 10:54:26 +00:00
|
|
|
mounts=[compose.topdir], chown_paths=[output_dir],
|
2019-03-21 09:58:13 +00:00
|
|
|
weight=compose.conf['runroot_weights'].get('ostree_installer'))
|