Remove live_images.py (LiveImagesPhase)
This phase was used to create live images with livecd-creator
and 32-bit ARM images with appliance-creator. We also remove
get_create_image_cmd from the Koji wrapper as it was only used
for this phase, remove associated tests, and remove related
configuration settings and documentation.
Fixes: https://pagure.io/pungi/issue/1753
Merges: https://pagure.io/pungi/pull-request/1774
Signed-off-by: Adam Williamson <awilliam@redhat.com>
(cherry picked from commit 531f0ef389)
This commit is contained in:
Adam Williamson
parent
32d5d32a6e
commit
bd91ef1d10
GPG Key ID:
AB9983172AB1E45B
@ -292,8 +292,8 @@ There a couple common format specifiers available for both the options:
|
|||||||
format string. The pattern should not overlap, otherwise it is undefined
|
format string. The pattern should not overlap, otherwise it is undefined
|
||||||
which one will be used.
|
which one will be used.
|
||||||
|
|
||||||
This format will be used for all phases generating images. Currently that
|
This format will be used for some phases generating images. Currently that
|
||||||
means ``createiso``, ``live_images`` and ``buildinstall``.
|
means ``createiso``, ``buildinstall`` and ``ostree_installer``.
|
||||||
|
|
||||||
Available extra keys are:
|
Available extra keys are:
|
||||||
* ``disc_num``
|
* ``disc_num``
|
||||||
@ -323,7 +323,6 @@ There a couple common format specifiers available for both the options:
|
|||||||
|
|
||||||
Available keys are:
|
Available keys are:
|
||||||
* ``boot`` -- for ``boot.iso`` images created in *buildinstall* phase
|
* ``boot`` -- for ``boot.iso`` images created in *buildinstall* phase
|
||||||
* ``live`` -- for images created by *live_images* phase
|
|
||||||
* ``dvd`` -- for images created by *createiso* phase
|
* ``dvd`` -- for images created by *createiso* phase
|
||||||
* ``ostree`` -- for ostree installer images
|
* ``ostree`` -- for ostree installer images
|
||||||
|
|
||||||
@ -351,7 +350,6 @@ Example
|
|||||||
|
|
||||||
disc_types = {
|
disc_types = {
|
||||||
'boot': 'netinst',
|
'boot': 'netinst',
|
||||||
'live': 'Live',
|
|
||||||
'dvd': 'DVD',
|
'dvd': 'DVD',
|
||||||
}
|
}
|
||||||
|
|
||||||
@ -1368,8 +1366,8 @@ All non-``RC`` milestones from label get appended to the version. For release
|
|||||||
either label is used or date, type and respin.
|
either label is used or date, type and respin.
|
||||||
|
|
||||||
|
|
||||||
Common options for Live Images, Live Media and Image Build
|
Common options for Live Media and Image Build
|
||||||
==========================================================
|
=============================================
|
||||||
|
|
||||||
All images can have ``ksurl``, ``version``, ``release`` and ``target``
|
All images can have ``ksurl``, ``version``, ``release`` and ``target``
|
||||||
specified. Since this can create a lot of duplication, there are global options
|
specified. Since this can create a lot of duplication, there are global options
|
||||||
@ -1385,14 +1383,12 @@ The kickstart URL is configured by these options.
|
|||||||
* ``global_ksurl`` -- global fallback setting
|
* ``global_ksurl`` -- global fallback setting
|
||||||
* ``live_media_ksurl``
|
* ``live_media_ksurl``
|
||||||
* ``image_build_ksurl``
|
* ``image_build_ksurl``
|
||||||
* ``live_images_ksurl``
|
|
||||||
|
|
||||||
Target is specified by these settings.
|
Target is specified by these settings.
|
||||||
|
|
||||||
* ``global_target`` -- global fallback setting
|
* ``global_target`` -- global fallback setting
|
||||||
* ``live_media_target``
|
* ``live_media_target``
|
||||||
* ``image_build_target``
|
* ``image_build_target``
|
||||||
* ``live_images_target``
|
|
||||||
* ``osbuild_target``
|
* ``osbuild_target``
|
||||||
|
|
||||||
Version is specified by these options. If no version is set, a default value
|
Version is specified by these options. If no version is set, a default value
|
||||||
@ -1401,7 +1397,6 @@ will be provided according to :ref:`automatic versioning <auto-version>`.
|
|||||||
* ``global_version`` -- global fallback setting
|
* ``global_version`` -- global fallback setting
|
||||||
* ``live_media_version``
|
* ``live_media_version``
|
||||||
* ``image_build_version``
|
* ``image_build_version``
|
||||||
* ``live_images_version``
|
|
||||||
* ``osbuild_version``
|
* ``osbuild_version``
|
||||||
|
|
||||||
Release is specified by these options. If set to a magic value to
|
Release is specified by these options. If set to a magic value to
|
||||||
@ -1411,44 +1406,14 @@ to :ref:`automatic versioning <auto-version>`.
|
|||||||
* ``global_release`` -- global fallback setting
|
* ``global_release`` -- global fallback setting
|
||||||
* ``live_media_release``
|
* ``live_media_release``
|
||||||
* ``image_build_release``
|
* ``image_build_release``
|
||||||
* ``live_images_release``
|
|
||||||
* ``osbuild_release``
|
* ``osbuild_release``
|
||||||
|
|
||||||
Each configuration block can also optionally specify a ``failable`` key. For
|
Each configuration block can also optionally specify a ``failable`` key. It
|
||||||
live images it should have a boolean value. For live media and image build it
|
|
||||||
should be a list of strings containing architectures that are optional. If any
|
should be a list of strings containing architectures that are optional. If any
|
||||||
deliverable fails on an optional architecture, it will not abort the whole
|
deliverable fails on an optional architecture, it will not abort the whole
|
||||||
compose. If the list contains only ``"*"``, all arches will be substituted.
|
compose. If the list contains only ``"*"``, all arches will be substituted.
|
||||||
|
|
||||||
|
|
||||||
Live Images Settings
|
|
||||||
====================
|
|
||||||
|
|
||||||
**live_images**
|
|
||||||
(*list*) -- Configuration for the particular image. The elements of the
|
|
||||||
list should be tuples ``(variant_uid_regex, {arch|*: config})``. The config
|
|
||||||
should be a dict with these keys:
|
|
||||||
|
|
||||||
* ``kickstart`` (*str*)
|
|
||||||
* ``ksurl`` (*str*) [optional] -- where to get the kickstart from
|
|
||||||
* ``name`` (*str*)
|
|
||||||
* ``version`` (*str*)
|
|
||||||
* ``target`` (*str*)
|
|
||||||
* ``repo`` (*str|[str]*) -- repos specified by URL or variant UID
|
|
||||||
* ``specfile`` (*str*) -- for images wrapped in RPM
|
|
||||||
* ``scratch`` (*bool*) -- only RPM-wrapped images can use scratch builds,
|
|
||||||
but by default this is turned off
|
|
||||||
* ``type`` (*str*) -- what kind of task to start in Koji. Defaults to
|
|
||||||
``live`` meaning ``koji spin-livecd`` will be used. Alternative option
|
|
||||||
is ``appliance`` corresponding to ``koji spin-appliance``.
|
|
||||||
* ``sign`` (*bool*) -- only RPM-wrapped images can be signed
|
|
||||||
|
|
||||||
**live_images_no_rename**
|
|
||||||
(*bool*) -- When set to ``True``, filenames generated by Koji will be used.
|
|
||||||
When ``False``, filenames will be generated based on ``image_name_format``
|
|
||||||
configuration option.
|
|
||||||
|
|
||||||
|
|
||||||
Live Media Settings
|
Live Media Settings
|
||||||
===================
|
===================
|
||||||
|
|
||||||
@ -2306,9 +2271,9 @@ Miscellaneous Settings
|
|||||||
format string accepting ``%(variant_name)s`` and ``%(arch)s`` placeholders.
|
format string accepting ``%(variant_name)s`` and ``%(arch)s`` placeholders.
|
||||||
|
|
||||||
**symlink_isos_to**
|
**symlink_isos_to**
|
||||||
(*str*) -- If set, the ISO files from ``buildinstall``, ``createiso`` and
|
(*str*) -- If set, the ISO files from ``buildinstall`` and ``createiso``
|
||||||
``live_images`` phases will be put into this destination, and a symlink
|
phases will be put into this destination, and a symlink pointing to this
|
||||||
pointing to this location will be created in actual compose directory.
|
location will be created in actual compose directory.
|
||||||
|
|
||||||
**dogpile_cache_backend**
|
**dogpile_cache_backend**
|
||||||
(*str*) -- If set, Pungi will use the configured Dogpile cache backend to
|
(*str*) -- If set, Pungi will use the configured Dogpile cache backend to
|
||||||
|
|||||||
@ -294,30 +294,6 @@ This is a shortened configuration for Fedora Radhide compose as of 2019-10-14.
|
|||||||
})
|
})
|
||||||
]
|
]
|
||||||
|
|
||||||
live_target = 'f32'
|
|
||||||
live_images_no_rename = True
|
|
||||||
live_images = [
|
|
||||||
('^Workstation$', {
|
|
||||||
'armhfp': {
|
|
||||||
'kickstart': 'fedora-arm-workstation.ks',
|
|
||||||
'name': 'Fedora-Workstation-armhfp',
|
|
||||||
# Again workstation takes packages from Everything.
|
|
||||||
'repo': 'Everything',
|
|
||||||
'type': 'appliance',
|
|
||||||
'failable': True,
|
|
||||||
}
|
|
||||||
}),
|
|
||||||
('^Server$', {
|
|
||||||
# But Server has its own repo.
|
|
||||||
'armhfp': {
|
|
||||||
'kickstart': 'fedora-arm-server.ks',
|
|
||||||
'name': 'Fedora-Server-armhfp',
|
|
||||||
'type': 'appliance',
|
|
||||||
'failable': True,
|
|
||||||
}
|
|
||||||
}),
|
|
||||||
]
|
|
||||||
|
|
||||||
ostree = {
|
ostree = {
|
||||||
"^Silverblue$": {
|
"^Silverblue$": {
|
||||||
"version": "!OSTREE_VERSION_FROM_LABEL_DATE_TYPE_RESPIN",
|
"version": "!OSTREE_VERSION_FROM_LABEL_DATE_TYPE_RESPIN",
|
||||||
|
|||||||
@ -553,26 +553,6 @@ def make_schema():
|
|||||||
"list_of_strings": {"type": "array", "items": {"type": "string"}},
|
"list_of_strings": {"type": "array", "items": {"type": "string"}},
|
||||||
"strings": _one_or_list({"type": "string"}),
|
"strings": _one_or_list({"type": "string"}),
|
||||||
"optional_string": {"anyOf": [{"type": "string"}, {"type": "null"}]},
|
"optional_string": {"anyOf": [{"type": "string"}, {"type": "null"}]},
|
||||||
"live_image_config": {
|
|
||||||
"type": "object",
|
|
||||||
"properties": {
|
|
||||||
"kickstart": {"type": "string"},
|
|
||||||
"ksurl": {"type": "url"},
|
|
||||||
"name": {"type": "string"},
|
|
||||||
"subvariant": {"type": "string"},
|
|
||||||
"target": {"type": "string"},
|
|
||||||
"version": {"type": "string"},
|
|
||||||
"repo": {"$ref": "#/definitions/repos"},
|
|
||||||
"specfile": {"type": "string"},
|
|
||||||
"scratch": {"type": "boolean"},
|
|
||||||
"type": {"type": "string"},
|
|
||||||
"sign": {"type": "boolean"},
|
|
||||||
"failable": {"type": "boolean"},
|
|
||||||
"release": {"$ref": "#/definitions/optional_string"},
|
|
||||||
},
|
|
||||||
"required": ["kickstart"],
|
|
||||||
"additionalProperties": False,
|
|
||||||
},
|
|
||||||
"osbs_config": {
|
"osbs_config": {
|
||||||
"type": "object",
|
"type": "object",
|
||||||
"properties": {
|
"properties": {
|
||||||
@ -916,11 +896,6 @@ def make_schema():
|
|||||||
},
|
},
|
||||||
"restricted_volid": {"type": "boolean", "default": False},
|
"restricted_volid": {"type": "boolean", "default": False},
|
||||||
"volume_id_substitutions": {"type": "object", "default": {}},
|
"volume_id_substitutions": {"type": "object", "default": {}},
|
||||||
"live_images_no_rename": {"type": "boolean", "default": False},
|
|
||||||
"live_images_ksurl": {"type": "url"},
|
|
||||||
"live_images_target": {"type": "string"},
|
|
||||||
"live_images_release": {"$ref": "#/definitions/optional_string"},
|
|
||||||
"live_images_version": {"type": "string"},
|
|
||||||
"image_build_ksurl": {"type": "url"},
|
"image_build_ksurl": {"type": "url"},
|
||||||
"image_build_target": {"type": "string"},
|
"image_build_target": {"type": "string"},
|
||||||
"image_build_release": {"$ref": "#/definitions/optional_string"},
|
"image_build_release": {"$ref": "#/definitions/optional_string"},
|
||||||
@ -953,8 +928,6 @@ def make_schema():
|
|||||||
"product_id": {"$ref": "#/definitions/str_or_scm_dict"},
|
"product_id": {"$ref": "#/definitions/str_or_scm_dict"},
|
||||||
"product_id_allow_missing": {"type": "boolean", "default": False},
|
"product_id_allow_missing": {"type": "boolean", "default": False},
|
||||||
"product_id_allow_name_prefix": {"type": "boolean", "default": True},
|
"product_id_allow_name_prefix": {"type": "boolean", "default": True},
|
||||||
# Deprecated in favour of regular local/phase/global setting.
|
|
||||||
"live_target": {"type": "string"},
|
|
||||||
"tree_arches": {"$ref": "#/definitions/list_of_strings", "default": []},
|
"tree_arches": {"$ref": "#/definitions/list_of_strings", "default": []},
|
||||||
"tree_variants": {"$ref": "#/definitions/list_of_strings", "default": []},
|
"tree_variants": {"$ref": "#/definitions/list_of_strings", "default": []},
|
||||||
"translate_paths": {"$ref": "#/definitions/string_pairs", "default": []},
|
"translate_paths": {"$ref": "#/definitions/string_pairs", "default": []},
|
||||||
@ -1176,9 +1149,6 @@ def make_schema():
|
|||||||
"ostree_container_use_koji_plugin": {"type": "boolean", "default": False},
|
"ostree_container_use_koji_plugin": {"type": "boolean", "default": False},
|
||||||
"ostree_installer_use_koji_plugin": {"type": "boolean", "default": False},
|
"ostree_installer_use_koji_plugin": {"type": "boolean", "default": False},
|
||||||
"ostree_installer_overwrite": {"type": "boolean", "default": False},
|
"ostree_installer_overwrite": {"type": "boolean", "default": False},
|
||||||
"live_images": _variant_arch_mapping(
|
|
||||||
_one_or_list({"$ref": "#/definitions/live_image_config"})
|
|
||||||
),
|
|
||||||
"image_build_allow_reuse": {"type": "boolean", "default": False},
|
"image_build_allow_reuse": {"type": "boolean", "default": False},
|
||||||
"image_build": {
|
"image_build": {
|
||||||
"type": "object",
|
"type": "object",
|
||||||
|
|||||||
@ -25,7 +25,6 @@ from .buildinstall import BuildinstallPhase # noqa
|
|||||||
from .extra_files import ExtraFilesPhase # noqa
|
from .extra_files import ExtraFilesPhase # noqa
|
||||||
from .createiso import CreateisoPhase # noqa
|
from .createiso import CreateisoPhase # noqa
|
||||||
from .extra_isos import ExtraIsosPhase # noqa
|
from .extra_isos import ExtraIsosPhase # noqa
|
||||||
from .live_images import LiveImagesPhase # noqa
|
|
||||||
from .image_build import ImageBuildPhase # noqa
|
from .image_build import ImageBuildPhase # noqa
|
||||||
from .image_container import ImageContainerPhase # noqa
|
from .image_container import ImageContainerPhase # noqa
|
||||||
from .kiwibuild import KiwiBuildPhase # noqa
|
from .kiwibuild import KiwiBuildPhase # noqa
|
||||||
|
|||||||
@ -1,406 +0,0 @@
|
|||||||
# -*- coding: utf-8 -*-
|
|
||||||
|
|
||||||
|
|
||||||
# This program is free software; you can redistribute it and/or modify
|
|
||||||
# it under the terms of the GNU General Public License as published by
|
|
||||||
# the Free Software Foundation; version 2 of the License.
|
|
||||||
#
|
|
||||||
# This program is distributed in the hope that it will be useful,
|
|
||||||
# but WITHOUT ANY WARRANTY; without even the implied warranty of
|
|
||||||
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
|
||||||
# GNU Library General Public License for more details.
|
|
||||||
#
|
|
||||||
# You should have received a copy of the GNU General Public License
|
|
||||||
# along with this program; if not, see <https://gnu.org/licenses/>.
|
|
||||||
|
|
||||||
|
|
||||||
import os
|
|
||||||
import sys
|
|
||||||
import time
|
|
||||||
import shutil
|
|
||||||
|
|
||||||
from kobo.threads import ThreadPool, WorkerThread
|
|
||||||
from kobo.shortcuts import run, save_to_file, force_list
|
|
||||||
from productmd.images import Image
|
|
||||||
from six.moves import shlex_quote
|
|
||||||
|
|
||||||
from pungi.wrappers.kojiwrapper import KojiWrapper
|
|
||||||
from pungi.wrappers import iso
|
|
||||||
from pungi.phases import base
|
|
||||||
from pungi.util import makedirs, get_mtime, get_file_size, failable
|
|
||||||
from pungi.util import get_repo_urls
|
|
||||||
|
|
||||||
|
|
||||||
# HACK: define cmp in python3
|
|
||||||
if sys.version_info[0] == 3:
|
|
||||||
|
|
||||||
def cmp(a, b):
|
|
||||||
return (a > b) - (a < b)
|
|
||||||
|
|
||||||
|
|
||||||
class LiveImagesPhase(
|
|
||||||
base.PhaseLoggerMixin, base.ImageConfigMixin, base.ConfigGuardedPhase
|
|
||||||
):
|
|
||||||
name = "live_images"
|
|
||||||
|
|
||||||
def __init__(self, compose):
|
|
||||||
super(LiveImagesPhase, self).__init__(compose)
|
|
||||||
self.pool = ThreadPool(logger=self.logger)
|
|
||||||
|
|
||||||
def _get_repos(self, arch, variant, data):
|
|
||||||
repos = []
|
|
||||||
if not variant.is_empty:
|
|
||||||
repos.append(variant.uid)
|
|
||||||
repos.extend(force_list(data.get("repo", [])))
|
|
||||||
return get_repo_urls(self.compose, repos, arch=arch)
|
|
||||||
|
|
||||||
def run(self):
|
|
||||||
symlink_isos_to = self.compose.conf.get("symlink_isos_to")
|
|
||||||
commands = []
|
|
||||||
|
|
||||||
for variant in self.compose.all_variants.values():
|
|
||||||
for arch in variant.arches + ["src"]:
|
|
||||||
for data in self.get_config_block(variant, arch):
|
|
||||||
subvariant = data.get("subvariant", variant.uid)
|
|
||||||
type = data.get("type", "live")
|
|
||||||
|
|
||||||
if type == "live":
|
|
||||||
dest_dir = self.compose.paths.compose.iso_dir(
|
|
||||||
arch, variant, symlink_to=symlink_isos_to
|
|
||||||
)
|
|
||||||
elif type == "appliance":
|
|
||||||
dest_dir = self.compose.paths.compose.image_dir(
|
|
||||||
variant, symlink_to=symlink_isos_to
|
|
||||||
)
|
|
||||||
dest_dir = dest_dir % {"arch": arch}
|
|
||||||
makedirs(dest_dir)
|
|
||||||
else:
|
|
||||||
raise RuntimeError("Unknown live image type %s" % type)
|
|
||||||
if not dest_dir:
|
|
||||||
continue
|
|
||||||
|
|
||||||
cmd = {
|
|
||||||
"name": data.get("name"),
|
|
||||||
"version": self.get_version(data),
|
|
||||||
"release": self.get_release(data),
|
|
||||||
"dest_dir": dest_dir,
|
|
||||||
"build_arch": arch,
|
|
||||||
"ks_file": data["kickstart"],
|
|
||||||
"ksurl": self.get_ksurl(data),
|
|
||||||
# Used for images wrapped in RPM
|
|
||||||
"specfile": data.get("specfile", None),
|
|
||||||
# Scratch (only taken in consideration if specfile
|
|
||||||
# specified) For images wrapped in rpm is scratch
|
|
||||||
# disabled by default For other images is scratch
|
|
||||||
# always on
|
|
||||||
"scratch": data.get("scratch", False),
|
|
||||||
"sign": False,
|
|
||||||
"type": type,
|
|
||||||
"label": "", # currently not used
|
|
||||||
"subvariant": subvariant,
|
|
||||||
"failable_arches": data.get("failable", []),
|
|
||||||
# First see if live_target is specified, then fall back
|
|
||||||
# to regular setup of local, phase and global setting.
|
|
||||||
"target": self.compose.conf.get("live_target")
|
|
||||||
or self.get_config(data, "target"),
|
|
||||||
}
|
|
||||||
|
|
||||||
cmd["repos"] = self._get_repos(arch, variant, data)
|
|
||||||
|
|
||||||
# Signing of the rpm wrapped image
|
|
||||||
if not cmd["scratch"] and data.get("sign"):
|
|
||||||
cmd["sign"] = True
|
|
||||||
|
|
||||||
cmd["filename"] = self._get_file_name(
|
|
||||||
arch, variant, cmd["name"], cmd["version"]
|
|
||||||
)
|
|
||||||
|
|
||||||
commands.append((cmd, variant, arch))
|
|
||||||
|
|
||||||
for cmd, variant, arch in commands:
|
|
||||||
self.pool.add(CreateLiveImageThread(self.pool))
|
|
||||||
self.pool.queue_put((self.compose, cmd, variant, arch))
|
|
||||||
|
|
||||||
self.pool.start()
|
|
||||||
|
|
||||||
def _get_file_name(self, arch, variant, name=None, version=None):
|
|
||||||
if self.compose.conf["live_images_no_rename"]:
|
|
||||||
return None
|
|
||||||
|
|
||||||
disc_type = self.compose.conf["disc_types"].get("live", "live")
|
|
||||||
|
|
||||||
format = (
|
|
||||||
"%(compose_id)s-%(variant)s-%(arch)s-%(disc_type)s%(disc_num)s%(suffix)s"
|
|
||||||
)
|
|
||||||
# Custom name (prefix)
|
|
||||||
if name:
|
|
||||||
custom_iso_name = name
|
|
||||||
if version:
|
|
||||||
custom_iso_name += "-%s" % version
|
|
||||||
format = (
|
|
||||||
custom_iso_name
|
|
||||||
+ "-%(variant)s-%(arch)s-%(disc_type)s%(disc_num)s%(suffix)s"
|
|
||||||
)
|
|
||||||
|
|
||||||
# XXX: hardcoded disc_num
|
|
||||||
return self.compose.get_image_name(
|
|
||||||
arch, variant, disc_type=disc_type, disc_num=None, format=format
|
|
||||||
)
|
|
||||||
|
|
||||||
|
|
||||||
class CreateLiveImageThread(WorkerThread):
|
|
||||||
EXTS = (".iso", ".raw.xz")
|
|
||||||
|
|
||||||
def process(self, item, num):
|
|
||||||
compose, cmd, variant, arch = item
|
|
||||||
self.failable_arches = cmd.get("failable_arches", [])
|
|
||||||
self.can_fail = bool(self.failable_arches)
|
|
||||||
with failable(
|
|
||||||
compose,
|
|
||||||
self.can_fail,
|
|
||||||
variant,
|
|
||||||
arch,
|
|
||||||
"live",
|
|
||||||
cmd.get("subvariant"),
|
|
||||||
logger=self.pool._logger,
|
|
||||||
):
|
|
||||||
self.worker(compose, cmd, variant, arch, num)
|
|
||||||
|
|
||||||
def worker(self, compose, cmd, variant, arch, num):
|
|
||||||
self.basename = "%(name)s-%(version)s-%(release)s" % cmd
|
|
||||||
log_file = compose.paths.log.log_file(arch, "liveimage-%s" % self.basename)
|
|
||||||
|
|
||||||
subvariant = cmd.pop("subvariant")
|
|
||||||
|
|
||||||
imgname = "%s-%s-%s-%s" % (
|
|
||||||
compose.ci_base.release.short,
|
|
||||||
subvariant,
|
|
||||||
"Live" if cmd["type"] == "live" else "Disk",
|
|
||||||
arch,
|
|
||||||
)
|
|
||||||
|
|
||||||
msg = "Creating ISO (arch: %s, variant: %s): %s" % (
|
|
||||||
arch,
|
|
||||||
variant,
|
|
||||||
self.basename,
|
|
||||||
)
|
|
||||||
self.pool.log_info("[BEGIN] %s" % msg)
|
|
||||||
|
|
||||||
koji_wrapper = KojiWrapper(compose)
|
|
||||||
_, version = compose.compose_id.rsplit("-", 1)
|
|
||||||
name = cmd["name"] or imgname
|
|
||||||
version = cmd["version"] or version
|
|
||||||
archive = False
|
|
||||||
if cmd["specfile"] and not cmd["scratch"]:
|
|
||||||
# Non scratch build are allowed only for rpm wrapped images
|
|
||||||
archive = True
|
|
||||||
koji_cmd = koji_wrapper.get_create_image_cmd(
|
|
||||||
name,
|
|
||||||
version,
|
|
||||||
cmd["target"],
|
|
||||||
cmd["build_arch"],
|
|
||||||
cmd["ks_file"],
|
|
||||||
cmd["repos"],
|
|
||||||
image_type=cmd["type"],
|
|
||||||
wait=True,
|
|
||||||
archive=archive,
|
|
||||||
specfile=cmd["specfile"],
|
|
||||||
release=cmd["release"],
|
|
||||||
ksurl=cmd["ksurl"],
|
|
||||||
)
|
|
||||||
|
|
||||||
# avoid race conditions?
|
|
||||||
# Kerberos authentication failed:
|
|
||||||
# Permission denied in replay cache code (-1765328215)
|
|
||||||
time.sleep(num * 3)
|
|
||||||
|
|
||||||
output = koji_wrapper.run_blocking_cmd(koji_cmd, log_file=log_file)
|
|
||||||
if output["retcode"] != 0:
|
|
||||||
raise RuntimeError(
|
|
||||||
"LiveImage task failed: %s. See %s for more details."
|
|
||||||
% (output["task_id"], log_file)
|
|
||||||
)
|
|
||||||
|
|
||||||
# copy finished image to isos/
|
|
||||||
image_path = [
|
|
||||||
path
|
|
||||||
for path in koji_wrapper.get_image_path(output["task_id"])
|
|
||||||
if self._is_image(path)
|
|
||||||
]
|
|
||||||
if len(image_path) != 1:
|
|
||||||
raise RuntimeError(
|
|
||||||
"Got %d images from task %d, expected 1."
|
|
||||||
% (len(image_path), output["task_id"])
|
|
||||||
)
|
|
||||||
image_path = compose.koji_downloader.get_file(image_path[0])
|
|
||||||
filename = cmd.get("filename") or os.path.basename(image_path)
|
|
||||||
destination = os.path.join(cmd["dest_dir"], filename)
|
|
||||||
shutil.copy2(image_path, destination)
|
|
||||||
|
|
||||||
# copy finished rpm to isos/ (if rpm wrapped ISO was built)
|
|
||||||
if cmd["specfile"]:
|
|
||||||
rpm_paths = koji_wrapper.get_wrapped_rpm_path(output["task_id"])
|
|
||||||
|
|
||||||
if cmd["sign"]:
|
|
||||||
# Sign the rpm wrapped images and get their paths
|
|
||||||
self.pool.log_info(
|
|
||||||
"Signing rpm wrapped images in task_id: %s (expected key ID: %s)"
|
|
||||||
% (output["task_id"], compose.conf.get("signing_key_id"))
|
|
||||||
)
|
|
||||||
signed_rpm_paths = self._sign_image(
|
|
||||||
koji_wrapper, compose, cmd, output["task_id"]
|
|
||||||
)
|
|
||||||
if signed_rpm_paths:
|
|
||||||
rpm_paths = signed_rpm_paths
|
|
||||||
|
|
||||||
for rpm_path in rpm_paths:
|
|
||||||
shutil.copy2(rpm_path, cmd["dest_dir"])
|
|
||||||
|
|
||||||
if cmd["type"] == "live":
|
|
||||||
# ISO manifest only makes sense for live images
|
|
||||||
self._write_manifest(destination)
|
|
||||||
|
|
||||||
self._add_to_images(
|
|
||||||
compose,
|
|
||||||
variant,
|
|
||||||
subvariant,
|
|
||||||
arch,
|
|
||||||
cmd["type"],
|
|
||||||
self._get_format(image_path),
|
|
||||||
destination,
|
|
||||||
)
|
|
||||||
|
|
||||||
self.pool.log_info("[DONE ] %s (task id: %s)" % (msg, output["task_id"]))
|
|
||||||
|
|
||||||
def _add_to_images(self, compose, variant, subvariant, arch, type, format, path):
|
|
||||||
"""Adds the image to images.json"""
|
|
||||||
img = Image(compose.im)
|
|
||||||
img.type = "raw-xz" if type == "appliance" else type
|
|
||||||
img.format = format
|
|
||||||
img.path = os.path.relpath(path, compose.paths.compose.topdir())
|
|
||||||
img.mtime = get_mtime(path)
|
|
||||||
img.size = get_file_size(path)
|
|
||||||
img.arch = arch
|
|
||||||
img.disc_number = 1 # We don't expect multiple disks
|
|
||||||
img.disc_count = 1
|
|
||||||
img.bootable = True
|
|
||||||
img.subvariant = subvariant
|
|
||||||
setattr(img, "can_fail", self.can_fail)
|
|
||||||
setattr(img, "deliverable", "live")
|
|
||||||
compose.im.add(variant=variant.uid, arch=arch, image=img)
|
|
||||||
|
|
||||||
def _is_image(self, path):
|
|
||||||
for ext in self.EXTS:
|
|
||||||
if path.endswith(ext):
|
|
||||||
return True
|
|
||||||
return False
|
|
||||||
|
|
||||||
def _get_format(self, path):
|
|
||||||
"""Get format based on extension."""
|
|
||||||
for ext in self.EXTS:
|
|
||||||
if path.endswith(ext):
|
|
||||||
return ext[1:]
|
|
||||||
raise RuntimeError("Getting format for unknown image %s" % path)
|
|
||||||
|
|
||||||
def _write_manifest(self, iso_path):
|
|
||||||
"""Generate manifest for ISO at given path.
|
|
||||||
|
|
||||||
:param iso_path: (str) absolute path to the ISO
|
|
||||||
"""
|
|
||||||
dir, filename = os.path.split(iso_path)
|
|
||||||
run("cd %s && %s" % (shlex_quote(dir), iso.get_manifest_cmd(filename)))
|
|
||||||
|
|
||||||
def _sign_image(self, koji_wrapper, compose, cmd, koji_task_id):
|
|
||||||
signing_key_id = compose.conf.get("signing_key_id")
|
|
||||||
signing_command = compose.conf.get("signing_command")
|
|
||||||
|
|
||||||
if not signing_key_id:
|
|
||||||
self.pool.log_warning(
|
|
||||||
"Signing is enabled but signing_key_id is not specified"
|
|
||||||
)
|
|
||||||
self.pool.log_warning("Signing skipped")
|
|
||||||
return None
|
|
||||||
if not signing_command:
|
|
||||||
self.pool.log_warning(
|
|
||||||
"Signing is enabled but signing_command is not specified"
|
|
||||||
)
|
|
||||||
self.pool.log_warning("Signing skipped")
|
|
||||||
return None
|
|
||||||
|
|
||||||
# Prepare signing log file
|
|
||||||
signing_log_file = compose.paths.log.log_file(
|
|
||||||
cmd["build_arch"], "live_images-signing-%s" % self.basename
|
|
||||||
)
|
|
||||||
|
|
||||||
# Sign the rpm wrapped images
|
|
||||||
try:
|
|
||||||
sign_builds_in_task(
|
|
||||||
koji_wrapper,
|
|
||||||
koji_task_id,
|
|
||||||
signing_command,
|
|
||||||
log_file=signing_log_file,
|
|
||||||
signing_key_password=compose.conf.get("signing_key_password"),
|
|
||||||
)
|
|
||||||
except RuntimeError:
|
|
||||||
self.pool.log_error(
|
|
||||||
"Error while signing rpm wrapped images. See log: %s" % signing_log_file
|
|
||||||
)
|
|
||||||
raise
|
|
||||||
|
|
||||||
# Get pats to the signed rpms
|
|
||||||
signing_key_id = signing_key_id.lower() # Koji uses lowercase in paths
|
|
||||||
rpm_paths = koji_wrapper.get_signed_wrapped_rpms_paths(
|
|
||||||
koji_task_id, signing_key_id
|
|
||||||
)
|
|
||||||
|
|
||||||
# Wait until files are available
|
|
||||||
if wait_paths(rpm_paths, 60 * 15):
|
|
||||||
# Files are ready
|
|
||||||
return rpm_paths
|
|
||||||
|
|
||||||
# Signed RPMs are not available
|
|
||||||
self.pool.log_warning("Signed files are not available: %s" % rpm_paths)
|
|
||||||
self.pool.log_warning("Unsigned files will be used")
|
|
||||||
return None
|
|
||||||
|
|
||||||
|
|
||||||
def wait_paths(paths, timeout=60):
|
|
||||||
started = time.time()
|
|
||||||
remaining = paths[:]
|
|
||||||
while True:
|
|
||||||
for path in remaining[:]:
|
|
||||||
if os.path.exists(path):
|
|
||||||
remaining.remove(path)
|
|
||||||
if not remaining:
|
|
||||||
break
|
|
||||||
time.sleep(1)
|
|
||||||
if timeout >= 0 and (time.time() - started) > timeout:
|
|
||||||
return False
|
|
||||||
return True
|
|
||||||
|
|
||||||
|
|
||||||
def sign_builds_in_task(
|
|
||||||
koji_wrapper, task_id, signing_command, log_file=None, signing_key_password=None
|
|
||||||
):
|
|
||||||
# Get list of nvrs that should be signed
|
|
||||||
nvrs = koji_wrapper.get_build_nvrs(task_id)
|
|
||||||
if not nvrs:
|
|
||||||
# No builds are available (scratch build, etc.?)
|
|
||||||
return
|
|
||||||
|
|
||||||
# Append builds to sign_cmd
|
|
||||||
for nvr in nvrs:
|
|
||||||
signing_command += " '%s'" % nvr
|
|
||||||
|
|
||||||
# Log signing command before password is filled in it
|
|
||||||
if log_file:
|
|
||||||
save_to_file(log_file, signing_command, append=True)
|
|
||||||
|
|
||||||
# Fill password into the signing command
|
|
||||||
if signing_key_password:
|
|
||||||
signing_command = signing_command % {
|
|
||||||
"signing_key_password": signing_key_password
|
|
||||||
}
|
|
||||||
|
|
||||||
# Sign the builds
|
|
||||||
run(signing_command, can_fail=False, show_cmd=False, logfile=log_file)
|
|
||||||
@ -128,7 +128,6 @@ def run(config, topdir, has_old, offline, defined_variables, schema_overrides):
|
|||||||
pungi.phases.OSTreePhase(compose),
|
pungi.phases.OSTreePhase(compose),
|
||||||
pungi.phases.CreateisoPhase(compose, buildinstall_phase),
|
pungi.phases.CreateisoPhase(compose, buildinstall_phase),
|
||||||
pungi.phases.ExtraIsosPhase(compose, buildinstall_phase),
|
pungi.phases.ExtraIsosPhase(compose, buildinstall_phase),
|
||||||
pungi.phases.LiveImagesPhase(compose),
|
|
||||||
pungi.phases.LiveMediaPhase(compose),
|
pungi.phases.LiveMediaPhase(compose),
|
||||||
pungi.phases.ImageBuildPhase(compose),
|
pungi.phases.ImageBuildPhase(compose),
|
||||||
pungi.phases.ImageChecksumPhase(compose),
|
pungi.phases.ImageChecksumPhase(compose),
|
||||||
|
|||||||
@ -426,7 +426,6 @@ def run_compose(
|
|||||||
ostree_container_phase = pungi.phases.OSTreeContainerPhase(compose, pkgset_phase)
|
ostree_container_phase = pungi.phases.OSTreeContainerPhase(compose, pkgset_phase)
|
||||||
createiso_phase = pungi.phases.CreateisoPhase(compose, buildinstall_phase)
|
createiso_phase = pungi.phases.CreateisoPhase(compose, buildinstall_phase)
|
||||||
extra_isos_phase = pungi.phases.ExtraIsosPhase(compose, buildinstall_phase)
|
extra_isos_phase = pungi.phases.ExtraIsosPhase(compose, buildinstall_phase)
|
||||||
liveimages_phase = pungi.phases.LiveImagesPhase(compose)
|
|
||||||
livemedia_phase = pungi.phases.LiveMediaPhase(compose)
|
livemedia_phase = pungi.phases.LiveMediaPhase(compose)
|
||||||
image_build_phase = pungi.phases.ImageBuildPhase(compose, buildinstall_phase)
|
image_build_phase = pungi.phases.ImageBuildPhase(compose, buildinstall_phase)
|
||||||
kiwibuild_phase = pungi.phases.KiwiBuildPhase(compose)
|
kiwibuild_phase = pungi.phases.KiwiBuildPhase(compose)
|
||||||
@ -446,7 +445,6 @@ def run_compose(
|
|||||||
gather_phase,
|
gather_phase,
|
||||||
extrafiles_phase,
|
extrafiles_phase,
|
||||||
createiso_phase,
|
createiso_phase,
|
||||||
liveimages_phase,
|
|
||||||
livemedia_phase,
|
livemedia_phase,
|
||||||
image_build_phase,
|
image_build_phase,
|
||||||
image_checksum_phase,
|
image_checksum_phase,
|
||||||
@ -483,7 +481,6 @@ def run_compose(
|
|||||||
"signing_key_password_file" in compose.conf
|
"signing_key_password_file" in compose.conf
|
||||||
and "signing_command" in compose.conf
|
and "signing_command" in compose.conf
|
||||||
and "%(signing_key_password)s" in compose.conf["signing_command"]
|
and "%(signing_key_password)s" in compose.conf["signing_command"]
|
||||||
and not liveimages_phase.skip()
|
|
||||||
):
|
):
|
||||||
# TODO: Don't require key if signing is turned off
|
# TODO: Don't require key if signing is turned off
|
||||||
# Obtain signing key password
|
# Obtain signing key password
|
||||||
@ -554,7 +551,6 @@ def run_compose(
|
|||||||
compose_images_schema = (
|
compose_images_schema = (
|
||||||
createiso_phase,
|
createiso_phase,
|
||||||
extra_isos_phase,
|
extra_isos_phase,
|
||||||
liveimages_phase,
|
|
||||||
image_build_phase,
|
image_build_phase,
|
||||||
livemedia_phase,
|
livemedia_phase,
|
||||||
osbuild_phase,
|
osbuild_phase,
|
||||||
@ -580,7 +576,6 @@ def run_compose(
|
|||||||
and ostree_installer_phase.skip()
|
and ostree_installer_phase.skip()
|
||||||
and createiso_phase.skip()
|
and createiso_phase.skip()
|
||||||
and extra_isos_phase.skip()
|
and extra_isos_phase.skip()
|
||||||
and liveimages_phase.skip()
|
|
||||||
and livemedia_phase.skip()
|
and livemedia_phase.skip()
|
||||||
and image_build_phase.skip()
|
and image_build_phase.skip()
|
||||||
and kiwibuild_phase.skip()
|
and kiwibuild_phase.skip()
|
||||||
|
|||||||
@ -414,92 +414,6 @@ class KojiWrapper(object):
|
|||||||
|
|
||||||
return cmd
|
return cmd
|
||||||
|
|
||||||
def get_create_image_cmd(
|
|
||||||
self,
|
|
||||||
name,
|
|
||||||
version,
|
|
||||||
target,
|
|
||||||
arch,
|
|
||||||
ks_file,
|
|
||||||
repos,
|
|
||||||
image_type="live",
|
|
||||||
image_format=None,
|
|
||||||
release=None,
|
|
||||||
wait=True,
|
|
||||||
archive=False,
|
|
||||||
specfile=None,
|
|
||||||
ksurl=None,
|
|
||||||
):
|
|
||||||
# Usage: koji spin-livecd [options] <name> <version> <target> <arch> <kickstart-file> # noqa: E501
|
|
||||||
# Usage: koji spin-appliance [options] <name> <version> <target> <arch> <kickstart-file> # noqa: E501
|
|
||||||
# Examples:
|
|
||||||
# * name: RHEL-7.0
|
|
||||||
# * name: Satellite-6.0.1-RHEL-6
|
|
||||||
# ** -<type>.<arch>
|
|
||||||
# * version: YYYYMMDD[.n|.t].X
|
|
||||||
# * release: 1
|
|
||||||
|
|
||||||
cmd = self._get_cmd()
|
|
||||||
|
|
||||||
if image_type == "live":
|
|
||||||
cmd.append("spin-livecd")
|
|
||||||
elif image_type == "appliance":
|
|
||||||
cmd.append("spin-appliance")
|
|
||||||
else:
|
|
||||||
raise ValueError("Invalid image type: %s" % image_type)
|
|
||||||
|
|
||||||
if not archive:
|
|
||||||
cmd.append("--scratch")
|
|
||||||
|
|
||||||
cmd.append("--noprogress")
|
|
||||||
|
|
||||||
if wait:
|
|
||||||
cmd.append("--wait")
|
|
||||||
else:
|
|
||||||
cmd.append("--nowait")
|
|
||||||
|
|
||||||
if specfile:
|
|
||||||
cmd.append("--specfile=%s" % specfile)
|
|
||||||
|
|
||||||
if ksurl:
|
|
||||||
cmd.append("--ksurl=%s" % ksurl)
|
|
||||||
|
|
||||||
if isinstance(repos, list):
|
|
||||||
for repo in repos:
|
|
||||||
cmd.append("--repo=%s" % repo)
|
|
||||||
else:
|
|
||||||
cmd.append("--repo=%s" % repos)
|
|
||||||
|
|
||||||
if image_format:
|
|
||||||
if image_type != "appliance":
|
|
||||||
raise ValueError("Format can be specified only for appliance images'")
|
|
||||||
supported_formats = ["raw", "qcow", "qcow2", "vmx"]
|
|
||||||
if image_format not in supported_formats:
|
|
||||||
raise ValueError(
|
|
||||||
"Format is not supported: %s. Supported formats: %s"
|
|
||||||
% (image_format, " ".join(sorted(supported_formats)))
|
|
||||||
)
|
|
||||||
cmd.append("--format=%s" % image_format)
|
|
||||||
|
|
||||||
if release is not None:
|
|
||||||
cmd.append("--release=%s" % release)
|
|
||||||
|
|
||||||
# IMPORTANT: all --opts have to be provided *before* args
|
|
||||||
# Usage:
|
|
||||||
# koji spin-livecd [options] <name> <version> <target> <arch> <kickstart-file>
|
|
||||||
|
|
||||||
cmd.append(name)
|
|
||||||
cmd.append(version)
|
|
||||||
cmd.append(target)
|
|
||||||
|
|
||||||
# i686 -> i386 etc.
|
|
||||||
arch = getBaseArch(arch)
|
|
||||||
cmd.append(arch)
|
|
||||||
|
|
||||||
cmd.append(ks_file)
|
|
||||||
|
|
||||||
return cmd
|
|
||||||
|
|
||||||
def _has_connection_error(self, output):
|
def _has_connection_error(self, output):
|
||||||
"""Checks if output indicates connection error."""
|
"""Checks if output indicates connection error."""
|
||||||
return re.search("error: failed to connect\n$", output)
|
return re.search("error: failed to connect\n$", output)
|
||||||
|
|||||||
@ -437,96 +437,6 @@ class LiveMediaTestCase(KojiWrapperBaseTestCase):
|
|||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
class LiveImageKojiWrapperTest(KojiWrapperBaseTestCase):
|
|
||||||
def test_get_create_image_cmd_minimal(self):
|
|
||||||
cmd = self.koji.get_create_image_cmd(
|
|
||||||
"my_name", "1.0", "f24-candidate", "x86_64", "/path/to/ks", ["/repo/1"]
|
|
||||||
)
|
|
||||||
self.assertEqual(cmd[0:3], ["koji", "--profile=custom-koji", "spin-livecd"])
|
|
||||||
six.assertCountEqual(
|
|
||||||
self, cmd[3:7], ["--noprogress", "--scratch", "--wait", "--repo=/repo/1"]
|
|
||||||
)
|
|
||||||
self.assertEqual(
|
|
||||||
cmd[7:], ["my_name", "1.0", "f24-candidate", "x86_64", "/path/to/ks"]
|
|
||||||
)
|
|
||||||
|
|
||||||
def test_get_create_image_cmd_full(self):
|
|
||||||
cmd = self.koji.get_create_image_cmd(
|
|
||||||
"my_name",
|
|
||||||
"1.0",
|
|
||||||
"f24-candidate",
|
|
||||||
"x86_64",
|
|
||||||
"/path/to/ks",
|
|
||||||
["/repo/1", "/repo/2"],
|
|
||||||
release="1",
|
|
||||||
wait=False,
|
|
||||||
archive=True,
|
|
||||||
specfile="foo.spec",
|
|
||||||
ksurl="https://git.example.com/",
|
|
||||||
)
|
|
||||||
self.assertEqual(cmd[0:3], ["koji", "--profile=custom-koji", "spin-livecd"])
|
|
||||||
self.assertEqual(
|
|
||||||
cmd[-5:], ["my_name", "1.0", "f24-candidate", "x86_64", "/path/to/ks"]
|
|
||||||
)
|
|
||||||
six.assertCountEqual(
|
|
||||||
self,
|
|
||||||
cmd[3:-5],
|
|
||||||
[
|
|
||||||
"--noprogress",
|
|
||||||
"--nowait",
|
|
||||||
"--repo=/repo/1",
|
|
||||||
"--repo=/repo/2",
|
|
||||||
"--release=1",
|
|
||||||
"--specfile=foo.spec",
|
|
||||||
"--ksurl=https://git.example.com/",
|
|
||||||
],
|
|
||||||
)
|
|
||||||
|
|
||||||
def test_spin_livecd_with_format(self):
|
|
||||||
with self.assertRaises(ValueError):
|
|
||||||
self.koji.get_create_image_cmd(
|
|
||||||
"my_name",
|
|
||||||
"1.0",
|
|
||||||
"f24-candidate",
|
|
||||||
"x86_64",
|
|
||||||
"/path/to/ks",
|
|
||||||
[],
|
|
||||||
image_format="qcow",
|
|
||||||
)
|
|
||||||
|
|
||||||
def test_spin_appliance_with_format(self):
|
|
||||||
cmd = self.koji.get_create_image_cmd(
|
|
||||||
"my_name",
|
|
||||||
"1.0",
|
|
||||||
"f24-candidate",
|
|
||||||
"x86_64",
|
|
||||||
"/path/to/ks",
|
|
||||||
[],
|
|
||||||
image_type="appliance",
|
|
||||||
image_format="qcow",
|
|
||||||
)
|
|
||||||
self.assertEqual(cmd[0:3], ["koji", "--profile=custom-koji", "spin-appliance"])
|
|
||||||
six.assertCountEqual(
|
|
||||||
self, cmd[3:7], ["--noprogress", "--scratch", "--wait", "--format=qcow"]
|
|
||||||
)
|
|
||||||
self.assertEqual(
|
|
||||||
cmd[7:], ["my_name", "1.0", "f24-candidate", "x86_64", "/path/to/ks"]
|
|
||||||
)
|
|
||||||
|
|
||||||
def test_spin_appliance_with_wrong_format(self):
|
|
||||||
with self.assertRaises(ValueError):
|
|
||||||
self.koji.get_create_image_cmd(
|
|
||||||
"my_name",
|
|
||||||
"1.0",
|
|
||||||
"f24-candidate",
|
|
||||||
"x86_64",
|
|
||||||
"/path/to/ks",
|
|
||||||
[],
|
|
||||||
image_type="appliance",
|
|
||||||
image_format="pretty",
|
|
||||||
)
|
|
||||||
|
|
||||||
|
|
||||||
@mock.patch.dict("os.environ", {"FOO": "BAR"}, clear=True)
|
@mock.patch.dict("os.environ", {"FOO": "BAR"}, clear=True)
|
||||||
class RunrootKojiWrapperTest(KojiWrapperBaseTestCase):
|
class RunrootKojiWrapperTest(KojiWrapperBaseTestCase):
|
||||||
def test_get_cmd_minimal(self):
|
def test_get_cmd_minimal(self):
|
||||||
|
|||||||
File diff suppressed because it is too large
Load Diff
Loading…
Reference in New Issue
Block a user