From 0e237db5f6f909e8a31a48e1f794b44c38556147 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lubom=C3=ADr=20Sedl=C3=A1=C5=99?= Date: Thu, 10 Dec 2015 12:51:18 +0100 Subject: [PATCH] Allow customizing image name and volume id MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The computation of image name has been moved to a separate function. This allowed simplification in how work dir for isos is computed, but the result is not changed. The live image phase has some special casing for names of RPM wrapped ISOs. This is moved to the actual phase. Since this is (and has been) undocumented, there should not be many users of this special case. The documentation is updated to describe how image names and volume ids are determined and how the process can be customized. Signed-off-by: Lubomír Sedlář --- doc/configuration.rst | 75 +++++++++++++++++++++++++++++++++++- pungi/compose.py | 36 +++++++++++++++++ pungi/paths.py | 32 +++------------ pungi/phases/buildinstall.py | 14 +++++-- pungi/phases/createiso.py | 19 +++++++-- pungi/phases/live_images.py | 8 +++- pungi/util.py | 32 +++++++++++---- tests/test_buildinstall.py | 20 ++++++---- 8 files changed, 183 insertions(+), 53 deletions(-) diff --git a/doc/configuration.rst b/doc/configuration.rst index fd821899..8f6cf7f4 100644 --- a/doc/configuration.rst +++ b/doc/configuration.rst @@ -127,7 +127,6 @@ Options **variants_file** [mandatory] (*scm_dict* or *str*) -- reference to variants XML file that defines release variants and architectures - Example ------- :: @@ -147,6 +146,80 @@ Example } +Image Naming +============ + +Both image name and volume id are generated based on the configuration. Since +the volume id is limited to 32 characters, there are more settings available. +The process for generating volume id is to get a list of possible formats and +try them sequentially until one fits in the length limit. If substitutions are +configured, each attempted volume id will be modified by it. + +For layered products, the candidate formats are first +``image_volid_layered_product_formats`` followed by ``image_volid_formats``. +Otherwise, only ``image_volid_formats`` are tried. + +If no format matches the length limit, an error will be reported and compose +aborted. + +Options +------- + +**image_name_format** [optional] + (*str*) -- Python's format string to serve as template for image names + + This format will be used for all phases generating images. Currently that + means ``createiso``, ``live_images`` and ``buildinstall``. + + Available keys are: + * compose_id + * variant + * arch + * disc_type + * disc_num + * suffix + * release_short + * version + +**image_volid_formats** [optional] + (*list*) -- A list of format strings for generating volume id. + + The available keys are: + * compose_id + * variant + * arch + * disc_type + * release_short + * version + * base_product_short + * base_product_version + +**image_volid_layered_product_formats** [optional] + (*list*) -- A listof format strings for generating volume id for layered + products. The keys available are the same as for ``image_volid_formats``. + +**volume_id_substitutions** [optional] + (*dict*) -- A mapping of string replacements to shorten the volume id. + +Example +------- +:: + + # Image name respecting Fedora's image naming policy + image_name_format = "%(release_short)s-%(variant)s-%(disc_type)s-%(arch)s-%(version)s%(suffix)s" + # Use the same format for volume id + image_volid_formats = [ + "%(release_short)s-%(variant)s-%(disc_type)s-%(arch)s-%(version)s" + ] + # No special handling for layered products, use same format as for regular images + image_volid_layered_product_formats = [] + # Replace "Cloud" with "C" in volume id etc. + volume_id_substitutions = { + 'Cloud': 'C', + 'Alpha': 'A', + 'Beta': 'B', + 'TC': 'T', + } Createrepo Settings diff --git a/pungi/compose.py b/pungi/compose.py index c0a47066..0d917f43 100644 --- a/pungi/compose.py +++ b/pungi/compose.py @@ -248,3 +248,39 @@ class Compose(kobo.log.LoggingBase): if not os.path.isfile(path): return return open(path, "r").read().strip() + + def get_image_name(self, arch, variant, disc_type='dvd', + disc_num=1, suffix='.iso', format=None): + """Create a filename for image with given parameters. + + :raises RuntimeError: when unknown ``disc_type`` is given + """ + default_format = "%(compose_id)s-%(variant)s-%(arch)s-%(disc_type)s%(disc_num)s%(suffix)s" + format = format or self.conf.get('image_name_format', default_format) + + if arch == "src": + arch = "source" + + if disc_type not in ("cd", "dvd", "ec2", "live", "boot"): + raise RuntimeError("Unsupported disc type: %s" % disc_type) + if disc_num: + disc_num = int(disc_num) + else: + disc_num = "" + + compose_id = self.ci_base[variant.uid].compose_id + if variant.type == "layered-product": + variant_uid = variant.parent.uid + else: + variant_uid = variant.uid + args = { + 'compose_id': compose_id, + 'variant': variant_uid, + 'arch': arch, + 'disc_type': disc_type, + 'disc_num': disc_num, + 'suffix': suffix, + 'release_short': self.ci_base.release.short, + 'version': self.ci_base.release.version, + } + return format % args diff --git a/pungi/paths.py b/pungi/paths.py index 73ca5aca..7cc7dd0c 100644 --- a/pungi/paths.py +++ b/pungi/paths.py @@ -267,14 +267,12 @@ class WorkPaths(object): path = os.path.join(path, file_name) return path - def iso_dir(self, arch, variant, disc_type="dvd", disc_num=1, create_dir=True): + def iso_dir(self, arch, filename, create_dir=True): """ Examples: - work/x86_64/iso/rhel-7.0-20120127.0-Server-x86_64-dvd1.iso + work/x86_64/iso/Project-1.0-20151203.0-Client-x86_64-dvd1.iso """ - dir_name = self.compose.paths.compose.iso_path(arch, variant, disc_type, disc_num, create_dir=False) - dir_name = os.path.basename(dir_name) - path = os.path.join(self.topdir(arch, create_dir=create_dir), "iso", dir_name) + path = os.path.join(self.topdir(arch, create_dir=create_dir), "iso", filename) if create_dir: makedirs(path) return path @@ -507,37 +505,17 @@ class ComposePaths(object): makedirs(path) return path - def iso_path(self, arch, variant, disc_type="dvd", disc_num=1, suffix=".iso", symlink_to=None, create_dir=True, relative=False, name=None): + def iso_path(self, arch, variant, filename, symlink_to=None, create_dir=True, relative=False): """ Examples: compose/Server/x86_64/iso/rhel-7.0-20120127.0-Server-x86_64-dvd1.iso None """ - if arch == "src": - arch = "source" - - if disc_type not in ("cd", "dvd", "ec2", "live", "boot"): - raise RuntimeError("Unsupported disc type: %s" % disc_type) - if disc_num: - disc_num = int(disc_num) - else: - disc_num = "" - path = self.iso_dir(arch, variant, symlink_to=symlink_to, create_dir=create_dir, relative=relative) if path is None: return None - compose_id = self.compose.ci_base[variant.uid].compose_id - if variant.type == "layered-product": - variant_uid = variant.parent.uid - else: - variant_uid = variant.uid - if not name: - file_name = "%s-%s-%s-%s%s%s" % (compose_id, variant_uid, arch, disc_type, disc_num, suffix) - else: - file_name = "%s-%s-%s-%s%s%s" % (name, variant_uid, arch, disc_type, disc_num, suffix) - result = os.path.join(path, file_name) - return result + return os.path.join(path, filename) def image_dir(self, arch, variant, symlink_to=None, create_dir=True, relative=False): """ diff --git a/pungi/phases/buildinstall.py b/pungi/phases/buildinstall.py index d58e041c..7f3d6f99 100644 --- a/pungi/phases/buildinstall.py +++ b/pungi/phases/buildinstall.py @@ -125,7 +125,7 @@ class BuildinstallPhase(PhaseBase): repo_baseurl = self.compose.paths.work.arch_repo(arch) output_dir = self.compose.paths.work.buildinstall_dir(arch) - volid = get_volid(self.compose, arch) + volid = get_volid(self.compose, arch, disc_type="boot") buildarch = get_valid_arches(arch)[0] if buildinstall_method == "lorax": @@ -171,7 +171,7 @@ class BuildinstallPhase(PhaseBase): os_tree = self.compose.paths.compose.os_tree(arch, variant) # TODO: label is not used label = "" - volid = get_volid(self.compose, arch, variant, escape_spaces=False) + volid = get_volid(self.compose, arch, variant, escape_spaces=False, disc_type="boot") tweak_buildinstall(buildinstall_dir, os_tree, arch, variant.uid, label, volid, kickstart_file) symlink_boot_iso(self.compose, arch, variant) @@ -308,8 +308,14 @@ def symlink_boot_iso(compose, arch, variant): return msg = "Symlinking boot.iso (arch: %s, variant: %s)" % (arch, variant) - new_boot_iso_path = compose.paths.compose.iso_path(arch, variant, disc_type="boot", disc_num=None, suffix=".iso", symlink_to=symlink_isos_to) - new_boot_iso_relative_path = compose.paths.compose.iso_path(arch, variant, disc_type="boot", disc_num=None, suffix=".iso", relative=True) + filename = compose.get_image_name(arch, variant, disc_type="boot", + disc_num=None, suffix=".iso") + new_boot_iso_path = compose.paths.compose.iso_path(arch, variant, filename, + symlink_to=symlink_isos_to) + new_boot_iso_relative_path = compose.paths.compose.iso_path(arch, + variant, + filename, + relative=True) if os.path.exists(new_boot_iso_path): # TODO: log compose.log_warning("[SKIP ] %s" % msg) diff --git a/pungi/phases/createiso.py b/pungi/phases/createiso.py index 098194c6..7f9efa3b 100644 --- a/pungi/phases/createiso.py +++ b/pungi/phases/createiso.py @@ -64,7 +64,7 @@ class CreateisoPhase(PhaseBase): self.compose.log_info("Skipping createiso for %s.%s due to config option" % (variant, arch)) continue - volid = get_volid(self.compose, arch, variant) + volid = get_volid(self.compose, arch, variant, disc_type='dvd') os_tree = self.compose.paths.compose.os_tree(arch, variant) iso_dir = self.compose.paths.compose.iso_dir(arch, variant, symlink_to=symlink_isos_to) @@ -91,8 +91,18 @@ class CreateisoPhase(PhaseBase): disc_num += 1 # XXX: hardcoded disc_type - iso_path = self.compose.paths.compose.iso_path(arch, variant, disc_type="dvd", disc_num=disc_num, symlink_to=symlink_isos_to) - relative_iso_path = self.compose.paths.compose.iso_path(arch, variant, disc_type="dvd", disc_num=disc_num, create_dir=False, relative=True) + filename = self.compose.get_image_name(arch, variant, + disc_type="dvd", + disc_num=disc_num) + iso_path = self.compose.paths.compose.iso_path(arch, + variant, + filename, + symlink_to=symlink_isos_to) + relative_iso_path = self.compose.paths.compose.iso_path(arch, + variant, + filename, + create_dir=False, + relative=True) if os.path.isfile(iso_path): self.compose.log_warning("Skipping mkisofs, image already exists: %s" % iso_path) continue @@ -354,7 +364,8 @@ def split_iso(compose, arch, variant): def prepare_iso(compose, arch, variant, disc_num=1, disc_count=None, split_iso_data=None): tree_dir = compose.paths.compose.os_tree(arch, variant) - iso_dir = compose.paths.work.iso_dir(arch, variant, disc_num=disc_num) + filename = compose.get_image_name(arch, variant, disc_num=disc_num) + iso_dir = compose.paths.work.iso_dir(arch, filename) # modify treeinfo ti_path = os.path.join(tree_dir, ".treeinfo") diff --git a/pungi/phases/live_images.py b/pungi/phases/live_images.py index aacc1290..93b082ee 100644 --- a/pungi/phases/live_images.py +++ b/pungi/phases/live_images.py @@ -116,15 +116,19 @@ class LiveImagesPhase(PhaseBase): # For other images is scratch always on cmd["scratch"] = data[0].get("scratch", False) + format = "%(compose_id)s-%(variant)s-%(arch)s-%(disc_type)s%(disc_num)s%(suffix)s" # Custom name (prefix) - custom_iso_name = None if cmd["name"]: custom_iso_name = cmd["name"] if cmd["version"]: custom_iso_name += "-%s" % cmd["version"] + format = custom_iso_name + "-%(variant)s-%(arch)s-%(disc_type)s%(disc_num)s%(suffix)s" # XXX: hardcoded disc_type and disc_num - iso_path = self.compose.paths.compose.iso_path(arch, variant, disc_type="live", disc_num=None, symlink_to=symlink_isos_to, name=custom_iso_name) + filename = self.compose.get_image_name(arch, variant, disc_type="live", + disc_num=None, format=format) + iso_path = self.compose.paths.compose.iso_path(arch, variant, filename, + symlink_to=symlink_isos_to) if os.path.isfile(iso_path): self.compose.log_warning("Skipping creating live image, it already exists: %s" % iso_path) continue diff --git a/pungi/util.py b/pungi/util.py index 18ef2352..5bfd02fa 100644 --- a/pungi/util.py +++ b/pungi/util.py @@ -282,7 +282,13 @@ def get_buildroot_rpms(compose, task_id): return result -def get_volid(compose, arch, variant=None, escape_spaces=False): +def _apply_substitutions(compose, volid): + for k, v in compose.conf.get('volume_id_substitutions', {}).iteritems(): + volid = volid.replace(k, v) + return volid + + +def get_volid(compose, arch, variant=None, escape_spaces=False, disc_type=False): """Get ISO volume ID for arch and variant""" if variant and variant.type == "addon": # addons are part of parent variant media @@ -304,13 +310,15 @@ def get_volid(compose, arch, variant=None, escape_spaces=False): variant_uid = variant and variant.uid or None products = [ - "%(release_short)s-%(release_version)s %(variant_uid)s.%(arch)s", - "%(release_short)s-%(release_version)s %(arch)s", + "%(release_short)s-%(version)s %(variant)s.%(arch)s", + "%(release_short)s-%(version)s %(arch)s", ] + products = compose.conf.get('image_volid_formats', products) layered_products = [ - "%(release_short)s-%(release_version)s %(base_product_short)s-%(base_product_version)s %(variant_uid)s.%(arch)s", - "%(release_short)s-%(release_version)s %(base_product_short)s-%(base_product_version)s %(arch)s", + "%(release_short)s-%(version)s %(base_product_short)s-%(base_product_version)s %(variant)s.%(arch)s", + "%(release_short)s-%(version)s %(base_product_short)s-%(base_product_version)s %(arch)s", ] + layered_products = compose.conf.get('image_volid_layered_product_formats', layered_products) volid = None if release_is_layered: @@ -319,9 +327,19 @@ def get_volid(compose, arch, variant=None, escape_spaces=False): all_products = products for i in all_products: - if not variant_uid and "%(variant_uid)s" in i: + if not variant_uid and "%(variant)s" in i: continue - volid = i % locals() + volid = i % { + 'compose_id': compose.compose_id, + 'variant': variant_uid, + 'arch': arch, + 'disc_type': disc_type or '', + 'release_short': release_short, + 'version': release_version, + 'base_product_short': base_product_short, + 'base_product_version': base_product_version, + } + volid = _apply_substitutions(compose, volid) if len(volid) <= 32: break diff --git a/tests/test_buildinstall.py b/tests/test_buildinstall.py index 04978ed3..6600f060 100755 --- a/tests/test_buildinstall.py +++ b/tests/test_buildinstall.py @@ -277,16 +277,18 @@ class TestCopyFiles(unittest.TestCase): 'buildinstall_method': 'buildinstall' }) - get_volid.side_effect = lambda compose, arch, variant, escape_spaces: "%s.%s" % (variant.uid, arch) + get_volid.side_effect = ( + lambda compose, arch, variant, escape_spaces, disc_type: "%s.%s" % (variant.uid, arch) + ) get_kickstart_file.return_value = 'kickstart' phase = BuildinstallPhase(compose) phase.copy_files() get_volid.assert_has_calls( - [mock.call(compose, 'x86_64', compose.variants['x86_64'][0], escape_spaces=False), - mock.call(compose, 'amd64', compose.variants['amd64'][0], escape_spaces=False), - mock.call(compose, 'amd64', compose.variants['amd64'][1], escape_spaces=False)], + [mock.call(compose, 'x86_64', compose.variants['x86_64'][0], escape_spaces=False, disc_type='boot'), + mock.call(compose, 'amd64', compose.variants['amd64'][0], escape_spaces=False, disc_type='boot'), + mock.call(compose, 'amd64', compose.variants['amd64'][1], escape_spaces=False, disc_type='boot')], any_order=True ) tweak_buildinstall.assert_has_calls( @@ -317,16 +319,18 @@ class TestCopyFiles(unittest.TestCase): 'buildinstall_method': 'lorax' }) - get_volid.side_effect = lambda compose, arch, variant, escape_spaces: "%s.%s" % (variant.uid, arch) + get_volid.side_effect = ( + lambda compose, arch, variant, escape_spaces, disc_type: "%s.%s" % (variant.uid, arch) + ) get_kickstart_file.return_value = 'kickstart' phase = BuildinstallPhase(compose) phase.copy_files() get_volid.assert_has_calls( - [mock.call(compose, 'x86_64', compose.variants['x86_64'][0], escape_spaces=False), - mock.call(compose, 'amd64', compose.variants['amd64'][0], escape_spaces=False), - mock.call(compose, 'amd64', compose.variants['amd64'][1], escape_spaces=False)], + [mock.call(compose, 'x86_64', compose.variants['x86_64'][0], escape_spaces=False, disc_type='boot'), + mock.call(compose, 'amd64', compose.variants['amd64'][0], escape_spaces=False, disc_type='boot'), + mock.call(compose, 'amd64', compose.variants['amd64'][1], escape_spaces=False, disc_type='boot')], any_order=True ) tweak_buildinstall.assert_has_calls(