diff --git a/doc/configuration.rst b/doc/configuration.rst index fd821899..8f6cf7f4 100644 --- a/doc/configuration.rst +++ b/doc/configuration.rst @@ -127,7 +127,6 @@ Options **variants_file** [mandatory] (*scm_dict* or *str*) -- reference to variants XML file that defines release variants and architectures - Example ------- :: @@ -147,6 +146,80 @@ Example } +Image Naming +============ + +Both image name and volume id are generated based on the configuration. Since +the volume id is limited to 32 characters, there are more settings available. +The process for generating volume id is to get a list of possible formats and +try them sequentially until one fits in the length limit. If substitutions are +configured, each attempted volume id will be modified by it. + +For layered products, the candidate formats are first +``image_volid_layered_product_formats`` followed by ``image_volid_formats``. +Otherwise, only ``image_volid_formats`` are tried. + +If no format matches the length limit, an error will be reported and compose +aborted. + +Options +------- + +**image_name_format** [optional] + (*str*) -- Python's format string to serve as template for image names + + This format will be used for all phases generating images. Currently that + means ``createiso``, ``live_images`` and ``buildinstall``. + + Available keys are: + * compose_id + * variant + * arch + * disc_type + * disc_num + * suffix + * release_short + * version + +**image_volid_formats** [optional] + (*list*) -- A list of format strings for generating volume id. + + The available keys are: + * compose_id + * variant + * arch + * disc_type + * release_short + * version + * base_product_short + * base_product_version + +**image_volid_layered_product_formats** [optional] + (*list*) -- A listof format strings for generating volume id for layered + products. The keys available are the same as for ``image_volid_formats``. + +**volume_id_substitutions** [optional] + (*dict*) -- A mapping of string replacements to shorten the volume id. + +Example +------- +:: + + # Image name respecting Fedora's image naming policy + image_name_format = "%(release_short)s-%(variant)s-%(disc_type)s-%(arch)s-%(version)s%(suffix)s" + # Use the same format for volume id + image_volid_formats = [ + "%(release_short)s-%(variant)s-%(disc_type)s-%(arch)s-%(version)s" + ] + # No special handling for layered products, use same format as for regular images + image_volid_layered_product_formats = [] + # Replace "Cloud" with "C" in volume id etc. + volume_id_substitutions = { + 'Cloud': 'C', + 'Alpha': 'A', + 'Beta': 'B', + 'TC': 'T', + } Createrepo Settings diff --git a/pungi/compose.py b/pungi/compose.py index c0a47066..0d917f43 100644 --- a/pungi/compose.py +++ b/pungi/compose.py @@ -248,3 +248,39 @@ class Compose(kobo.log.LoggingBase): if not os.path.isfile(path): return return open(path, "r").read().strip() + + def get_image_name(self, arch, variant, disc_type='dvd', + disc_num=1, suffix='.iso', format=None): + """Create a filename for image with given parameters. + + :raises RuntimeError: when unknown ``disc_type`` is given + """ + default_format = "%(compose_id)s-%(variant)s-%(arch)s-%(disc_type)s%(disc_num)s%(suffix)s" + format = format or self.conf.get('image_name_format', default_format) + + if arch == "src": + arch = "source" + + if disc_type not in ("cd", "dvd", "ec2", "live", "boot"): + raise RuntimeError("Unsupported disc type: %s" % disc_type) + if disc_num: + disc_num = int(disc_num) + else: + disc_num = "" + + compose_id = self.ci_base[variant.uid].compose_id + if variant.type == "layered-product": + variant_uid = variant.parent.uid + else: + variant_uid = variant.uid + args = { + 'compose_id': compose_id, + 'variant': variant_uid, + 'arch': arch, + 'disc_type': disc_type, + 'disc_num': disc_num, + 'suffix': suffix, + 'release_short': self.ci_base.release.short, + 'version': self.ci_base.release.version, + } + return format % args diff --git a/pungi/paths.py b/pungi/paths.py index 73ca5aca..7cc7dd0c 100644 --- a/pungi/paths.py +++ b/pungi/paths.py @@ -267,14 +267,12 @@ class WorkPaths(object): path = os.path.join(path, file_name) return path - def iso_dir(self, arch, variant, disc_type="dvd", disc_num=1, create_dir=True): + def iso_dir(self, arch, filename, create_dir=True): """ Examples: - work/x86_64/iso/rhel-7.0-20120127.0-Server-x86_64-dvd1.iso + work/x86_64/iso/Project-1.0-20151203.0-Client-x86_64-dvd1.iso """ - dir_name = self.compose.paths.compose.iso_path(arch, variant, disc_type, disc_num, create_dir=False) - dir_name = os.path.basename(dir_name) - path = os.path.join(self.topdir(arch, create_dir=create_dir), "iso", dir_name) + path = os.path.join(self.topdir(arch, create_dir=create_dir), "iso", filename) if create_dir: makedirs(path) return path @@ -507,37 +505,17 @@ class ComposePaths(object): makedirs(path) return path - def iso_path(self, arch, variant, disc_type="dvd", disc_num=1, suffix=".iso", symlink_to=None, create_dir=True, relative=False, name=None): + def iso_path(self, arch, variant, filename, symlink_to=None, create_dir=True, relative=False): """ Examples: compose/Server/x86_64/iso/rhel-7.0-20120127.0-Server-x86_64-dvd1.iso None """ - if arch == "src": - arch = "source" - - if disc_type not in ("cd", "dvd", "ec2", "live", "boot"): - raise RuntimeError("Unsupported disc type: %s" % disc_type) - if disc_num: - disc_num = int(disc_num) - else: - disc_num = "" - path = self.iso_dir(arch, variant, symlink_to=symlink_to, create_dir=create_dir, relative=relative) if path is None: return None - compose_id = self.compose.ci_base[variant.uid].compose_id - if variant.type == "layered-product": - variant_uid = variant.parent.uid - else: - variant_uid = variant.uid - if not name: - file_name = "%s-%s-%s-%s%s%s" % (compose_id, variant_uid, arch, disc_type, disc_num, suffix) - else: - file_name = "%s-%s-%s-%s%s%s" % (name, variant_uid, arch, disc_type, disc_num, suffix) - result = os.path.join(path, file_name) - return result + return os.path.join(path, filename) def image_dir(self, arch, variant, symlink_to=None, create_dir=True, relative=False): """ diff --git a/pungi/phases/buildinstall.py b/pungi/phases/buildinstall.py index d58e041c..7f3d6f99 100644 --- a/pungi/phases/buildinstall.py +++ b/pungi/phases/buildinstall.py @@ -125,7 +125,7 @@ class BuildinstallPhase(PhaseBase): repo_baseurl = self.compose.paths.work.arch_repo(arch) output_dir = self.compose.paths.work.buildinstall_dir(arch) - volid = get_volid(self.compose, arch) + volid = get_volid(self.compose, arch, disc_type="boot") buildarch = get_valid_arches(arch)[0] if buildinstall_method == "lorax": @@ -171,7 +171,7 @@ class BuildinstallPhase(PhaseBase): os_tree = self.compose.paths.compose.os_tree(arch, variant) # TODO: label is not used label = "" - volid = get_volid(self.compose, arch, variant, escape_spaces=False) + volid = get_volid(self.compose, arch, variant, escape_spaces=False, disc_type="boot") tweak_buildinstall(buildinstall_dir, os_tree, arch, variant.uid, label, volid, kickstart_file) symlink_boot_iso(self.compose, arch, variant) @@ -308,8 +308,14 @@ def symlink_boot_iso(compose, arch, variant): return msg = "Symlinking boot.iso (arch: %s, variant: %s)" % (arch, variant) - new_boot_iso_path = compose.paths.compose.iso_path(arch, variant, disc_type="boot", disc_num=None, suffix=".iso", symlink_to=symlink_isos_to) - new_boot_iso_relative_path = compose.paths.compose.iso_path(arch, variant, disc_type="boot", disc_num=None, suffix=".iso", relative=True) + filename = compose.get_image_name(arch, variant, disc_type="boot", + disc_num=None, suffix=".iso") + new_boot_iso_path = compose.paths.compose.iso_path(arch, variant, filename, + symlink_to=symlink_isos_to) + new_boot_iso_relative_path = compose.paths.compose.iso_path(arch, + variant, + filename, + relative=True) if os.path.exists(new_boot_iso_path): # TODO: log compose.log_warning("[SKIP ] %s" % msg) diff --git a/pungi/phases/createiso.py b/pungi/phases/createiso.py index 098194c6..7f9efa3b 100644 --- a/pungi/phases/createiso.py +++ b/pungi/phases/createiso.py @@ -64,7 +64,7 @@ class CreateisoPhase(PhaseBase): self.compose.log_info("Skipping createiso for %s.%s due to config option" % (variant, arch)) continue - volid = get_volid(self.compose, arch, variant) + volid = get_volid(self.compose, arch, variant, disc_type='dvd') os_tree = self.compose.paths.compose.os_tree(arch, variant) iso_dir = self.compose.paths.compose.iso_dir(arch, variant, symlink_to=symlink_isos_to) @@ -91,8 +91,18 @@ class CreateisoPhase(PhaseBase): disc_num += 1 # XXX: hardcoded disc_type - iso_path = self.compose.paths.compose.iso_path(arch, variant, disc_type="dvd", disc_num=disc_num, symlink_to=symlink_isos_to) - relative_iso_path = self.compose.paths.compose.iso_path(arch, variant, disc_type="dvd", disc_num=disc_num, create_dir=False, relative=True) + filename = self.compose.get_image_name(arch, variant, + disc_type="dvd", + disc_num=disc_num) + iso_path = self.compose.paths.compose.iso_path(arch, + variant, + filename, + symlink_to=symlink_isos_to) + relative_iso_path = self.compose.paths.compose.iso_path(arch, + variant, + filename, + create_dir=False, + relative=True) if os.path.isfile(iso_path): self.compose.log_warning("Skipping mkisofs, image already exists: %s" % iso_path) continue @@ -354,7 +364,8 @@ def split_iso(compose, arch, variant): def prepare_iso(compose, arch, variant, disc_num=1, disc_count=None, split_iso_data=None): tree_dir = compose.paths.compose.os_tree(arch, variant) - iso_dir = compose.paths.work.iso_dir(arch, variant, disc_num=disc_num) + filename = compose.get_image_name(arch, variant, disc_num=disc_num) + iso_dir = compose.paths.work.iso_dir(arch, filename) # modify treeinfo ti_path = os.path.join(tree_dir, ".treeinfo") diff --git a/pungi/phases/live_images.py b/pungi/phases/live_images.py index aacc1290..93b082ee 100644 --- a/pungi/phases/live_images.py +++ b/pungi/phases/live_images.py @@ -116,15 +116,19 @@ class LiveImagesPhase(PhaseBase): # For other images is scratch always on cmd["scratch"] = data[0].get("scratch", False) + format = "%(compose_id)s-%(variant)s-%(arch)s-%(disc_type)s%(disc_num)s%(suffix)s" # Custom name (prefix) - custom_iso_name = None if cmd["name"]: custom_iso_name = cmd["name"] if cmd["version"]: custom_iso_name += "-%s" % cmd["version"] + format = custom_iso_name + "-%(variant)s-%(arch)s-%(disc_type)s%(disc_num)s%(suffix)s" # XXX: hardcoded disc_type and disc_num - iso_path = self.compose.paths.compose.iso_path(arch, variant, disc_type="live", disc_num=None, symlink_to=symlink_isos_to, name=custom_iso_name) + filename = self.compose.get_image_name(arch, variant, disc_type="live", + disc_num=None, format=format) + iso_path = self.compose.paths.compose.iso_path(arch, variant, filename, + symlink_to=symlink_isos_to) if os.path.isfile(iso_path): self.compose.log_warning("Skipping creating live image, it already exists: %s" % iso_path) continue diff --git a/pungi/util.py b/pungi/util.py index 18ef2352..5bfd02fa 100644 --- a/pungi/util.py +++ b/pungi/util.py @@ -282,7 +282,13 @@ def get_buildroot_rpms(compose, task_id): return result -def get_volid(compose, arch, variant=None, escape_spaces=False): +def _apply_substitutions(compose, volid): + for k, v in compose.conf.get('volume_id_substitutions', {}).iteritems(): + volid = volid.replace(k, v) + return volid + + +def get_volid(compose, arch, variant=None, escape_spaces=False, disc_type=False): """Get ISO volume ID for arch and variant""" if variant and variant.type == "addon": # addons are part of parent variant media @@ -304,13 +310,15 @@ def get_volid(compose, arch, variant=None, escape_spaces=False): variant_uid = variant and variant.uid or None products = [ - "%(release_short)s-%(release_version)s %(variant_uid)s.%(arch)s", - "%(release_short)s-%(release_version)s %(arch)s", + "%(release_short)s-%(version)s %(variant)s.%(arch)s", + "%(release_short)s-%(version)s %(arch)s", ] + products = compose.conf.get('image_volid_formats', products) layered_products = [ - "%(release_short)s-%(release_version)s %(base_product_short)s-%(base_product_version)s %(variant_uid)s.%(arch)s", - "%(release_short)s-%(release_version)s %(base_product_short)s-%(base_product_version)s %(arch)s", + "%(release_short)s-%(version)s %(base_product_short)s-%(base_product_version)s %(variant)s.%(arch)s", + "%(release_short)s-%(version)s %(base_product_short)s-%(base_product_version)s %(arch)s", ] + layered_products = compose.conf.get('image_volid_layered_product_formats', layered_products) volid = None if release_is_layered: @@ -319,9 +327,19 @@ def get_volid(compose, arch, variant=None, escape_spaces=False): all_products = products for i in all_products: - if not variant_uid and "%(variant_uid)s" in i: + if not variant_uid and "%(variant)s" in i: continue - volid = i % locals() + volid = i % { + 'compose_id': compose.compose_id, + 'variant': variant_uid, + 'arch': arch, + 'disc_type': disc_type or '', + 'release_short': release_short, + 'version': release_version, + 'base_product_short': base_product_short, + 'base_product_version': base_product_version, + } + volid = _apply_substitutions(compose, volid) if len(volid) <= 32: break diff --git a/tests/test_buildinstall.py b/tests/test_buildinstall.py index 04978ed3..6600f060 100755 --- a/tests/test_buildinstall.py +++ b/tests/test_buildinstall.py @@ -277,16 +277,18 @@ class TestCopyFiles(unittest.TestCase): 'buildinstall_method': 'buildinstall' }) - get_volid.side_effect = lambda compose, arch, variant, escape_spaces: "%s.%s" % (variant.uid, arch) + get_volid.side_effect = ( + lambda compose, arch, variant, escape_spaces, disc_type: "%s.%s" % (variant.uid, arch) + ) get_kickstart_file.return_value = 'kickstart' phase = BuildinstallPhase(compose) phase.copy_files() get_volid.assert_has_calls( - [mock.call(compose, 'x86_64', compose.variants['x86_64'][0], escape_spaces=False), - mock.call(compose, 'amd64', compose.variants['amd64'][0], escape_spaces=False), - mock.call(compose, 'amd64', compose.variants['amd64'][1], escape_spaces=False)], + [mock.call(compose, 'x86_64', compose.variants['x86_64'][0], escape_spaces=False, disc_type='boot'), + mock.call(compose, 'amd64', compose.variants['amd64'][0], escape_spaces=False, disc_type='boot'), + mock.call(compose, 'amd64', compose.variants['amd64'][1], escape_spaces=False, disc_type='boot')], any_order=True ) tweak_buildinstall.assert_has_calls( @@ -317,16 +319,18 @@ class TestCopyFiles(unittest.TestCase): 'buildinstall_method': 'lorax' }) - get_volid.side_effect = lambda compose, arch, variant, escape_spaces: "%s.%s" % (variant.uid, arch) + get_volid.side_effect = ( + lambda compose, arch, variant, escape_spaces, disc_type: "%s.%s" % (variant.uid, arch) + ) get_kickstart_file.return_value = 'kickstart' phase = BuildinstallPhase(compose) phase.copy_files() get_volid.assert_has_calls( - [mock.call(compose, 'x86_64', compose.variants['x86_64'][0], escape_spaces=False), - mock.call(compose, 'amd64', compose.variants['amd64'][0], escape_spaces=False), - mock.call(compose, 'amd64', compose.variants['amd64'][1], escape_spaces=False)], + [mock.call(compose, 'x86_64', compose.variants['x86_64'][0], escape_spaces=False, disc_type='boot'), + mock.call(compose, 'amd64', compose.variants['amd64'][0], escape_spaces=False, disc_type='boot'), + mock.call(compose, 'amd64', compose.variants['amd64'][1], escape_spaces=False, disc_type='boot')], any_order=True ) tweak_buildinstall.assert_has_calls(