almalinux
/
pungi
15
1
Fork 0
Code Issues Pull Requests Releases Activity

Allow customizing image name and volume id 

The computation of image name has been moved to a separate function.
This allowed simplification in how work dir for isos is computed, but
the result is not changed.

The live image phase has some special casing for names of RPM wrapped
ISOs. This is moved to the actual phase. Since this is (and has been)
undocumented, there should not be many users of this special case.

The documentation is updated to describe how image names and volume ids
are determined and how the process can be customized.

Signed-off-by: Lubomír Sedlář <lsedlar@redhat.com>
This commit is contained in:
Lubomír Sedlář 2015-12-10 12:51:18 +01:00
parent 719ec458f4
commit 0e237db5f6
8 changed files with 183 additions and 53 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

75
doc/configuration.rst
View File

@ -127,7 +127,6 @@ Options
**variants_file** [mandatory]
(*scm_dict* or *str*) -- reference to variants XML file that defines release variants and architectures
Example
-------
::
@ -147,6 +146,80 @@ Example
}
Image Naming
============
Both image name and volume id are generated based on the configuration. Since
the volume id is limited to 32 characters, there are more settings available.
The process for generating volume id is to get a list of possible formats and
try them sequentially until one fits in the length limit. If substitutions are
configured, each attempted volume id will be modified by it.
For layered products, the candidate formats are first
``image_volid_layered_product_formats`` followed by ``image_volid_formats``.
Otherwise, only ``image_volid_formats`` are tried.
If no format matches the length limit, an error will be reported and compose
aborted.
Options
-------
**image_name_format** [optional]
(*str*) -- Python's format string to serve as template for image names
This format will be used for all phases generating images. Currently that
means ``createiso``, ``live_images`` and ``buildinstall``.
Available keys are:
* compose_id
* variant
* arch
* disc_type
* disc_num
* suffix
* release_short
* version
**image_volid_formats** [optional]
(*list*) -- A list of format strings for generating volume id.
The available keys are:
* compose_id
* variant
* arch
* disc_type
* release_short
* version
* base_product_short
* base_product_version
**image_volid_layered_product_formats** [optional]
(*list*) -- A listof format strings for generating volume id for layered
products. The keys available are the same as for ``image_volid_formats``.
**volume_id_substitutions** [optional]
(*dict*) -- A mapping of string replacements to shorten the volume id.
Example
-------
::
# Image name respecting Fedora's image naming policy
image_name_format = "%(release_short)s-%(variant)s-%(disc_type)s-%(arch)s-%(version)s%(suffix)s"
# Use the same format for volume id
image_volid_formats = [
"%(release_short)s-%(variant)s-%(disc_type)s-%(arch)s-%(version)s"
]
# No special handling for layered products, use same format as for regular images
image_volid_layered_product_formats = []
# Replace "Cloud" with "C" in volume id etc.
volume_id_substitutions = {
'Cloud': 'C',
'Alpha': 'A',
'Beta': 'B',
'TC': 'T',
}
Createrepo Settings

36
pungi/compose.py
View File

@ -248,3 +248,39 @@ class Compose(kobo.log.LoggingBase):
if not os.path.isfile(path):
return
return open(path, "r").read().strip()
def get_image_name(self, arch, variant, disc_type='dvd',
disc_num=1, suffix='.iso', format=None):
"""Create a filename for image with given parameters.
:raises RuntimeError: when unknown ``disc_type`` is given
"""
default_format = "%(compose_id)s-%(variant)s-%(arch)s-%(disc_type)s%(disc_num)s%(suffix)s"
format = format or self.conf.get('image_name_format', default_format)
if arch == "src":
arch = "source"
if disc_type not in ("cd", "dvd", "ec2", "live", "boot"):
raise RuntimeError("Unsupported disc type: %s" % disc_type)
if disc_num:
disc_num = int(disc_num)
else:
disc_num = ""
compose_id = self.ci_base[variant.uid].compose_id
if variant.type == "layered-product":
variant_uid = variant.parent.uid
else:
variant_uid = variant.uid
args = {
'compose_id': compose_id,
'variant': variant_uid,
'arch': arch,
'disc_type': disc_type,
'disc_num': disc_num,
'suffix': suffix,
'release_short': self.ci_base.release.short,
'version': self.ci_base.release.version,
}
return format % args

32
pungi/paths.py
View File

@ -267,14 +267,12 @@ class WorkPaths(object):
path = os.path.join(path, file_name)
return path
def iso_dir(self, arch, variant, disc_type="dvd", disc_num=1, create_dir=True):
def iso_dir(self, arch, filename, create_dir=True):
"""
Examples:
work/x86_64/iso/rhel-7.0-20120127.0-Server-x86_64-dvd1.iso
work/x86_64/iso/Project-1.0-20151203.0-Client-x86_64-dvd1.iso
"""
dir_name = self.compose.paths.compose.iso_path(arch, variant, disc_type, disc_num, create_dir=False)
dir_name = os.path.basename(dir_name)
path = os.path.join(self.topdir(arch, create_dir=create_dir), "iso", dir_name)
path = os.path.join(self.topdir(arch, create_dir=create_dir), "iso", filename)
if create_dir:
makedirs(path)
return path
@ -507,37 +505,17 @@ class ComposePaths(object):
makedirs(path)
return path
def iso_path(self, arch, variant, disc_type="dvd", disc_num=1, suffix=".iso", symlink_to=None, create_dir=True, relative=False, name=None):
def iso_path(self, arch, variant, filename, symlink_to=None, create_dir=True, relative=False):
"""
Examples:
compose/Server/x86_64/iso/rhel-7.0-20120127.0-Server-x86_64-dvd1.iso
None
"""
if arch == "src":
arch = "source"
if disc_type not in ("cd", "dvd", "ec2", "live", "boot"):
raise RuntimeError("Unsupported disc type: %s" % disc_type)
if disc_num:
disc_num = int(disc_num)
else:
disc_num = ""
path = self.iso_dir(arch, variant, symlink_to=symlink_to, create_dir=create_dir, relative=relative)
if path is None:
return None
compose_id = self.compose.ci_base[variant.uid].compose_id
if variant.type == "layered-product":
variant_uid = variant.parent.uid
else:
variant_uid = variant.uid
if not name:
file_name = "%s-%s-%s-%s%s%s" % (compose_id, variant_uid, arch, disc_type, disc_num, suffix)
else:
file_name = "%s-%s-%s-%s%s%s" % (name, variant_uid, arch, disc_type, disc_num, suffix)
result = os.path.join(path, file_name)
return result
return os.path.join(path, filename)
def image_dir(self, arch, variant, symlink_to=None, create_dir=True, relative=False):
"""

14
pungi/phases/buildinstall.py
View File

@ -125,7 +125,7 @@ class BuildinstallPhase(PhaseBase):
repo_baseurl = self.compose.paths.work.arch_repo(arch)
output_dir = self.compose.paths.work.buildinstall_dir(arch)
volid = get_volid(self.compose, arch)
volid = get_volid(self.compose, arch, disc_type="boot")
buildarch = get_valid_arches(arch)[0]
if buildinstall_method == "lorax":
@ -171,7 +171,7 @@ class BuildinstallPhase(PhaseBase):
os_tree = self.compose.paths.compose.os_tree(arch, variant)
# TODO: label is not used
label = ""
volid = get_volid(self.compose, arch, variant, escape_spaces=False)
volid = get_volid(self.compose, arch, variant, escape_spaces=False, disc_type="boot")
tweak_buildinstall(buildinstall_dir, os_tree, arch, variant.uid, label, volid, kickstart_file)
symlink_boot_iso(self.compose, arch, variant)
@ -308,8 +308,14 @@ def symlink_boot_iso(compose, arch, variant):
return
msg = "Symlinking boot.iso (arch: %s, variant: %s)" % (arch, variant)
new_boot_iso_path = compose.paths.compose.iso_path(arch, variant, disc_type="boot", disc_num=None, suffix=".iso", symlink_to=symlink_isos_to)
new_boot_iso_relative_path = compose.paths.compose.iso_path(arch, variant, disc_type="boot", disc_num=None, suffix=".iso", relative=True)
filename = compose.get_image_name(arch, variant, disc_type="boot",
disc_num=None, suffix=".iso")
new_boot_iso_path = compose.paths.compose.iso_path(arch, variant, filename,
symlink_to=symlink_isos_to)
new_boot_iso_relative_path = compose.paths.compose.iso_path(arch,
variant,
filename,
relative=True)
if os.path.exists(new_boot_iso_path):
# TODO: log
compose.log_warning("[SKIP ] %s" % msg)

19
pungi/phases/createiso.py
View File

@ -64,7 +64,7 @@ class CreateisoPhase(PhaseBase):
self.compose.log_info("Skipping createiso for %s.%s due to config option" % (variant, arch))
continue
volid = get_volid(self.compose, arch, variant)
volid = get_volid(self.compose, arch, variant, disc_type='dvd')
os_tree = self.compose.paths.compose.os_tree(arch, variant)
iso_dir = self.compose.paths.compose.iso_dir(arch, variant, symlink_to=symlink_isos_to)
@ -91,8 +91,18 @@ class CreateisoPhase(PhaseBase):
disc_num += 1
# XXX: hardcoded disc_type
iso_path = self.compose.paths.compose.iso_path(arch, variant, disc_type="dvd", disc_num=disc_num, symlink_to=symlink_isos_to)
relative_iso_path = self.compose.paths.compose.iso_path(arch, variant, disc_type="dvd", disc_num=disc_num, create_dir=False, relative=True)
filename = self.compose.get_image_name(arch, variant,
disc_type="dvd",
disc_num=disc_num)
iso_path = self.compose.paths.compose.iso_path(arch,
variant,
filename,
symlink_to=symlink_isos_to)
relative_iso_path = self.compose.paths.compose.iso_path(arch,
variant,
filename,
create_dir=False,
relative=True)
if os.path.isfile(iso_path):
self.compose.log_warning("Skipping mkisofs, image already exists: %s" % iso_path)
continue
@ -354,7 +364,8 @@ def split_iso(compose, arch, variant):
def prepare_iso(compose, arch, variant, disc_num=1, disc_count=None, split_iso_data=None):
tree_dir = compose.paths.compose.os_tree(arch, variant)
iso_dir = compose.paths.work.iso_dir(arch, variant, disc_num=disc_num)
filename = compose.get_image_name(arch, variant, disc_num=disc_num)
iso_dir = compose.paths.work.iso_dir(arch, filename)
# modify treeinfo
ti_path = os.path.join(tree_dir, ".treeinfo")

8
pungi/phases/live_images.py
View File

@ -116,15 +116,19 @@ class LiveImagesPhase(PhaseBase):
# For other images is scratch always on
cmd["scratch"] = data[0].get("scratch", False)
format = "%(compose_id)s-%(variant)s-%(arch)s-%(disc_type)s%(disc_num)s%(suffix)s"
# Custom name (prefix)
custom_iso_name = None
if cmd["name"]:
custom_iso_name = cmd["name"]
if cmd["version"]:
custom_iso_name += "-%s" % cmd["version"]
format = custom_iso_name + "-%(variant)s-%(arch)s-%(disc_type)s%(disc_num)s%(suffix)s"
# XXX: hardcoded disc_type and disc_num
iso_path = self.compose.paths.compose.iso_path(arch, variant, disc_type="live", disc_num=None, symlink_to=symlink_isos_to, name=custom_iso_name)
filename = self.compose.get_image_name(arch, variant, disc_type="live",
disc_num=None, format=format)
iso_path = self.compose.paths.compose.iso_path(arch, variant, filename,
symlink_to=symlink_isos_to)
if os.path.isfile(iso_path):
self.compose.log_warning("Skipping creating live image, it already exists: %s" % iso_path)
continue

32
pungi/util.py
View File

@ -282,7 +282,13 @@ def get_buildroot_rpms(compose, task_id):
return result
def get_volid(compose, arch, variant=None, escape_spaces=False):
def _apply_substitutions(compose, volid):
for k, v in compose.conf.get('volume_id_substitutions', {}).iteritems():
volid = volid.replace(k, v)
return volid
def get_volid(compose, arch, variant=None, escape_spaces=False, disc_type=False):
"""Get ISO volume ID for arch and variant"""
if variant and variant.type == "addon":
# addons are part of parent variant media
@ -304,13 +310,15 @@ def get_volid(compose, arch, variant=None, escape_spaces=False):
variant_uid = variant and variant.uid or None
products = [
"%(release_short)s-%(release_version)s %(variant_uid)s.%(arch)s",
"%(release_short)s-%(release_version)s %(arch)s",
"%(release_short)s-%(version)s %(variant)s.%(arch)s",
"%(release_short)s-%(version)s %(arch)s",
]
products = compose.conf.get('image_volid_formats', products)
layered_products = [
"%(release_short)s-%(release_version)s %(base_product_short)s-%(base_product_version)s %(variant_uid)s.%(arch)s",
"%(release_short)s-%(release_version)s %(base_product_short)s-%(base_product_version)s %(arch)s",
"%(release_short)s-%(version)s %(base_product_short)s-%(base_product_version)s %(variant)s.%(arch)s",
"%(release_short)s-%(version)s %(base_product_short)s-%(base_product_version)s %(arch)s",
]
layered_products = compose.conf.get('image_volid_layered_product_formats', layered_products)
volid = None
if release_is_layered:
@ -319,9 +327,19 @@ def get_volid(compose, arch, variant=None, escape_spaces=False):
all_products = products
for i in all_products:
if not variant_uid and "%(variant_uid)s" in i:
if not variant_uid and "%(variant)s" in i:
continue
volid = i % locals()
volid = i % {
'compose_id': compose.compose_id,
'variant': variant_uid,
'arch': arch,
'disc_type': disc_type or '',
'release_short': release_short,
'version': release_version,
'base_product_short': base_product_short,
'base_product_version': base_product_version,
}
volid = _apply_substitutions(compose, volid)
if len(volid) <= 32:
break

20
tests/test_buildinstall.py
View File

@ -277,16 +277,18 @@ class TestCopyFiles(unittest.TestCase):
'buildinstall_method': 'buildinstall'
})
get_volid.side_effect = lambda compose, arch, variant, escape_spaces: "%s.%s" % (variant.uid, arch)
get_volid.side_effect = (
lambda compose, arch, variant, escape_spaces, disc_type: "%s.%s" % (variant.uid, arch)
)
get_kickstart_file.return_value = 'kickstart'
phase = BuildinstallPhase(compose)
phase.copy_files()
get_volid.assert_has_calls(
[mock.call(compose, 'x86_64', compose.variants['x86_64'][0], escape_spaces=False),
mock.call(compose, 'amd64', compose.variants['amd64'][0], escape_spaces=False),
mock.call(compose, 'amd64', compose.variants['amd64'][1], escape_spaces=False)],
[mock.call(compose, 'x86_64', compose.variants['x86_64'][0], escape_spaces=False, disc_type='boot'),
mock.call(compose, 'amd64', compose.variants['amd64'][0], escape_spaces=False, disc_type='boot'),
mock.call(compose, 'amd64', compose.variants['amd64'][1], escape_spaces=False, disc_type='boot')],
any_order=True
)
tweak_buildinstall.assert_has_calls(
@ -317,16 +319,18 @@ class TestCopyFiles(unittest.TestCase):
'buildinstall_method': 'lorax'
})
get_volid.side_effect = lambda compose, arch, variant, escape_spaces: "%s.%s" % (variant.uid, arch)
get_volid.side_effect = (
lambda compose, arch, variant, escape_spaces, disc_type: "%s.%s" % (variant.uid, arch)
)
get_kickstart_file.return_value = 'kickstart'
phase = BuildinstallPhase(compose)
phase.copy_files()
get_volid.assert_has_calls(
[mock.call(compose, 'x86_64', compose.variants['x86_64'][0], escape_spaces=False),
mock.call(compose, 'amd64', compose.variants['amd64'][0], escape_spaces=False),
mock.call(compose, 'amd64', compose.variants['amd64'][1], escape_spaces=False)],
[mock.call(compose, 'x86_64', compose.variants['x86_64'][0], escape_spaces=False, disc_type='boot'),
mock.call(compose, 'amd64', compose.variants['amd64'][0], escape_spaces=False, disc_type='boot'),
mock.call(compose, 'amd64', compose.variants['amd64'][1], escape_spaces=False, disc_type='boot')],
any_order=True
)
tweak_buildinstall.assert_has_calls(