From 61e4f05f1bb41b2c856c227244a9b0fcd5759ffd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marcus=20Sch=C3=A4fer?= Date: Sun, 6 Mar 2022 18:03:51 +0100 Subject: [PATCH] Add support for partition cloning Support creating block level clones of certain partitions used in the image. Clones can be created from the root, boot and any partition listed in the element. --- doc/source/image_description/elements.rst | 14 ++ doc/source/working_with_images.rst | 1 + .../working_with_images/clone_partitions.rst | 128 +++++++++++++++++ .../working_with_images/custom_partitions.rst | 10 ++ kiwi/builder/disk.py | 131 +++++++++++++++--- kiwi/schema/kiwi.rnc | 28 +++- kiwi/schema/kiwi.rng | 45 ++++++ kiwi/xml_parse.py | 58 +++++++- kiwi/xml_state.py | 6 +- test/unit/builder/disk_test.py | 124 +++++++++++++++-- 10 files changed, 505 insertions(+), 40 deletions(-) create mode 100644 doc/source/working_with_images/clone_partitions.rst diff --git a/doc/source/image_description/elements.rst b/doc/source/image_description/elements.rst index 04ee5a49..25627d60 100644 --- a/doc/source/image_description/elements.rst +++ b/doc/source/image_description/elements.rst @@ -485,6 +485,20 @@ kernelcmdline="string": Additional kernel parameters passed to the kernel by the bootloader. +root_clone="number" + For oem disk images, this attribute allows to create `number` + clone(s) of the root partition, with `number` >= 1. A clone partition + is content wise an exact byte for byte copy of the origin root partition. + However, to avoid conflicts at boot time the UUID of any + cloned partition will be made unique. In the sequence of partitions, + the clone(s) will always be created first followed by the + partition considered the origin. The origin partition is the + one that will be referenced and used by the system. + Also see :ref:`clone_partitions` + +boot_clone="number" + Same as `root_clone` but applied to the boot partition if present + luks="passphrase|file:///path/to/keyfile": Supplying a value will trigger the encryption of the partition serving the root filesystem using the LUKS extension. The supplied diff --git a/doc/source/working_with_images.rst b/doc/source/working_with_images.rst index 9473bee9..ae913c8f 100644 --- a/doc/source/working_with_images.rst +++ b/doc/source/working_with_images.rst @@ -23,6 +23,7 @@ Working with Images working_with_images/custom_partitions working_with_images/custom_volumes + working_with_images/clone_partitions working_with_images/setup_network_bootserver working_with_images/legacy_netboot_root_filesystem diff --git a/doc/source/working_with_images/clone_partitions.rst b/doc/source/working_with_images/clone_partitions.rst new file mode 100644 index 00000000..4c4c5a4f --- /dev/null +++ b/doc/source/working_with_images/clone_partitions.rst @@ -0,0 +1,128 @@ +.. _clone_partitions: + +Partition Clones +================ + +.. sidebar:: Abstract + + This page provides details about the partition clone feature + and its use cases + +{kiwi} allows to create block level clones of certain partitions +used in the image. Clones can be created from the `root`, `boot` +and any other partition listed in the `` element. + +A partition clone is a simple byte dump from one +block storage device to another. However, this would cause conflicts +during boot of the system because all unique identifiers like +the UUID of a filesystem will no longer be unique. The clone +feature of {kiwi} takes care of this part and re-creates the +relevant unique identifiers per cloned partition. {kiwi} allows +this also for complex partitions like LVM, LUKS or RAID. + +The partition clone(s) will always appear first in the partition table, +followed by the origin partition. The origin partition is the one +whose identifier will be referenced and used by the system. By default +no cloned partition will be mounted or used by the system at boot time. + +Let's take a look at the following example: + +.. code:: xml + + + + + + + +With the above setup {kiwi} will create a disk image that +contains the following partition table: + +.. code:: + + Number Start (sector) End (sector) Size Code Name + 1 2048 6143 2.0 MiB EF02 p.legacy + 2 6144 47103 20.0 MiB EF00 p.UEFI + 3 47104 661503 300.0 MiB 8300 p.lxbootclone1 + 4 661504 1275903 300.0 MiB 8300 p.lxboot + 5 1275904 1296383 10.0 MiB 8300 p.lxhomeclone1 + 6 1296384 1316863 10.0 MiB 8300 p.lxhomeclone2 + 7 1316864 1337343 10.0 MiB 8300 p.lxhome + 8 1337344 3864575 1.2 GiB 8300 p.lxrootclone1 + 9 3864576 6287326 1.2 GiB 8300 p.lxroot + +When booting the system only the origin partitions `p.lxboot`, `p.lxroot` +and `p.lxhome` will be mounted and visible in e.g. :file:`/etc/fstab`, +the bootloader or the initrd. Thus partition clones are present as a data +source but are not relevant for the operating system from a functional +perspective. + +As shown in the above example there is one clone request for root and boot +and a two clone requests for the home partition. {kiwi} does not sanity- +check the provided number of clones (e.g. whether your partition table +can hold that many partitions). + +.. warning:: + + There is a limit how many partitions a partition table can hold. + This also limits how many clones can be created. + +Use Case +-------- + +Potential use cases for which a clone of one or more partitions +is useful include among others: + +Factory Resets: + Creating an image with the option to rollback to the + state of the system at deployment time can be very helpful + for disaster recovery + +System Updates with Rollbacks e.g A/B: + Creating an image which holds extra space allowing to rollback + modified data can make a system more robust. For example + in a simple A/B update concept, partition A would get updated + but would flip to B if A is considered broken after applying the + update. + +.. note:: + + Most probably any use case based on partition clones requires + additional software to manage them. {kiwi} provides the + option to create the clone layout but it does not provide + the software to implement the actual use case for which the + partition clones are needed. + +Developers writing applications based on a clone layout created +with {kiwi} can leverage the metadata file :file:`/config.partids`. +This file is created at build time and contains the mapping between +the partition `name` and the actual partition number in the partition +table. For partition clones, the following naming convention applies: + +.. code:: + + kiwi_(name)PartClone(id)="(partition_number)" + +The `(name)` is either taken from the `name` attribute +of the `` element or it is a fixed name assigned by {kiwi}. +There are the following reserved partition names for which cloning +is supported: + +* root +* readonly +* boot + +For the mentioned example this will result in the +following :file:`/config.partids`: + +.. code:: + + kiwi_BiosGrub="1" + kiwi_EfiPart="2" + kiwi_bootPartClone1="3" + kiwi_BootPart="4" + kiwi_homePartClone1="5" + kiwi_homePartClone2="6" + kiwi_HomePart="7" + kiwi_rootPartClone1="8" + kiwi_RootPart="9" diff --git a/doc/source/working_with_images/custom_partitions.rst b/doc/source/working_with_images/custom_partitions.rst index 261543a4..6d40bf2f 100644 --- a/doc/source/working_with_images/custom_partitions.rst +++ b/doc/source/working_with_images/custom_partitions.rst @@ -71,6 +71,16 @@ filesystem="btrfs|ext2|ext3|ext4|squashfs|xfs Mandatory filesystem configuration to create one of the supported filesystems on the partition. +clone="number" + Optional setting to indicate that this partition should be + cloned `number` of times. A clone partition is content wise an + exact byte for byte copy of the origin. However, to avoid conflicts at boot + time the UUID of any cloned partition will be made unique. In the + sequence of partitions, the clone(s) will always be created first + followed by the partition considered the origin. The origin + partition is the one that will be referenced and used by the + system + Despite the customization options of the partition table shown above there are the following limitations: diff --git a/kiwi/builder/disk.py b/kiwi/builder/disk.py index d1a87496..59aca16e 100644 --- a/kiwi/builder/disk.py +++ b/kiwi/builder/disk.py @@ -36,6 +36,7 @@ from kiwi.system.identifier import SystemIdentifier from kiwi.boot.image import BootImage from kiwi.storage.setup import DiskSetup from kiwi.storage.loop_device import LoopDevice +from kiwi.storage.clone_device import CloneDevice from kiwi.firmware import FirmWare from kiwi.storage.disk import Disk from kiwi.storage.raid_device import RaidDevice @@ -105,6 +106,10 @@ class DiskBuilder: xml_state.build_type.get_embed_verity_metadata() self.dosparttable_extended_layout = \ xml_state.build_type.get_dosparttable_extended_layout() + self.boot_clone_count = int(xml_state.build_type.get_boot_clone()) \ + if xml_state.build_type.get_boot_clone() else 0 + self.root_clone_count = int(xml_state.build_type.get_root_clone()) \ + if xml_state.build_type.get_root_clone() else 0 self.custom_root_mount_args = xml_state.get_fs_mount_option_list() self.custom_root_creation_args = xml_state.get_fs_create_option_list() self.build_type_name = xml_state.get_build_type_name() @@ -235,7 +240,7 @@ class DiskBuilder: # a list of instances with the sync_data capability # representing the custom partitions area of the disk - system_custom_parts: List[FileSystemBase] = [] + system_custom_parts: Dict[str, FileSystemBase] = {} if self.install_media and self.build_type_name != 'oem': raise KiwiInstallMediaError( @@ -258,7 +263,9 @@ class DiskBuilder: self.boot_image.prepare() # precalculate needed disk size - disksize_mbytes = self.disk_setup.get_disksize_mbytes() + disksize_mbytes = self.disk_setup.get_disksize_mbytes( + root_clone=self.root_clone_count, boot_clone=self.boot_clone_count + ) # create the disk log.info('Creating raw disk image %s', self.diskname) @@ -736,8 +743,8 @@ class DiskBuilder: def _build_custom_parts_filesystem( self, device_map: Dict, custom_partitions: Dict['str', ptable_entry_type] - ) -> List[FileSystemBase]: - filesystem_list = [] + ) -> Dict[str, FileSystemBase]: + filesystem_dict = {} if custom_partitions: for map_name in sorted(custom_partitions.keys()): if map_name in device_map: @@ -751,8 +758,8 @@ class DiskBuilder: filesystem.create_on_device( label=map_name.upper() ) - filesystem_list.append(filesystem) - return filesystem_list + filesystem_dict[map_name] = filesystem + return filesystem_dict def _build_spare_filesystem(self, device_map: Dict) -> Optional[FileSystemBase]: if 'spare' in device_map and self.spare_part_fs: @@ -845,12 +852,18 @@ class DiskBuilder: disksize_used_mbytes += partition_mbsize if self.disk_setup.need_boot_partition(): - log.info('--> creating boot partition') + log.info( + '--> creating boot partition [with {0} clone(s)]'.format( + self.boot_clone_count + ) + ) partition_mbsize = self.disk_setup.boot_partition_size() disk.create_boot_partition( - partition_mbsize + partition_mbsize, self.boot_clone_count ) - disksize_used_mbytes += partition_mbsize + disksize_used_mbytes += \ + (self.boot_clone_count + 1) * partition_mbsize if \ + self.boot_clone_count else partition_mbsize if self.swap_mbytes: if not self.volume_manager_name or self.volume_manager_name != 'lvm': @@ -894,9 +907,11 @@ class DiskBuilder: os.path.getsize(squashed_root_file.name) / 1048576 ) + Defaults.get_min_partition_mbytes() disk.create_root_readonly_partition( - squashed_rootfs_mbsize + squashed_rootfs_mbsize, self.root_clone_count ) - disksize_used_mbytes += squashed_rootfs_mbsize + disksize_used_mbytes += \ + (self.root_clone_count + 1) * squashed_rootfs_mbsize if \ + self.root_clone_count else squashed_rootfs_mbsize if self.spare_part_mbsize and self.spare_part_is_last: rootfs_mbsize = disksize_mbytes - disksize_used_mbytes - \ @@ -910,15 +925,39 @@ class DiskBuilder: '--> overlayroot explicitly requested no write partition' ) else: + root_clone_count = self.root_clone_count + if self.root_filesystem_is_overlay: + # in overlay mode an eventual root clone is created from + # the root readonly partition and not from the root (rw) + # partition. Thus no further action needed here in this + # case + root_clone_count = 0 + if root_clone_count: + clone_rootfs_mbsize = int( + (disksize_mbytes - disksize_used_mbytes) / (root_clone_count + 1) + ) + Defaults.get_min_partition_mbytes() + rootfs_mbsize = f'clone:all_free:{clone_rootfs_mbsize}' if self.volume_manager_name and self.volume_manager_name == 'lvm': - log.info('--> creating LVM root partition') - disk.create_root_lvm_partition(rootfs_mbsize) + log.info( + '--> creating {0} partition [with {1} clone(s)]'.format( + 'root(LVM)', root_clone_count + ) + ) + disk.create_root_lvm_partition(rootfs_mbsize, root_clone_count) elif self.mdraid: - log.info('--> creating mdraid root partition') - disk.create_root_raid_partition(rootfs_mbsize) + log.info( + '--> creating {0} partition [with {1} clone(s)]'.format( + f'root(mdraid={self.mdraid})', root_clone_count + ) + ) + disk.create_root_raid_partition(rootfs_mbsize, root_clone_count) else: - log.info('--> creating root partition') - disk.create_root_partition(rootfs_mbsize) + log.info( + '--> creating root partition [with {0} clone(s)]'.format( + root_clone_count + ) + ) + disk.create_root_partition(rootfs_mbsize, root_clone_count) if self.spare_part_mbsize and self.spare_part_is_last: log.info('--> creating spare partition') @@ -944,7 +983,11 @@ class DiskBuilder: disk.map_partitions() - return disk.get_device() + device_map = disk.get_device() + device_map['origin_root'] = \ + device_map.get('readonly') or device_map['root'] + + return device_map def _write_partition_id_config_to_boot_image(self, disk: Disk) -> None: log.info('Creating config.partids in boot system') @@ -1200,16 +1243,27 @@ class DiskBuilder: system_boot: Optional[FileSystemBase], system_efi: Optional[FileSystemBase], system_spare: Optional[FileSystemBase], - system_custom_parts: List[FileSystemBase] + system_custom_parts: Dict[str, FileSystemBase] ) -> None: log.info('Syncing system to image') if system_spare: log.info('--> Syncing spare partition data') system_spare.sync_data() - for system_custom_part in system_custom_parts: + for map_name in sorted(system_custom_parts.keys()): + system_custom_part = system_custom_parts[map_name] log.info('--> Syncing custom partition(s) data') system_custom_part.sync_data() + if device_map.get(f'{map_name}clone1'): + log.info( + f'--> Dumping {map_name} clone data at extra partition' + ) + system_custom_part_clone = CloneDevice( + system_custom_part.device_provider, self.root_dir + ) + system_custom_part_clone.clone( + self._get_clone_devices(f'{map_name}clone', device_map) + ) if system_efi: log.info('--> Syncing EFI boot data to EFI partition') @@ -1220,6 +1274,13 @@ class DiskBuilder: system_boot.sync_data( self._get_exclude_list_for_boot_data_sync() ) + if device_map.get('bootclone1'): + log.info( + '--> Dumping boot clone data at extra partition' + ) + CloneDevice(system_boot.device_provider, self.root_dir).clone( + self._get_clone_devices('bootclone', device_map) + ) log.info('--> Syncing root filesystem data') if self.root_filesystem_is_overlay: @@ -1270,6 +1331,13 @@ class DiskBuilder: squashed_root.create_verification_metadata( readonly_target ) + if device_map.get('rootclone1'): + log.info( + '--> Dumping readonly root clone data at extra partition' + ) + CloneDevice(device_map['origin_root'], self.root_dir).clone( + self._get_clone_devices('rootclone', device_map) + ) elif self.root_filesystem_verity_blocks: root_target = device_map['root'].get_device() root_target_bytesize = device_map['root'].get_byte_size( @@ -1328,10 +1396,24 @@ class DiskBuilder: filesystem.create_verification_metadata( root_target ) + if device_map.get('rootclone1'): + log.info( + '--> Dumping root clone data at extra partition' + ) + CloneDevice(device_map['origin_root'], self.root_dir).clone( + self._get_clone_devices('rootclone', device_map) + ) else: system.sync_data( self._get_exclude_list_for_root_data_sync(device_map) ) + if device_map.get('rootclone1'): + log.info( + '--> Dumping root clone data at extra partition' + ) + CloneDevice(device_map['origin_root'], self.root_dir).clone( + self._get_clone_devices('rootclone', device_map) + ) if self.integrity_root and \ self.root_filesystem_embed_integrity_metadata: @@ -1450,3 +1532,12 @@ class DiskBuilder: self.root_dir + ''.join(['/boot/', boot_names.initrd_name]) ] ) + + def _get_clone_devices( + self, match: str, device_map: Dict[str, DeviceProvider] + ) -> List[DeviceProvider]: + result = [] + for map_name in sorted(device_map.keys()): + if map_name.startswith(match): + result.append(device_map[map_name]) + return result diff --git a/kiwi/schema/kiwi.rnc b/kiwi/schema/kiwi.rnc index bd7db3c3..743889f9 100644 --- a/kiwi/schema/kiwi.rnc +++ b/kiwi/schema/kiwi.rnc @@ -29,6 +29,7 @@ safe-posix-short-name = xsd:token {pattern = "[a-zA-Z0-9_\-\.]{1,32}"} locale-name = xsd:token {pattern = "(POSIX|[a-z]{2,3}_[A-Z]{2})(,[a-z]{2,3}_[A-Z]{2})*"} mac-address-type = xsd:token {pattern = "([0-9a-fA-F]{2}:){5}[0-9a-fA-F]{2}"} size-type = xsd:token {pattern = "(\d*|image)"} +number-type = xsd:token {pattern = "\d+"} blocks-type = xsd:token {pattern = "(\d*|all)"} volume-size-type = xsd:token {pattern = "(\d+|\d+M|\d+G|all)"} partition-size-type = xsd:token {pattern = "(\d+|\d+M|\d+G)"} @@ -2049,6 +2050,25 @@ div { sch:param [ name = "attr" value = "disk_start_sector" ] sch:param [ name = "types" value = "oem" ] ] + k.type.root_clone.attribute = + ## Clone root partition N times + attribute root_clone { number-type } + >> sch:pattern [ id = "root_clone" is-a = "image_type" + sch:param [ name = "attr" value = "root_clone" ] + sch:param [ name = "types" value = "oem" ] + ] + k.type.boot_clone.attribute = + ## Clone boot partition N times. If no boot partition is + ## used, the attribute has no effect. The use of a boot + ## partition can be enforced through the bootpartition + ## attribute or is implicitly activated according to the + ## combination of type settings that makes the use of an + ## extra boot partition a requirement + attribute boot_clone { number-type } + >> sch:pattern [ id = "boot_clone" is-a = "image_type" + sch:param [ name = "attr" value = "boot_clone" ] + sch:param [ name = "types" value = "oem" ] + ] k.type.bundle_format.attribute = ## Specifies the bundle format pattern ## The format string can contain placeholders for the @@ -2141,6 +2161,8 @@ div { k.type.xen_server.attribute? & k.type.publisher.attribute? & k.type.disk_start_sector.attribute? & + k.type.root_clone.attribute? & + k.type.boot_clone.attribute? & k.type.bundle_format.attribute? k.type = ## The Image Type of the Logical Extend @@ -2375,13 +2397,17 @@ div { attribute filesystem { "btrfs" | "ext2" | "ext3" | "ext4" | "squashfs" | "xfs" } + k.partition.clone.attribute = + ## Clone this partition N times + attribute clone { number-type } k.partition.attlist = k.partition.name.attribute & k.partition.size.attribute & k.partition.partition_name.attribute? & k.partition.partition_type.attribute? & k.partition.mountpoint.attribute? & - k.partition.filesystem.attribute? + k.partition.filesystem.attribute? & + k.partition.clone.attribute? k.partition = ## Specify custom partition in the partition table element partition { diff --git a/kiwi/schema/kiwi.rng b/kiwi/schema/kiwi.rng index d9df83a1..bbd0ef1d 100644 --- a/kiwi/schema/kiwi.rng +++ b/kiwi/schema/kiwi.rng @@ -46,6 +46,11 @@ (\d*|image) + + + \d+ + + (\d*|all) @@ -2918,6 +2923,31 @@ default. + + + Clone root partition N times + + + + + + + + + + Clone boot partition N times. If no boot partition is +used, the attribute has no effect. The use of a boot +partition can be enforced through the bootpartition +attribute or is implicitly activated according to the +combination of type settings that makes the use of an +extra boot partition a requirement + + + + + + + Specifies the bundle format pattern @@ -3156,6 +3186,12 @@ kiwi-ng result bundle ... + + + + + + @@ -3556,6 +3592,12 @@ Allowed values are: t.linux + + + Clone this partition N times + + + @@ -3572,6 +3614,9 @@ Allowed values are: t.linux + + + diff --git a/kiwi/xml_parse.py b/kiwi/xml_parse.py index 69d3b145..64671c57 100644 --- a/kiwi/xml_parse.py +++ b/kiwi/xml_parse.py @@ -3,7 +3,7 @@ # # Generated by generateDS.py version 2.29.24. -# Python 3.10.4 (main, Apr 2 2022, 09:04:19) [GCC 11.2.0] +# Python 3.6.15 (default, Sep 23 2021, 15:41:43) [GCC] # # Command line options: # ('-f', '') @@ -16,7 +16,7 @@ # kiwi/schema/kiwi_for_generateDS.xsd # # Command line: -# /mnt/storage/kiwi/.tox/3/bin/generateDS.py -f --external-encoding="utf-8" --no-dates --no-warnings -o "kiwi/xml_parse.py" kiwi/schema/kiwi_for_generateDS.xsd +# /home/ms/Project/kiwi/.tox/3.6/bin/generateDS.py -f --external-encoding="utf-8" --no-dates --no-warnings -o "kiwi/xml_parse.py" kiwi/schema/kiwi_for_generateDS.xsd # # Current working directory (os.getcwd()): # kiwi @@ -2798,7 +2798,7 @@ class type_(GeneratedsSuper): """The Image Type of the Logical Extend""" subclass = None superclass = None - def __init__(self, boot=None, bootfilesystem=None, firmware=None, bootkernel=None, bootpartition=None, bootpartsize=None, efipartsize=None, efiparttable=None, dosparttable_extended_layout=None, bootprofile=None, btrfs_quota_groups=None, btrfs_root_is_snapshot=None, btrfs_root_is_readonly_snapshot=None, compressed=None, devicepersistency=None, editbootconfig=None, editbootinstall=None, filesystem=None, flags=None, format=None, formatoptions=None, fsmountoptions=None, fscreateoptions=None, squashfscompression=None, gcelicense=None, hybridpersistent=None, hybridpersistent_filesystem=None, gpt_hybrid_mbr=None, force_mbr=None, initrd_system=None, image=None, metadata_path=None, installboot=None, install_continue_on_timeout=None, installprovidefailsafe=None, installiso=None, installstick=None, installpxe=None, mediacheck=None, kernelcmdline=None, luks=None, luks_version=None, luksOS=None, mdraid=None, overlayroot=None, overlayroot_write_partition=None, overlayroot_readonly_partsize=None, verity_blocks=None, embed_verity_metadata=None, standalone_integrity=None, embed_integrity_metadata=None, integrity_metadata_key_description=None, integrity_keyfile=None, primary=None, ramonly=None, rootfs_label=None, spare_part=None, spare_part_mountpoint=None, spare_part_fs=None, spare_part_fs_attributes=None, spare_part_is_last=None, target_blocksize=None, target_removable=None, vga=None, vhdfixedtag=None, volid=None, wwid_wait_timeout=None, derived_from=None, ensure_empty_tmpdirs=None, xen_server=None, publisher=None, disk_start_sector=None, bundle_format=None, bootloader=None, containerconfig=None, machine=None, oemconfig=None, size=None, systemdisk=None, partitions=None, vagrantconfig=None, installmedia=None, luksformat=None): + def __init__(self, boot=None, bootfilesystem=None, firmware=None, bootkernel=None, bootpartition=None, bootpartsize=None, efipartsize=None, efiparttable=None, dosparttable_extended_layout=None, bootprofile=None, btrfs_quota_groups=None, btrfs_root_is_snapshot=None, btrfs_root_is_readonly_snapshot=None, compressed=None, devicepersistency=None, editbootconfig=None, editbootinstall=None, filesystem=None, flags=None, format=None, formatoptions=None, fsmountoptions=None, fscreateoptions=None, squashfscompression=None, gcelicense=None, hybridpersistent=None, hybridpersistent_filesystem=None, gpt_hybrid_mbr=None, force_mbr=None, initrd_system=None, image=None, metadata_path=None, installboot=None, install_continue_on_timeout=None, installprovidefailsafe=None, installiso=None, installstick=None, installpxe=None, mediacheck=None, kernelcmdline=None, luks=None, luks_version=None, luksOS=None, mdraid=None, overlayroot=None, overlayroot_write_partition=None, overlayroot_readonly_partsize=None, verity_blocks=None, embed_verity_metadata=None, standalone_integrity=None, embed_integrity_metadata=None, integrity_metadata_key_description=None, integrity_keyfile=None, primary=None, ramonly=None, rootfs_label=None, spare_part=None, spare_part_mountpoint=None, spare_part_fs=None, spare_part_fs_attributes=None, spare_part_is_last=None, target_blocksize=None, target_removable=None, vga=None, vhdfixedtag=None, volid=None, wwid_wait_timeout=None, derived_from=None, ensure_empty_tmpdirs=None, xen_server=None, publisher=None, disk_start_sector=None, root_clone=None, boot_clone=None, bundle_format=None, bootloader=None, containerconfig=None, machine=None, oemconfig=None, size=None, systemdisk=None, partitions=None, vagrantconfig=None, installmedia=None, luksformat=None): self.original_tagname_ = None self.boot = _cast(None, boot) self.bootfilesystem = _cast(None, bootfilesystem) @@ -2872,6 +2872,8 @@ class type_(GeneratedsSuper): self.xen_server = _cast(bool, xen_server) self.publisher = _cast(None, publisher) self.disk_start_sector = _cast(int, disk_start_sector) + self.root_clone = _cast(None, root_clone) + self.boot_clone = _cast(None, boot_clone) self.bundle_format = _cast(None, bundle_format) if bootloader is None: self.bootloader = [] @@ -3118,6 +3120,10 @@ class type_(GeneratedsSuper): def set_publisher(self, publisher): self.publisher = publisher def get_disk_start_sector(self): return self.disk_start_sector def set_disk_start_sector(self, disk_start_sector): self.disk_start_sector = disk_start_sector + def get_root_clone(self): return self.root_clone + def set_root_clone(self, root_clone): self.root_clone = root_clone + def get_boot_clone(self): return self.boot_clone + def set_boot_clone(self, boot_clone): self.boot_clone = boot_clone def get_bundle_format(self): return self.bundle_format def set_bundle_format(self, bundle_format): self.bundle_format = bundle_format def validate_blocks_type(self, value): @@ -3155,6 +3161,13 @@ class type_(GeneratedsSuper): self.validate_safe_posix_short_name_patterns_, value): warnings_.warn('Value "%s" does not match xsd pattern restrictions: %s' % (value.encode('utf-8'), self.validate_safe_posix_short_name_patterns_, )) validate_safe_posix_short_name_patterns_ = [['^[a-zA-Z0-9_\\-\\.]{1,32}$']] + def validate_number_type(self, value): + # Validate type number-type, a restriction on xs:token. + if value is not None and Validate_simpletypes_: + if not self.gds_validate_simple_patterns( + self.validate_number_type_patterns_, value): + warnings_.warn('Value "%s" does not match xsd pattern restrictions: %s' % (value.encode('utf-8'), self.validate_number_type_patterns_, )) + validate_number_type_patterns_ = [['^\\d+$']] def hasContent_(self): if ( self.bootloader or @@ -3409,6 +3422,12 @@ class type_(GeneratedsSuper): if self.disk_start_sector is not None and 'disk_start_sector' not in already_processed: already_processed.add('disk_start_sector') outfile.write(' disk_start_sector="%s"' % self.gds_format_integer(self.disk_start_sector, input_name='disk_start_sector')) + if self.root_clone is not None and 'root_clone' not in already_processed: + already_processed.add('root_clone') + outfile.write(' root_clone=%s' % (quote_attrib(self.root_clone), )) + if self.boot_clone is not None and 'boot_clone' not in already_processed: + already_processed.add('boot_clone') + outfile.write(' boot_clone=%s' % (quote_attrib(self.boot_clone), )) if self.bundle_format is not None and 'bundle_format' not in already_processed: already_processed.add('bundle_format') outfile.write(' bundle_format=%s' % (self.gds_encode(self.gds_format_string(quote_attrib(self.bundle_format), input_name='bundle_format')), )) @@ -3917,6 +3936,18 @@ class type_(GeneratedsSuper): self.disk_start_sector = int(value) except ValueError as exp: raise_parse_error(node, 'Bad integer attribute: %s' % exp) + value = find_attr_value_('root_clone', node) + if value is not None and 'root_clone' not in already_processed: + already_processed.add('root_clone') + self.root_clone = value + self.root_clone = ' '.join(self.root_clone.split()) + self.validate_number_type(self.root_clone) # validate type number-type + value = find_attr_value_('boot_clone', node) + if value is not None and 'boot_clone' not in already_processed: + already_processed.add('boot_clone') + self.boot_clone = value + self.boot_clone = ' '.join(self.boot_clone.split()) + self.validate_number_type(self.boot_clone) # validate type number-type value = find_attr_value_('bundle_format', node) if value is not None and 'bundle_format' not in already_processed: already_processed.add('bundle_format') @@ -4447,7 +4478,7 @@ class partition(GeneratedsSuper): """Specify custom partition in the partition table""" subclass = None superclass = None - def __init__(self, name=None, size=None, partition_name=None, partition_type=None, mountpoint=None, filesystem=None): + def __init__(self, name=None, size=None, partition_name=None, partition_type=None, mountpoint=None, filesystem=None, clone=None): self.original_tagname_ = None self.name = _cast(None, name) self.size = _cast(None, size) @@ -4455,6 +4486,7 @@ class partition(GeneratedsSuper): self.partition_type = _cast(None, partition_type) self.mountpoint = _cast(None, mountpoint) self.filesystem = _cast(None, filesystem) + self.clone = _cast(None, clone) def factory(*args_, **kwargs_): if CurrentSubclassModule_ is not None: subclass = getSubclassFromModule_( @@ -4478,6 +4510,8 @@ class partition(GeneratedsSuper): def set_mountpoint(self, mountpoint): self.mountpoint = mountpoint def get_filesystem(self): return self.filesystem def set_filesystem(self, filesystem): self.filesystem = filesystem + def get_clone(self): return self.clone + def set_clone(self, clone): self.clone = clone def validate_partition_size_type(self, value): # Validate type partition-size-type, a restriction on xs:token. if value is not None and Validate_simpletypes_: @@ -4492,6 +4526,13 @@ class partition(GeneratedsSuper): self.validate_safe_posix_short_name_patterns_, value): warnings_.warn('Value "%s" does not match xsd pattern restrictions: %s' % (value.encode('utf-8'), self.validate_safe_posix_short_name_patterns_, )) validate_safe_posix_short_name_patterns_ = [['^[a-zA-Z0-9_\\-\\.]{1,32}$']] + def validate_number_type(self, value): + # Validate type number-type, a restriction on xs:token. + if value is not None and Validate_simpletypes_: + if not self.gds_validate_simple_patterns( + self.validate_number_type_patterns_, value): + warnings_.warn('Value "%s" does not match xsd pattern restrictions: %s' % (value.encode('utf-8'), self.validate_number_type_patterns_, )) + validate_number_type_patterns_ = [['^\\d+$']] def hasContent_(self): if ( @@ -4538,6 +4579,9 @@ class partition(GeneratedsSuper): if self.filesystem is not None and 'filesystem' not in already_processed: already_processed.add('filesystem') outfile.write(' filesystem=%s' % (self.gds_encode(self.gds_format_string(quote_attrib(self.filesystem), input_name='filesystem')), )) + if self.clone is not None and 'clone' not in already_processed: + already_processed.add('clone') + outfile.write(' clone=%s' % (quote_attrib(self.clone), )) def exportChildren(self, outfile, level, namespaceprefix_='', name_='partition', fromsubclass_=False, pretty_print=True): pass def build(self, node): @@ -4578,6 +4622,12 @@ class partition(GeneratedsSuper): already_processed.add('filesystem') self.filesystem = value self.filesystem = ' '.join(self.filesystem.split()) + value = find_attr_value_('clone', node) + if value is not None and 'clone' not in already_processed: + already_processed.add('clone') + self.clone = value + self.clone = ' '.join(self.clone.split()) + self.validate_number_type(self.clone) # validate type number-type def buildChildren(self, child_, node, nodeName_, fromsubclass_=False): pass # end class partition diff --git a/kiwi/xml_state.py b/kiwi/xml_state.py index 3c44cdf7..9d4461f4 100644 --- a/kiwi/xml_state.py +++ b/kiwi/xml_state.py @@ -1437,11 +1437,7 @@ class XMLState: partition_name = partition.get_partition_name() or f'p.lx{name}' partitions[name] = ptable_entry_type( mbsize=self._to_mega_byte(partition.get_size()), - # There is currently no clone attribute in the - # element. This will be added on completion of the partition - # clone feature. The internal API structure however, already - # knows about the capability - clone=0, + clone=int(partition.get_clone()) if partition.get_clone() else 0, partition_name=partition_name, partition_type=partition.get_partition_type() or 't.linux', mountpoint=partition.get_mountpoint(), diff --git a/test/unit/builder/disk_test.py b/test/unit/builder/disk_test.py index 869fe68b..eacf837d 100644 --- a/test/unit/builder/disk_test.py +++ b/test/unit/builder/disk_test.py @@ -319,7 +319,9 @@ class TestDiskBuilder: self.setup.set_selinux_file_contexts.assert_called_once_with( '/etc/selinux/targeted/contexts/files/file_contexts' ) - self.disk_setup.get_disksize_mbytes.assert_called_once_with() + self.disk_setup.get_disksize_mbytes.assert_called_once_with( + root_clone=0, boot_clone=0 + ) self.loop_provider.create.assert_called_once_with() self.disk.wipe.assert_called_once_with() self.disk.create_efi_csm_partition.assert_called_once_with( @@ -329,7 +331,7 @@ class TestDiskBuilder: self.firmware.get_efi_partition_size() ) self.disk.create_boot_partition.assert_called_once_with( - self.disk_setup.boot_partition_size() + self.disk_setup.boot_partition_size(), 0 ) self.disk.create_swap_partition.assert_called_once_with( '128' @@ -338,7 +340,7 @@ class TestDiskBuilder: self.firmware.get_prep_partition_size() ) self.disk.create_root_partition.assert_called_once_with( - 'all_free' + 'all_free', 0 ) self.disk.map_partitions.assert_called_once_with() self.bootloader_config.setup_disk_boot_images.assert_called_once_with( @@ -418,6 +420,103 @@ class TestDiskBuilder: 'target_dir' ) + @patch('kiwi.builder.disk.FileSystem.new') + @patch('kiwi.builder.disk.Command.run') + @patch('kiwi.builder.disk.Defaults.get_grub_boot_directory_name') + @patch('os.path.exists') + @patch('os.path.getsize') + @patch('kiwi.builder.disk.SystemSetup') + @patch('kiwi.builder.disk.ImageSystem') + @patch('kiwi.builder.disk.Temporary.new_file') + @patch('kiwi.builder.disk.CloneDevice') + def test_create_disk_standard_root_with_clone( + self, mock_CloneDevice, mock_Temporary_new_file, mock_ImageSystem, + mock_SystemSetup, mock_os_path_getsize, mock_path, + mock_grub_dir, mock_command, mock_fs + ): + tempfile = Mock() + tempfile.name = 'tempfile' + mock_Temporary_new_file.return_value = tempfile + mock_os_path_getsize.return_value = 42 + self.boot_image_task.get_boot_names.return_value = self.boot_names_type( + kernel_name='vmlinuz-1.2.3-default', + initrd_name='initramfs-1.2.3.img' + ) + mock_path.return_value = True + filesystem = Mock() + mock_fs.return_value = filesystem + self.disk_builder.custom_partitions = { + 'var': ptable_entry_type( + mbsize=100, + clone=1, + partition_name='p.lxvar', + partition_type='t.linux', + mountpoint='/var', + filesystem='ext3' + ) + } + self.disk_builder.root_clone_count = 1 + self.disk_builder.boot_clone_count = 1 + self.disk_builder.root_filesystem_is_overlay = False + self.disk_builder.volume_manager_name = None + self.disk_builder.initrd_system = 'dracut' + disk_system = Mock() + mock_SystemSetup.return_value = disk_system + + self.device_map['rootclone1'] = MappedDevice('/dev/root-device', Mock()) + self.device_map['bootclone1'] = MappedDevice('/dev/boot-device', Mock()) + self.device_map['varclone1'] = MappedDevice('/dev/var-device', Mock()) + + # Test in standard mode (no root overlay) + m_open = mock_open() + with patch('builtins.open', m_open, create=True): + self.disk_builder.create_disk() + + self.disk.create_boot_partition.assert_called_once_with( + self.disk_setup.boot_partition_size(), 1 + ) + self.disk.create_root_partition.assert_called_once_with( + 'clone:all_free:458', 1 + ) + self.disk.create_custom_partitions.assert_called_once_with( + self.disk_builder.custom_partitions + ) + assert mock_CloneDevice.return_value.clone.call_args_list == [ + call([self.device_map['varclone1']]), + call([self.device_map['bootclone1']]), + call([self.device_map['rootclone1']]) + ] + + # Test in overlay mode a root clone is created from + # the root readonly partition and not from the root (rw) + # partition. + self.disk_builder.root_filesystem_is_overlay = True + self.disk.create_root_partition.reset_mock() + with patch('builtins.open', m_open, create=True): + self.disk_builder.create_disk() + + self.disk.create_root_readonly_partition.assert_called_once_with( + 10, 1 + ) + self.disk.create_root_partition.assert_called_once_with( + 'all_free', 0 + ) + + # Test in verity mode + self.disk_builder.root_filesystem_verity_blocks = 10 + self.disk_builder.root_filesystem_is_overlay = False + mock_CloneDevice.reset_mock() + self.disk.create_root_partition.reset_mock() + with patch('builtins.open', m_open, create=True): + self.disk_builder.create_disk() + + filesystem.create_verity_layer.assert_called_once_with(10, 'tempfile') + assert mock_CloneDevice.return_value.clone.call_args_list == [ + call([self.device_map['varclone1']]), + call([self.device_map['bootclone1']]), + call([self.device_map['rootclone1']]) + ] + @patch('kiwi.builder.disk.FileSystem.new') @patch('kiwi.builder.disk.Command.run') @patch('kiwi.builder.disk.Defaults.get_grub_boot_directory_name') @@ -512,7 +611,9 @@ class TestDiskBuilder: self.setup.set_selinux_file_contexts.assert_called_once_with( '/etc/selinux/targeted/contexts/files/file_contexts' ) - self.disk_setup.get_disksize_mbytes.assert_called_once_with() + self.disk_setup.get_disksize_mbytes.assert_called_once_with( + root_clone=0, boot_clone=0 + ) assert self.loop_provider.create.call_args_list == [ call(), call() ] @@ -524,13 +625,13 @@ class TestDiskBuilder: self.firmware.get_efi_partition_size() ) self.disk.create_boot_partition.assert_called_once_with( - self.disk_setup.boot_partition_size() + self.disk_setup.boot_partition_size(), 0 ) self.disk.create_prep_partition.assert_called_once_with( self.firmware.get_prep_partition_size() ) self.disk.create_root_partition.assert_called_once_with( - 'all_free' + 'all_free', 0 ) self.disk.map_partitions.assert_called_once_with() self.bootloader_config.setup_disk_boot_images.assert_called_once_with( @@ -662,7 +763,6 @@ class TestDiskBuilder: self.disk.public_partition_id_map = self.id_map self.disk.public_partition_id_map['kiwi_ROPart'] = 1 m_open = mock_open() -# FIXME with patch('builtins.open', m_open, create=True): self.disk_builder.create_disk() @@ -692,7 +792,9 @@ class TestDiskBuilder: self.integrity_root.create_integrity_metadata.assert_called_once_with() self.integrity_root.sign_integrity_metadata.assert_called_once_with() self.integrity_root.write_integrity_metadata.assert_called_once_with() - self.disk.create_root_readonly_partition.assert_called_once_with(11) + self.disk.create_root_readonly_partition.assert_called_once_with( + 11, 0 + ) assert mock_command.call_args_list[2] == call( ['blockdev', '--getsize64', '/dev/integrityRoot'] ) @@ -805,7 +907,7 @@ class TestDiskBuilder: self.disk_builder.create_disk() self.disk.create_root_raid_partition.assert_called_once_with( - 'all_free' + 'all_free', 0 ) self.raid_root.create_degraded_raid.assert_called_once_with( raid_level='mirroring' @@ -931,7 +1033,9 @@ class TestDiskBuilder: with patch('builtins.open'): self.disk_builder.create_disk() - self.disk.create_root_lvm_partition.assert_called_once_with('all_free') + self.disk.create_root_lvm_partition.assert_called_once_with( + 'all_free', 0 + ) volume_manager.setup.assert_called_once_with('systemVG') volume_manager.create_volumes.assert_called_once_with('btrfs') volume_manager.mount_volumes.call_args_list[0].assert_called_once_with()