diff --git a/doc/source/image_description/elements.rst b/doc/source/image_description/elements.rst index 04ee5a49..25627d60 100644 --- a/doc/source/image_description/elements.rst +++ b/doc/source/image_description/elements.rst @@ -485,6 +485,20 @@ kernelcmdline="string": Additional kernel parameters passed to the kernel by the bootloader. +root_clone="number" + For oem disk images, this attribute allows to create `number` + clone(s) of the root partition, with `number` >= 1. A clone partition + is content wise an exact byte for byte copy of the origin root partition. + However, to avoid conflicts at boot time the UUID of any + cloned partition will be made unique. In the sequence of partitions, + the clone(s) will always be created first followed by the + partition considered the origin. The origin partition is the + one that will be referenced and used by the system. + Also see :ref:`clone_partitions` + +boot_clone="number" + Same as `root_clone` but applied to the boot partition if present + luks="passphrase|file:///path/to/keyfile": Supplying a value will trigger the encryption of the partition serving the root filesystem using the LUKS extension. The supplied diff --git a/doc/source/working_with_images.rst b/doc/source/working_with_images.rst index 9473bee9..ae913c8f 100644 --- a/doc/source/working_with_images.rst +++ b/doc/source/working_with_images.rst @@ -23,6 +23,7 @@ Working with Images working_with_images/custom_partitions working_with_images/custom_volumes + working_with_images/clone_partitions working_with_images/setup_network_bootserver working_with_images/legacy_netboot_root_filesystem diff --git a/doc/source/working_with_images/clone_partitions.rst b/doc/source/working_with_images/clone_partitions.rst new file mode 100644 index 00000000..4c4c5a4f --- /dev/null +++ b/doc/source/working_with_images/clone_partitions.rst @@ -0,0 +1,128 @@ +.. _clone_partitions: + +Partition Clones +================ + +.. sidebar:: Abstract + + This page provides details about the partition clone feature + and its use cases + +{kiwi} allows to create block level clones of certain partitions +used in the image. Clones can be created from the `root`, `boot` +and any other partition listed in the `` element. + +A partition clone is a simple byte dump from one +block storage device to another. However, this would cause conflicts +during boot of the system because all unique identifiers like +the UUID of a filesystem will no longer be unique. The clone +feature of {kiwi} takes care of this part and re-creates the +relevant unique identifiers per cloned partition. {kiwi} allows +this also for complex partitions like LVM, LUKS or RAID. + +The partition clone(s) will always appear first in the partition table, +followed by the origin partition. The origin partition is the one +whose identifier will be referenced and used by the system. By default +no cloned partition will be mounted or used by the system at boot time. + +Let's take a look at the following example: + +.. code:: xml + + + + + + + +With the above setup {kiwi} will create a disk image that +contains the following partition table: + +.. code:: + + Number Start (sector) End (sector) Size Code Name + 1 2048 6143 2.0 MiB EF02 p.legacy + 2 6144 47103 20.0 MiB EF00 p.UEFI + 3 47104 661503 300.0 MiB 8300 p.lxbootclone1 + 4 661504 1275903 300.0 MiB 8300 p.lxboot + 5 1275904 1296383 10.0 MiB 8300 p.lxhomeclone1 + 6 1296384 1316863 10.0 MiB 8300 p.lxhomeclone2 + 7 1316864 1337343 10.0 MiB 8300 p.lxhome + 8 1337344 3864575 1.2 GiB 8300 p.lxrootclone1 + 9 3864576 6287326 1.2 GiB 8300 p.lxroot + +When booting the system only the origin partitions `p.lxboot`, `p.lxroot` +and `p.lxhome` will be mounted and visible in e.g. :file:`/etc/fstab`, +the bootloader or the initrd. Thus partition clones are present as a data +source but are not relevant for the operating system from a functional +perspective. + +As shown in the above example there is one clone request for root and boot +and a two clone requests for the home partition. {kiwi} does not sanity- +check the provided number of clones (e.g. whether your partition table +can hold that many partitions). + +.. warning:: + + There is a limit how many partitions a partition table can hold. + This also limits how many clones can be created. + +Use Case +-------- + +Potential use cases for which a clone of one or more partitions +is useful include among others: + +Factory Resets: + Creating an image with the option to rollback to the + state of the system at deployment time can be very helpful + for disaster recovery + +System Updates with Rollbacks e.g A/B: + Creating an image which holds extra space allowing to rollback + modified data can make a system more robust. For example + in a simple A/B update concept, partition A would get updated + but would flip to B if A is considered broken after applying the + update. + +.. note:: + + Most probably any use case based on partition clones requires + additional software to manage them. {kiwi} provides the + option to create the clone layout but it does not provide + the software to implement the actual use case for which the + partition clones are needed. + +Developers writing applications based on a clone layout created +with {kiwi} can leverage the metadata file :file:`/config.partids`. +This file is created at build time and contains the mapping between +the partition `name` and the actual partition number in the partition +table. For partition clones, the following naming convention applies: + +.. code:: + + kiwi_(name)PartClone(id)="(partition_number)" + +The `(name)` is either taken from the `name` attribute +of the `` element or it is a fixed name assigned by {kiwi}. +There are the following reserved partition names for which cloning +is supported: + +* root +* readonly +* boot + +For the mentioned example this will result in the +following :file:`/config.partids`: + +.. code:: + + kiwi_BiosGrub="1" + kiwi_EfiPart="2" + kiwi_bootPartClone1="3" + kiwi_BootPart="4" + kiwi_homePartClone1="5" + kiwi_homePartClone2="6" + kiwi_HomePart="7" + kiwi_rootPartClone1="8" + kiwi_RootPart="9" diff --git a/doc/source/working_with_images/custom_partitions.rst b/doc/source/working_with_images/custom_partitions.rst index 261543a4..6d40bf2f 100644 --- a/doc/source/working_with_images/custom_partitions.rst +++ b/doc/source/working_with_images/custom_partitions.rst @@ -71,6 +71,16 @@ filesystem="btrfs|ext2|ext3|ext4|squashfs|xfs Mandatory filesystem configuration to create one of the supported filesystems on the partition. +clone="number" + Optional setting to indicate that this partition should be + cloned `number` of times. A clone partition is content wise an + exact byte for byte copy of the origin. However, to avoid conflicts at boot + time the UUID of any cloned partition will be made unique. In the + sequence of partitions, the clone(s) will always be created first + followed by the partition considered the origin. The origin + partition is the one that will be referenced and used by the + system + Despite the customization options of the partition table shown above there are the following limitations: diff --git a/kiwi/builder/disk.py b/kiwi/builder/disk.py index d1a87496..59aca16e 100644 --- a/kiwi/builder/disk.py +++ b/kiwi/builder/disk.py @@ -36,6 +36,7 @@ from kiwi.system.identifier import SystemIdentifier from kiwi.boot.image import BootImage from kiwi.storage.setup import DiskSetup from kiwi.storage.loop_device import LoopDevice +from kiwi.storage.clone_device import CloneDevice from kiwi.firmware import FirmWare from kiwi.storage.disk import Disk from kiwi.storage.raid_device import RaidDevice @@ -105,6 +106,10 @@ class DiskBuilder: xml_state.build_type.get_embed_verity_metadata() self.dosparttable_extended_layout = \ xml_state.build_type.get_dosparttable_extended_layout() + self.boot_clone_count = int(xml_state.build_type.get_boot_clone()) \ + if xml_state.build_type.get_boot_clone() else 0 + self.root_clone_count = int(xml_state.build_type.get_root_clone()) \ + if xml_state.build_type.get_root_clone() else 0 self.custom_root_mount_args = xml_state.get_fs_mount_option_list() self.custom_root_creation_args = xml_state.get_fs_create_option_list() self.build_type_name = xml_state.get_build_type_name() @@ -235,7 +240,7 @@ class DiskBuilder: # a list of instances with the sync_data capability # representing the custom partitions area of the disk - system_custom_parts: List[FileSystemBase] = [] + system_custom_parts: Dict[str, FileSystemBase] = {} if self.install_media and self.build_type_name != 'oem': raise KiwiInstallMediaError( @@ -258,7 +263,9 @@ class DiskBuilder: self.boot_image.prepare() # precalculate needed disk size - disksize_mbytes = self.disk_setup.get_disksize_mbytes() + disksize_mbytes = self.disk_setup.get_disksize_mbytes( + root_clone=self.root_clone_count, boot_clone=self.boot_clone_count + ) # create the disk log.info('Creating raw disk image %s', self.diskname) @@ -736,8 +743,8 @@ class DiskBuilder: def _build_custom_parts_filesystem( self, device_map: Dict, custom_partitions: Dict['str', ptable_entry_type] - ) -> List[FileSystemBase]: - filesystem_list = [] + ) -> Dict[str, FileSystemBase]: + filesystem_dict = {} if custom_partitions: for map_name in sorted(custom_partitions.keys()): if map_name in device_map: @@ -751,8 +758,8 @@ class DiskBuilder: filesystem.create_on_device( label=map_name.upper() ) - filesystem_list.append(filesystem) - return filesystem_list + filesystem_dict[map_name] = filesystem + return filesystem_dict def _build_spare_filesystem(self, device_map: Dict) -> Optional[FileSystemBase]: if 'spare' in device_map and self.spare_part_fs: @@ -845,12 +852,18 @@ class DiskBuilder: disksize_used_mbytes += partition_mbsize if self.disk_setup.need_boot_partition(): - log.info('--> creating boot partition') + log.info( + '--> creating boot partition [with {0} clone(s)]'.format( + self.boot_clone_count + ) + ) partition_mbsize = self.disk_setup.boot_partition_size() disk.create_boot_partition( - partition_mbsize + partition_mbsize, self.boot_clone_count ) - disksize_used_mbytes += partition_mbsize + disksize_used_mbytes += \ + (self.boot_clone_count + 1) * partition_mbsize if \ + self.boot_clone_count else partition_mbsize if self.swap_mbytes: if not self.volume_manager_name or self.volume_manager_name != 'lvm': @@ -894,9 +907,11 @@ class DiskBuilder: os.path.getsize(squashed_root_file.name) / 1048576 ) + Defaults.get_min_partition_mbytes() disk.create_root_readonly_partition( - squashed_rootfs_mbsize + squashed_rootfs_mbsize, self.root_clone_count ) - disksize_used_mbytes += squashed_rootfs_mbsize + disksize_used_mbytes += \ + (self.root_clone_count + 1) * squashed_rootfs_mbsize if \ + self.root_clone_count else squashed_rootfs_mbsize if self.spare_part_mbsize and self.spare_part_is_last: rootfs_mbsize = disksize_mbytes - disksize_used_mbytes - \ @@ -910,15 +925,39 @@ class DiskBuilder: '--> overlayroot explicitly requested no write partition' ) else: + root_clone_count = self.root_clone_count + if self.root_filesystem_is_overlay: + # in overlay mode an eventual root clone is created from + # the root readonly partition and not from the root (rw) + # partition. Thus no further action needed here in this + # case + root_clone_count = 0 + if root_clone_count: + clone_rootfs_mbsize = int( + (disksize_mbytes - disksize_used_mbytes) / (root_clone_count + 1) + ) + Defaults.get_min_partition_mbytes() + rootfs_mbsize = f'clone:all_free:{clone_rootfs_mbsize}' if self.volume_manager_name and self.volume_manager_name == 'lvm': - log.info('--> creating LVM root partition') - disk.create_root_lvm_partition(rootfs_mbsize) + log.info( + '--> creating {0} partition [with {1} clone(s)]'.format( + 'root(LVM)', root_clone_count + ) + ) + disk.create_root_lvm_partition(rootfs_mbsize, root_clone_count) elif self.mdraid: - log.info('--> creating mdraid root partition') - disk.create_root_raid_partition(rootfs_mbsize) + log.info( + '--> creating {0} partition [with {1} clone(s)]'.format( + f'root(mdraid={self.mdraid})', root_clone_count + ) + ) + disk.create_root_raid_partition(rootfs_mbsize, root_clone_count) else: - log.info('--> creating root partition') - disk.create_root_partition(rootfs_mbsize) + log.info( + '--> creating root partition [with {0} clone(s)]'.format( + root_clone_count + ) + ) + disk.create_root_partition(rootfs_mbsize, root_clone_count) if self.spare_part_mbsize and self.spare_part_is_last: log.info('--> creating spare partition') @@ -944,7 +983,11 @@ class DiskBuilder: disk.map_partitions() - return disk.get_device() + device_map = disk.get_device() + device_map['origin_root'] = \ + device_map.get('readonly') or device_map['root'] + + return device_map def _write_partition_id_config_to_boot_image(self, disk: Disk) -> None: log.info('Creating config.partids in boot system') @@ -1200,16 +1243,27 @@ class DiskBuilder: system_boot: Optional[FileSystemBase], system_efi: Optional[FileSystemBase], system_spare: Optional[FileSystemBase], - system_custom_parts: List[FileSystemBase] + system_custom_parts: Dict[str, FileSystemBase] ) -> None: log.info('Syncing system to image') if system_spare: log.info('--> Syncing spare partition data') system_spare.sync_data() - for system_custom_part in system_custom_parts: + for map_name in sorted(system_custom_parts.keys()): + system_custom_part = system_custom_parts[map_name] log.info('--> Syncing custom partition(s) data') system_custom_part.sync_data() + if device_map.get(f'{map_name}clone1'): + log.info( + f'--> Dumping {map_name} clone data at extra partition' + ) + system_custom_part_clone = CloneDevice( + system_custom_part.device_provider, self.root_dir + ) + system_custom_part_clone.clone( + self._get_clone_devices(f'{map_name}clone', device_map) + ) if system_efi: log.info('--> Syncing EFI boot data to EFI partition') @@ -1220,6 +1274,13 @@ class DiskBuilder: system_boot.sync_data( self._get_exclude_list_for_boot_data_sync() ) + if device_map.get('bootclone1'): + log.info( + '--> Dumping boot clone data at extra partition' + ) + CloneDevice(system_boot.device_provider, self.root_dir).clone( + self._get_clone_devices('bootclone', device_map) + ) log.info('--> Syncing root filesystem data') if self.root_filesystem_is_overlay: @@ -1270,6 +1331,13 @@ class DiskBuilder: squashed_root.create_verification_metadata( readonly_target ) + if device_map.get('rootclone1'): + log.info( + '--> Dumping readonly root clone data at extra partition' + ) + CloneDevice(device_map['origin_root'], self.root_dir).clone( + self._get_clone_devices('rootclone', device_map) + ) elif self.root_filesystem_verity_blocks: root_target = device_map['root'].get_device() root_target_bytesize = device_map['root'].get_byte_size( @@ -1328,10 +1396,24 @@ class DiskBuilder: filesystem.create_verification_metadata( root_target ) + if device_map.get('rootclone1'): + log.info( + '--> Dumping root clone data at extra partition' + ) + CloneDevice(device_map['origin_root'], self.root_dir).clone( + self._get_clone_devices('rootclone', device_map) + ) else: system.sync_data( self._get_exclude_list_for_root_data_sync(device_map) ) + if device_map.get('rootclone1'): + log.info( + '--> Dumping root clone data at extra partition' + ) + CloneDevice(device_map['origin_root'], self.root_dir).clone( + self._get_clone_devices('rootclone', device_map) + ) if self.integrity_root and \ self.root_filesystem_embed_integrity_metadata: @@ -1450,3 +1532,12 @@ class DiskBuilder: self.root_dir + ''.join(['/boot/', boot_names.initrd_name]) ] ) + + def _get_clone_devices( + self, match: str, device_map: Dict[str, DeviceProvider] + ) -> List[DeviceProvider]: + result = [] + for map_name in sorted(device_map.keys()): + if map_name.startswith(match): + result.append(device_map[map_name]) + return result diff --git a/kiwi/schema/kiwi.rnc b/kiwi/schema/kiwi.rnc index bd7db3c3..743889f9 100644 --- a/kiwi/schema/kiwi.rnc +++ b/kiwi/schema/kiwi.rnc @@ -29,6 +29,7 @@ safe-posix-short-name = xsd:token {pattern = "[a-zA-Z0-9_\-\.]{1,32}"} locale-name = xsd:token {pattern = "(POSIX|[a-z]{2,3}_[A-Z]{2})(,[a-z]{2,3}_[A-Z]{2})*"} mac-address-type = xsd:token {pattern = "([0-9a-fA-F]{2}:){5}[0-9a-fA-F]{2}"} size-type = xsd:token {pattern = "(\d*|image)"} +number-type = xsd:token {pattern = "\d+"} blocks-type = xsd:token {pattern = "(\d*|all)"} volume-size-type = xsd:token {pattern = "(\d+|\d+M|\d+G|all)"} partition-size-type = xsd:token {pattern = "(\d+|\d+M|\d+G)"} @@ -2049,6 +2050,25 @@ div { sch:param [ name = "attr" value = "disk_start_sector" ] sch:param [ name = "types" value = "oem" ] ] + k.type.root_clone.attribute = + ## Clone root partition N times + attribute root_clone { number-type } + >> sch:pattern [ id = "root_clone" is-a = "image_type" + sch:param [ name = "attr" value = "root_clone" ] + sch:param [ name = "types" value = "oem" ] + ] + k.type.boot_clone.attribute = + ## Clone boot partition N times. If no boot partition is + ## used, the attribute has no effect. The use of a boot + ## partition can be enforced through the bootpartition + ## attribute or is implicitly activated according to the + ## combination of type settings that makes the use of an + ## extra boot partition a requirement + attribute boot_clone { number-type } + >> sch:pattern [ id = "boot_clone" is-a = "image_type" + sch:param [ name = "attr" value = "boot_clone" ] + sch:param [ name = "types" value = "oem" ] + ] k.type.bundle_format.attribute = ## Specifies the bundle format pattern ## The format string can contain placeholders for the @@ -2141,6 +2161,8 @@ div { k.type.xen_server.attribute? & k.type.publisher.attribute? & k.type.disk_start_sector.attribute? & + k.type.root_clone.attribute? & + k.type.boot_clone.attribute? & k.type.bundle_format.attribute? k.type = ## The Image Type of the Logical Extend @@ -2375,13 +2397,17 @@ div { attribute filesystem { "btrfs" | "ext2" | "ext3" | "ext4" | "squashfs" | "xfs" } + k.partition.clone.attribute = + ## Clone this partition N times + attribute clone { number-type } k.partition.attlist = k.partition.name.attribute & k.partition.size.attribute & k.partition.partition_name.attribute? & k.partition.partition_type.attribute? & k.partition.mountpoint.attribute? & - k.partition.filesystem.attribute? + k.partition.filesystem.attribute? & + k.partition.clone.attribute? k.partition = ## Specify custom partition in the partition table element partition { diff --git a/kiwi/schema/kiwi.rng b/kiwi/schema/kiwi.rng index d9df83a1..bbd0ef1d 100644 --- a/kiwi/schema/kiwi.rng +++ b/kiwi/schema/kiwi.rng @@ -46,6 +46,11 @@ (\d*|image) + + + \d+ + + (\d*|all) @@ -2918,6 +2923,31 @@ default. + + + Clone root partition N times + + + + + + + + + + Clone boot partition N times. If no boot partition is +used, the attribute has no effect. The use of a boot +partition can be enforced through the bootpartition +attribute or is implicitly activated according to the +combination of type settings that makes the use of an +extra boot partition a requirement + + + + + + + Specifies the bundle format pattern @@ -3156,6 +3186,12 @@ kiwi-ng result bundle ... + + + + + + @@ -3556,6 +3592,12 @@ Allowed values are: t.linux + + + Clone this partition N times + + + @@ -3572,6 +3614,9 @@ Allowed values are: t.linux + + + diff --git a/kiwi/xml_parse.py b/kiwi/xml_parse.py index 69d3b145..64671c57 100644 --- a/kiwi/xml_parse.py +++ b/kiwi/xml_parse.py @@ -3,7 +3,7 @@ # # Generated by generateDS.py version 2.29.24. -# Python 3.10.4 (main, Apr 2 2022, 09:04:19) [GCC 11.2.0] +# Python 3.6.15 (default, Sep 23 2021, 15:41:43) [GCC] # # Command line options: # ('-f', '') @@ -16,7 +16,7 @@ # kiwi/schema/kiwi_for_generateDS.xsd # # Command line: -# /mnt/storage/kiwi/.tox/3/bin/generateDS.py -f --external-encoding="utf-8" --no-dates --no-warnings -o "kiwi/xml_parse.py" kiwi/schema/kiwi_for_generateDS.xsd +# /home/ms/Project/kiwi/.tox/3.6/bin/generateDS.py -f --external-encoding="utf-8" --no-dates --no-warnings -o "kiwi/xml_parse.py" kiwi/schema/kiwi_for_generateDS.xsd # # Current working directory (os.getcwd()): # kiwi @@ -2798,7 +2798,7 @@ class type_(GeneratedsSuper): """The Image Type of the Logical Extend""" subclass = None superclass = None - def __init__(self, boot=None, bootfilesystem=None, firmware=None, bootkernel=None, bootpartition=None, bootpartsize=None, efipartsize=None, efiparttable=None, dosparttable_extended_layout=None, bootprofile=None, btrfs_quota_groups=None, btrfs_root_is_snapshot=None, btrfs_root_is_readonly_snapshot=None, compressed=None, devicepersistency=None, editbootconfig=None, editbootinstall=None, filesystem=None, flags=None, format=None, formatoptions=None, fsmountoptions=None, fscreateoptions=None, squashfscompression=None, gcelicense=None, hybridpersistent=None, hybridpersistent_filesystem=None, gpt_hybrid_mbr=None, force_mbr=None, initrd_system=None, image=None, metadata_path=None, installboot=None, install_continue_on_timeout=None, installprovidefailsafe=None, installiso=None, installstick=None, installpxe=None, mediacheck=None, kernelcmdline=None, luks=None, luks_version=None, luksOS=None, mdraid=None, overlayroot=None, overlayroot_write_partition=None, overlayroot_readonly_partsize=None, verity_blocks=None, embed_verity_metadata=None, standalone_integrity=None, embed_integrity_metadata=None, integrity_metadata_key_description=None, integrity_keyfile=None, primary=None, ramonly=None, rootfs_label=None, spare_part=None, spare_part_mountpoint=None, spare_part_fs=None, spare_part_fs_attributes=None, spare_part_is_last=None, target_blocksize=None, target_removable=None, vga=None, vhdfixedtag=None, volid=None, wwid_wait_timeout=None, derived_from=None, ensure_empty_tmpdirs=None, xen_server=None, publisher=None, disk_start_sector=None, bundle_format=None, bootloader=None, containerconfig=None, machine=None, oemconfig=None, size=None, systemdisk=None, partitions=None, vagrantconfig=None, installmedia=None, luksformat=None): + def __init__(self, boot=None, bootfilesystem=None, firmware=None, bootkernel=None, bootpartition=None, bootpartsize=None, efipartsize=None, efiparttable=None, dosparttable_extended_layout=None, bootprofile=None, btrfs_quota_groups=None, btrfs_root_is_snapshot=None, btrfs_root_is_readonly_snapshot=None, compressed=None, devicepersistency=None, editbootconfig=None, editbootinstall=None, filesystem=None, flags=None, format=None, formatoptions=None, fsmountoptions=None, fscreateoptions=None, squashfscompression=None, gcelicense=None, hybridpersistent=None, hybridpersistent_filesystem=None, gpt_hybrid_mbr=None, force_mbr=None, initrd_system=None, image=None, metadata_path=None, installboot=None, install_continue_on_timeout=None, installprovidefailsafe=None, installiso=None, installstick=None, installpxe=None, mediacheck=None, kernelcmdline=None, luks=None, luks_version=None, luksOS=None, mdraid=None, overlayroot=None, overlayroot_write_partition=None, overlayroot_readonly_partsize=None, verity_blocks=None, embed_verity_metadata=None, standalone_integrity=None, embed_integrity_metadata=None, integrity_metadata_key_description=None, integrity_keyfile=None, primary=None, ramonly=None, rootfs_label=None, spare_part=None, spare_part_mountpoint=None, spare_part_fs=None, spare_part_fs_attributes=None, spare_part_is_last=None, target_blocksize=None, target_removable=None, vga=None, vhdfixedtag=None, volid=None, wwid_wait_timeout=None, derived_from=None, ensure_empty_tmpdirs=None, xen_server=None, publisher=None, disk_start_sector=None, root_clone=None, boot_clone=None, bundle_format=None, bootloader=None, containerconfig=None, machine=None, oemconfig=None, size=None, systemdisk=None, partitions=None, vagrantconfig=None, installmedia=None, luksformat=None): self.original_tagname_ = None self.boot = _cast(None, boot) self.bootfilesystem = _cast(None, bootfilesystem) @@ -2872,6 +2872,8 @@ class type_(GeneratedsSuper): self.xen_server = _cast(bool, xen_server) self.publisher = _cast(None, publisher) self.disk_start_sector = _cast(int, disk_start_sector) + self.root_clone = _cast(None, root_clone) + self.boot_clone = _cast(None, boot_clone) self.bundle_format = _cast(None, bundle_format) if bootloader is None: self.bootloader = [] @@ -3118,6 +3120,10 @@ class type_(GeneratedsSuper): def set_publisher(self, publisher): self.publisher = publisher def get_disk_start_sector(self): return self.disk_start_sector def set_disk_start_sector(self, disk_start_sector): self.disk_start_sector = disk_start_sector + def get_root_clone(self): return self.root_clone + def set_root_clone(self, root_clone): self.root_clone = root_clone + def get_boot_clone(self): return self.boot_clone + def set_boot_clone(self, boot_clone): self.boot_clone = boot_clone def get_bundle_format(self): return self.bundle_format def set_bundle_format(self, bundle_format): self.bundle_format = bundle_format def validate_blocks_type(self, value): @@ -3155,6 +3161,13 @@ class type_(GeneratedsSuper): self.validate_safe_posix_short_name_patterns_, value): warnings_.warn('Value "%s" does not match xsd pattern restrictions: %s' % (value.encode('utf-8'), self.validate_safe_posix_short_name_patterns_, )) validate_safe_posix_short_name_patterns_ = [['^[a-zA-Z0-9_\\-\\.]{1,32}$']] + def validate_number_type(self, value): + # Validate type number-type, a restriction on xs:token. + if value is not None and Validate_simpletypes_: + if not self.gds_validate_simple_patterns( + self.validate_number_type_patterns_, value): + warnings_.warn('Value "%s" does not match xsd pattern restrictions: %s' % (value.encode('utf-8'), self.validate_number_type_patterns_, )) + validate_number_type_patterns_ = [['^\\d+$']] def hasContent_(self): if ( self.bootloader or @@ -3409,6 +3422,12 @@ class type_(GeneratedsSuper): if self.disk_start_sector is not None and 'disk_start_sector' not in already_processed: already_processed.add('disk_start_sector') outfile.write(' disk_start_sector="%s"' % self.gds_format_integer(self.disk_start_sector, input_name='disk_start_sector')) + if self.root_clone is not None and 'root_clone' not in already_processed: + already_processed.add('root_clone') + outfile.write(' root_clone=%s' % (quote_attrib(self.root_clone), )) + if self.boot_clone is not None and 'boot_clone' not in already_processed: + already_processed.add('boot_clone') + outfile.write(' boot_clone=%s' % (quote_attrib(self.boot_clone), )) if self.bundle_format is not None and 'bundle_format' not in already_processed: already_processed.add('bundle_format') outfile.write(' bundle_format=%s' % (self.gds_encode(self.gds_format_string(quote_attrib(self.bundle_format), input_name='bundle_format')), )) @@ -3917,6 +3936,18 @@ class type_(GeneratedsSuper): self.disk_start_sector = int(value) except ValueError as exp: raise_parse_error(node, 'Bad integer attribute: %s' % exp) + value = find_attr_value_('root_clone', node) + if value is not None and 'root_clone' not in already_processed: + already_processed.add('root_clone') + self.root_clone = value + self.root_clone = ' '.join(self.root_clone.split()) + self.validate_number_type(self.root_clone) # validate type number-type + value = find_attr_value_('boot_clone', node) + if value is not None and 'boot_clone' not in already_processed: + already_processed.add('boot_clone') + self.boot_clone = value + self.boot_clone = ' '.join(self.boot_clone.split()) + self.validate_number_type(self.boot_clone) # validate type number-type value = find_attr_value_('bundle_format', node) if value is not None and 'bundle_format' not in already_processed: already_processed.add('bundle_format') @@ -4447,7 +4478,7 @@ class partition(GeneratedsSuper): """Specify custom partition in the partition table""" subclass = None superclass = None - def __init__(self, name=None, size=None, partition_name=None, partition_type=None, mountpoint=None, filesystem=None): + def __init__(self, name=None, size=None, partition_name=None, partition_type=None, mountpoint=None, filesystem=None, clone=None): self.original_tagname_ = None self.name = _cast(None, name) self.size = _cast(None, size) @@ -4455,6 +4486,7 @@ class partition(GeneratedsSuper): self.partition_type = _cast(None, partition_type) self.mountpoint = _cast(None, mountpoint) self.filesystem = _cast(None, filesystem) + self.clone = _cast(None, clone) def factory(*args_, **kwargs_): if CurrentSubclassModule_ is not None: subclass = getSubclassFromModule_( @@ -4478,6 +4510,8 @@ class partition(GeneratedsSuper): def set_mountpoint(self, mountpoint): self.mountpoint = mountpoint def get_filesystem(self): return self.filesystem def set_filesystem(self, filesystem): self.filesystem = filesystem + def get_clone(self): return self.clone + def set_clone(self, clone): self.clone = clone def validate_partition_size_type(self, value): # Validate type partition-size-type, a restriction on xs:token. if value is not None and Validate_simpletypes_: @@ -4492,6 +4526,13 @@ class partition(GeneratedsSuper): self.validate_safe_posix_short_name_patterns_, value): warnings_.warn('Value "%s" does not match xsd pattern restrictions: %s' % (value.encode('utf-8'), self.validate_safe_posix_short_name_patterns_, )) validate_safe_posix_short_name_patterns_ = [['^[a-zA-Z0-9_\\-\\.]{1,32}$']] + def validate_number_type(self, value): + # Validate type number-type, a restriction on xs:token. + if value is not None and Validate_simpletypes_: + if not self.gds_validate_simple_patterns( + self.validate_number_type_patterns_, value): + warnings_.warn('Value "%s" does not match xsd pattern restrictions: %s' % (value.encode('utf-8'), self.validate_number_type_patterns_, )) + validate_number_type_patterns_ = [['^\\d+$']] def hasContent_(self): if ( @@ -4538,6 +4579,9 @@ class partition(GeneratedsSuper): if self.filesystem is not None and 'filesystem' not in already_processed: already_processed.add('filesystem') outfile.write(' filesystem=%s' % (self.gds_encode(self.gds_format_string(quote_attrib(self.filesystem), input_name='filesystem')), )) + if self.clone is not None and 'clone' not in already_processed: + already_processed.add('clone') + outfile.write(' clone=%s' % (quote_attrib(self.clone), )) def exportChildren(self, outfile, level, namespaceprefix_='', name_='partition', fromsubclass_=False, pretty_print=True): pass def build(self, node): @@ -4578,6 +4622,12 @@ class partition(GeneratedsSuper): already_processed.add('filesystem') self.filesystem = value self.filesystem = ' '.join(self.filesystem.split()) + value = find_attr_value_('clone', node) + if value is not None and 'clone' not in already_processed: + already_processed.add('clone') + self.clone = value + self.clone = ' '.join(self.clone.split()) + self.validate_number_type(self.clone) # validate type number-type def buildChildren(self, child_, node, nodeName_, fromsubclass_=False): pass # end class partition diff --git a/kiwi/xml_state.py b/kiwi/xml_state.py index 3c44cdf7..9d4461f4 100644 --- a/kiwi/xml_state.py +++ b/kiwi/xml_state.py @@ -1437,11 +1437,7 @@ class XMLState: partition_name = partition.get_partition_name() or f'p.lx{name}' partitions[name] = ptable_entry_type( mbsize=self._to_mega_byte(partition.get_size()), - # There is currently no clone attribute in the - # element. This will be added on completion of the partition - # clone feature. The internal API structure however, already - # knows about the capability - clone=0, + clone=int(partition.get_clone()) if partition.get_clone() else 0, partition_name=partition_name, partition_type=partition.get_partition_type() or 't.linux', mountpoint=partition.get_mountpoint(), diff --git a/test/unit/builder/disk_test.py b/test/unit/builder/disk_test.py index 869fe68b..eacf837d 100644 --- a/test/unit/builder/disk_test.py +++ b/test/unit/builder/disk_test.py @@ -319,7 +319,9 @@ class TestDiskBuilder: self.setup.set_selinux_file_contexts.assert_called_once_with( '/etc/selinux/targeted/contexts/files/file_contexts' ) - self.disk_setup.get_disksize_mbytes.assert_called_once_with() + self.disk_setup.get_disksize_mbytes.assert_called_once_with( + root_clone=0, boot_clone=0 + ) self.loop_provider.create.assert_called_once_with() self.disk.wipe.assert_called_once_with() self.disk.create_efi_csm_partition.assert_called_once_with( @@ -329,7 +331,7 @@ class TestDiskBuilder: self.firmware.get_efi_partition_size() ) self.disk.create_boot_partition.assert_called_once_with( - self.disk_setup.boot_partition_size() + self.disk_setup.boot_partition_size(), 0 ) self.disk.create_swap_partition.assert_called_once_with( '128' @@ -338,7 +340,7 @@ class TestDiskBuilder: self.firmware.get_prep_partition_size() ) self.disk.create_root_partition.assert_called_once_with( - 'all_free' + 'all_free', 0 ) self.disk.map_partitions.assert_called_once_with() self.bootloader_config.setup_disk_boot_images.assert_called_once_with( @@ -418,6 +420,103 @@ class TestDiskBuilder: 'target_dir' ) + @patch('kiwi.builder.disk.FileSystem.new') + @patch('kiwi.builder.disk.Command.run') + @patch('kiwi.builder.disk.Defaults.get_grub_boot_directory_name') + @patch('os.path.exists') + @patch('os.path.getsize') + @patch('kiwi.builder.disk.SystemSetup') + @patch('kiwi.builder.disk.ImageSystem') + @patch('kiwi.builder.disk.Temporary.new_file') + @patch('kiwi.builder.disk.CloneDevice') + def test_create_disk_standard_root_with_clone( + self, mock_CloneDevice, mock_Temporary_new_file, mock_ImageSystem, + mock_SystemSetup, mock_os_path_getsize, mock_path, + mock_grub_dir, mock_command, mock_fs + ): + tempfile = Mock() + tempfile.name = 'tempfile' + mock_Temporary_new_file.return_value = tempfile + mock_os_path_getsize.return_value = 42 + self.boot_image_task.get_boot_names.return_value = self.boot_names_type( + kernel_name='vmlinuz-1.2.3-default', + initrd_name='initramfs-1.2.3.img' + ) + mock_path.return_value = True + filesystem = Mock() + mock_fs.return_value = filesystem + self.disk_builder.custom_partitions = { + 'var': ptable_entry_type( + mbsize=100, + clone=1, + partition_name='p.lxvar', + partition_type='t.linux', + mountpoint='/var', + filesystem='ext3' + ) + } + self.disk_builder.root_clone_count = 1 + self.disk_builder.boot_clone_count = 1 + self.disk_builder.root_filesystem_is_overlay = False + self.disk_builder.volume_manager_name = None + self.disk_builder.initrd_system = 'dracut' + disk_system = Mock() + mock_SystemSetup.return_value = disk_system + + self.device_map['rootclone1'] = MappedDevice('/dev/root-device', Mock()) + self.device_map['bootclone1'] = MappedDevice('/dev/boot-device', Mock()) + self.device_map['varclone1'] = MappedDevice('/dev/var-device', Mock()) + + # Test in standard mode (no root overlay) + m_open = mock_open() + with patch('builtins.open', m_open, create=True): + self.disk_builder.create_disk() + + self.disk.create_boot_partition.assert_called_once_with( + self.disk_setup.boot_partition_size(), 1 + ) + self.disk.create_root_partition.assert_called_once_with( + 'clone:all_free:458', 1 + ) + self.disk.create_custom_partitions.assert_called_once_with( + self.disk_builder.custom_partitions + ) + assert mock_CloneDevice.return_value.clone.call_args_list == [ + call([self.device_map['varclone1']]), + call([self.device_map['bootclone1']]), + call([self.device_map['rootclone1']]) + ] + + # Test in overlay mode a root clone is created from + # the root readonly partition and not from the root (rw) + # partition. + self.disk_builder.root_filesystem_is_overlay = True + self.disk.create_root_partition.reset_mock() + with patch('builtins.open', m_open, create=True): + self.disk_builder.create_disk() + + self.disk.create_root_readonly_partition.assert_called_once_with( + 10, 1 + ) + self.disk.create_root_partition.assert_called_once_with( + 'all_free', 0 + ) + + # Test in verity mode + self.disk_builder.root_filesystem_verity_blocks = 10 + self.disk_builder.root_filesystem_is_overlay = False + mock_CloneDevice.reset_mock() + self.disk.create_root_partition.reset_mock() + with patch('builtins.open', m_open, create=True): + self.disk_builder.create_disk() + + filesystem.create_verity_layer.assert_called_once_with(10, 'tempfile') + assert mock_CloneDevice.return_value.clone.call_args_list == [ + call([self.device_map['varclone1']]), + call([self.device_map['bootclone1']]), + call([self.device_map['rootclone1']]) + ] + @patch('kiwi.builder.disk.FileSystem.new') @patch('kiwi.builder.disk.Command.run') @patch('kiwi.builder.disk.Defaults.get_grub_boot_directory_name') @@ -512,7 +611,9 @@ class TestDiskBuilder: self.setup.set_selinux_file_contexts.assert_called_once_with( '/etc/selinux/targeted/contexts/files/file_contexts' ) - self.disk_setup.get_disksize_mbytes.assert_called_once_with() + self.disk_setup.get_disksize_mbytes.assert_called_once_with( + root_clone=0, boot_clone=0 + ) assert self.loop_provider.create.call_args_list == [ call(), call() ] @@ -524,13 +625,13 @@ class TestDiskBuilder: self.firmware.get_efi_partition_size() ) self.disk.create_boot_partition.assert_called_once_with( - self.disk_setup.boot_partition_size() + self.disk_setup.boot_partition_size(), 0 ) self.disk.create_prep_partition.assert_called_once_with( self.firmware.get_prep_partition_size() ) self.disk.create_root_partition.assert_called_once_with( - 'all_free' + 'all_free', 0 ) self.disk.map_partitions.assert_called_once_with() self.bootloader_config.setup_disk_boot_images.assert_called_once_with( @@ -662,7 +763,6 @@ class TestDiskBuilder: self.disk.public_partition_id_map = self.id_map self.disk.public_partition_id_map['kiwi_ROPart'] = 1 m_open = mock_open() -# FIXME with patch('builtins.open', m_open, create=True): self.disk_builder.create_disk() @@ -692,7 +792,9 @@ class TestDiskBuilder: self.integrity_root.create_integrity_metadata.assert_called_once_with() self.integrity_root.sign_integrity_metadata.assert_called_once_with() self.integrity_root.write_integrity_metadata.assert_called_once_with() - self.disk.create_root_readonly_partition.assert_called_once_with(11) + self.disk.create_root_readonly_partition.assert_called_once_with( + 11, 0 + ) assert mock_command.call_args_list[2] == call( ['blockdev', '--getsize64', '/dev/integrityRoot'] ) @@ -805,7 +907,7 @@ class TestDiskBuilder: self.disk_builder.create_disk() self.disk.create_root_raid_partition.assert_called_once_with( - 'all_free' + 'all_free', 0 ) self.raid_root.create_degraded_raid.assert_called_once_with( raid_level='mirroring' @@ -931,7 +1033,9 @@ class TestDiskBuilder: with patch('builtins.open'): self.disk_builder.create_disk() - self.disk.create_root_lvm_partition.assert_called_once_with('all_free') + self.disk.create_root_lvm_partition.assert_called_once_with( + 'all_free', 0 + ) volume_manager.setup.assert_called_once_with('systemVG') volume_manager.create_volumes.assert_called_once_with('btrfs') volume_manager.mount_volumes.call_args_list[0].assert_called_once_with()