From 94de1336d811084adfe9661a400309c336300246 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marcus=20Sch=C3=A4fer?= Date: Thu, 21 Oct 2021 23:33:09 +0200 Subject: [PATCH] Support custom partitions In addition to the volume volume management settings also allow to setup low level table entries like in the following example: --- doc/source/image_description/elements.rst | 20 +- .../working_with_images/custom_partitions.rst | 201 +++++----- kiwi/builder/disk.py | 68 +++- kiwi/exceptions.py | 7 + kiwi/schema/kiwi.rnc | 110 ++++-- kiwi/schema/kiwi.rng | 168 +++++--- kiwi/storage/disk.py | 57 ++- kiwi/xml_parse.py | 367 ++++++++++++------ kiwi/xml_state.py | 55 +++ setup.cfg | 2 +- test/data/example_partitions_config.xml | 31 ++ test/unit/builder/disk_test.py | 36 +- test/unit/storage/disk_test.py | 36 +- test/unit/xml_state_test.py | 17 + 14 files changed, 841 insertions(+), 334 deletions(-) create mode 100644 test/data/example_partitions_config.xml diff --git a/doc/source/image_description/elements.rst b/doc/source/image_description/elements.rst index 4494175f..dfded175 100644 --- a/doc/source/image_description/elements.rst +++ b/doc/source/image_description/elements.rst @@ -707,8 +707,24 @@ that is being used as a vagrant box. For details see: :ref:`setup_vagrant` ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -Used to describe the geometry, partitions and volumes, in a -disk image. For details see: :ref:`custom_volumes` +Used to describe the volumes of the disk area which +contains the root filesystem. Volumes are either a feature +of the used filesystem or LVM is used for this purpose. +For details see: :ref:`custom_volumes` + +.. note:: + + When both `` and `` are used, `` + are evaluated first and mount points defined in `` cannot + be redefined as `` volumes. The two types define a + complete disk setup, so there cannot be any overlapping volumes + or mount points. As a result, whatever is written in `` + cannot be expressed in the same way in ``. + + +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +Used to describe the geometry of the disk on the level of the +partition table. For details see: :ref:`custom_partitions` ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ diff --git a/doc/source/working_with_images/custom_partitions.rst b/doc/source/working_with_images/custom_partitions.rst index 0ef8e158..261543a4 100644 --- a/doc/source/working_with_images/custom_partitions.rst +++ b/doc/source/working_with_images/custom_partitions.rst @@ -5,132 +5,107 @@ Custom Disk Partitions .. sidebar:: Abstract - This page provides some details about what {kiwi} supports and does - not support regarding customization over the partition scheme. It also - provides some guidance in case the user requires some custom layout - beyond {kiwi} supported features. - -By design, {kiwi} does not support a customized partition table. Alternatively, -{kiwi} supports the definition of user-defined volumes which covers most of -common use cases. See :ref:`Custom Disk Volumes ` for -further details about that. + This page provides details about the opportunities and limitations + to customize the partition table in addition to the volume management + settings from :ref:`custom_volumes`. {kiwi} has its own partitioning schema which is defined according to several different user configurations: boot firmware, boot partition, expandable layouts, etc. Those supported features have an impact on the -partitioning schema. MBR or GUID partition tables are not flexible, -carry limitations and are tied to some specific disk geometry. Because -of that the preferred alternative to disk layouts based on traditional -partition tables is using flexible approaches like logic volumes. +partitioning schema. -As an example, expandable OEM images is a relevant {kiwi} feature that -is incompatible with the idea of adding user defined partitions on the -system area. +MBR or GUID partition tables are not flexible, carry limitations and are +tied to some specific disk geometry. Because of that the preferred alternative +to disk layouts based on traditional partition tables is using flexible +approaches like logic volumes. -Despite no full customization is supported, some aspects of the partition -schema can be customized. {kiwi} supports: +However, on certain conditions additional entries to the low level +partition table are needed. For this purpose the `` section +exists and allows to add custom entries like shown in the following +example: -1. Adding a spare partition *before* the root (`/`) partition. +.. code:: xml - It can be achieved by using the `spare_part` type attribute. + + + -2. Leaving some unpartitioned area at the *end* of the disk. +Each `` entry puts a partition of the configured size in the +low level partition table, creates a filesystem on it and includes +it to the system's fstab file. If parts of the root filesystem are +moved into its own partition like it's the case in the above example, +this partition will also contain the data that gets installed during +the image creation process to that area. + +The following attributes must/can be set to configured a partition entry: + +name="identifier" + Mandatory name of the partition as handled by {kiwi}. + + .. note:: + + There are the following reserved names which cannot be used + because they are already represented by existing attributes: + `root`, `readonly`, `boot`, `prep`, `spare`, `swap`, `efi_csm` + and `efi`. + +partition_name="name" + Optional name of the partition as it appears when listing the + table contents with tools like `gdisk`. If no name is set + {kiwi} constructs a name of the form `p.lx(identifier_from_name_attr)` + +partition_type="type_identifier" + Optional partition type identifier as handled by {kiwi}. + Allowed values are `t.linux` and `t.raid`. If not specified + `t.linux` is the default. + +size="size_string" + Mandatory size of the partition. A size string can end with `M` or + `G` to indicate a mega-Byte or giga-Byte value. Without a unit + specification mega-Byte is used. + +mountpoint="path" + Mandatory mountpoint to mount the partition in the system. + +filesystem="btrfs|ext2|ext3|ext4|squashfs|xfs + Mandatory filesystem configuration to create one of the supported + filesystems on the partition. + +Despite the customization options of the partition table shown above +there are the following limitations: + +1. The root partition is always the last one + + Disk imags build with {kiwi} are designed to be expandable. + For this feature to work the partition containing the system + rootfs must always be the last one. If this is unwanted for + some reason {kiwi} offers an opportunity for one extra/spare + partition with the option to be also placed at the end of the + table. For details lookup `spare_part` in :ref:`image-description-elements` + +2. There can be no gaps in the partition table + + The way partitions are configured does not allow for gaps in the + table. As of today there was no use case were it made sense to + leave a gap between table entries. However, leaving some space + free at the **end** of the partition geometry is possible in the + following ways: + + * **Deploy with unpartitioned free space.** + + To leave space unpartitioned on first boot of a disk image + it is possible to configured an `` which is + smaller than the disk the image gets deployed to. Details + about this setting can be found in :ref:`image-description-elements` + + * **Build with unpartitioned free space.** Setting some unpartitioned free space on the disk can be done using - the `unpartitioned` attribute of `size` element in type's section. [LINK] + the `unpartitioned` attribute of `size` element in type's section. + For details see :ref:`disk-the-size-element` -3. Expand built disks to a new size adding unpartitioned free space at - the *end* of the disk. + * **Resize built image adding unpartitioned free space.** A built image can be resized by using the `kiwi-ng image resize` command and set a new extended size for the disk. See {kiwi} commands docs :ref:`here `. - -Custom Partitioning at Boot Time -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -Adding additional partitions at boot time of {kiwi} images is also possible, -however, setting the tools and scripts for doing so needs to be handled by -the user. A possible strategy to add partitions on system area are described -below. - -The main idea consists on running a first boot service that creates the -partitions that are needed. Adding custom services is simple, use the -following steps: - -1. Create a unit file for a systemd service: - - .. code:: shell - - [Unit] - Description=Add a data partition - After=basic.target - Wants=basic.target - - [Service] - Type=oneshot - ExecStart=/bin/bash /usr/local/bin/create_part - - - This systemd unit file will run at boot time once systemd reaches the basic - target. At this stage all basic services are up an running (devices mounted, - network interfaces up, etc.). In case the service is required to run on - earlier stages for some reason, default dependencies need to be disabled, - see `systemd man pages `_. - -2. Create partitioner shell script matching your specific needs - - Consider the following steps for a partitioner shell script that - creates a new partition. Following the above unit file example - the `/usr/local/bin/create_part` script should cover the following - steps: - - a. Verify partition exists - - Verify the required partition is not mounted neither exists. Exit - zero (0) if is already there. - - Use tools such `findmnt` to find the root device and `blkid` - or `lsblk` to find a partition with certain label or similar - criteria. - - b. Create a new partition - - Create a new partition. On error, exit with non zero. - - Use partitioner tools such as `sgdisk` that can be easily used - in non interactive scripts. Using `partprobe` to reload partition - table to make OS aware of the changes is handy. - - c. Make filesystem - - Add the desired filesystem to the new partitions. On error, exit - with non zero. - - Regular filesystem formatting tools (`mkfs.ext4` just to mention one) - can be used to apply the desired filesystem to the just created - new partition. At this stage it is handy to add a label to the - filesystem for easy recognition on later stages or script reruns. - - d. Update fstab file - - Just echo and append the desired entry in /etc/fstab. - - e. Mount partition - - `mount --all` will try to mount all fstab volumes, it just omits - any already mounted device. - - -3. Add additional files into the root overlay tree. - - The above described unit files and partition creation shell script - need to be included into the overlay tree of the image, thus they should - be placed into the expected paths in root folder (or in - :file:`root.tar.gz` tarball). - -4. Activate the service in :file:`config.sh` - - The service needs to be enabled during image built time to be - run during the very first boot. In can be done by adding the following - snipped inside the :file:`config.sh`. diff --git a/kiwi/builder/disk.py b/kiwi/builder/disk.py index c3635f44..b48a6249 100644 --- a/kiwi/builder/disk.py +++ b/kiwi/builder/disk.py @@ -25,6 +25,7 @@ from typing import ( import kiwi.defaults as defaults from kiwi.utils.temporary import Temporary +from kiwi.storage.disk import ptable_entry_type from kiwi.defaults import Defaults from kiwi.filesystem.base import FileSystemBase from kiwi.bootloader.config import BootLoaderConfig @@ -98,6 +99,7 @@ class DiskBuilder: self.blocksize = xml_state.build_type.get_target_blocksize() self.volume_manager_name = xml_state.get_volume_management() self.volumes = xml_state.get_volumes() + self.custom_partitions = xml_state.get_partitions() self.volume_group_name = xml_state.get_volume_group_name() self.mdraid = xml_state.build_type.get_mdraid() self.hybrid_mbr = xml_state.build_type.get_gpt_hybrid_mbr() @@ -208,6 +210,10 @@ class DiskBuilder: # representing the spare_part_mountpoint area of the disk system_spare: Optional[FileSystemBase] = None + # a list of instances with the sync_data capability + # representing the custom partitions area of the disk + system_custom_parts: List[FileSystemBase] = [] + if self.install_media and self.build_type_name != 'oem': raise KiwiInstallMediaError( 'Install media requires oem type setup, got {0}'.format( @@ -316,6 +322,10 @@ class DiskBuilder: # create spare filesystem on spare partition if present system_spare = self._build_spare_filesystem(device_map) + system_custom_parts = self._build_custom_parts_filesystem( + device_map, self.custom_partitions + ) + # create filesystems on boot partition(s) if any system_boot, system_efi = self._build_boot_filesystems(device_map) @@ -439,7 +449,8 @@ class DiskBuilder: # syncing system data to disk image self._sync_system_to_image( - device_map, system, system_boot, system_efi, system_spare + device_map, system, system_boot, system_efi, system_spare, + system_custom_parts ) # run post sync script hook @@ -630,12 +641,40 @@ class DiskBuilder: if 'efi' in device_map: exclude_list.append('boot/efi/*') exclude_list.append('boot/efi/.*') + if self.custom_partitions: + for map_name in sorted(self.custom_partitions.keys()): + if map_name in device_map: + mountpoint = os.path.normpath( + self.custom_partitions[map_name].mountpoint + ).lstrip(os.sep) + exclude_list.append(f'{mountpoint}/*') + exclude_list.append(f'{mountpoint}/.*') return exclude_list @staticmethod def _get_exclude_list_for_boot_data_sync() -> list: return ['efi/*'] + def _build_custom_parts_filesystem( + self, device_map: Dict, + custom_partitions: Dict['str', ptable_entry_type] + ) -> List[FileSystemBase]: + filesystem_list = [] + if custom_partitions: + for map_name in sorted(custom_partitions.keys()): + if map_name in device_map: + ptable_entry = custom_partitions[map_name] + filesystem = FileSystem.new( + ptable_entry.filesystem, + device_map[map_name], + f'{self.root_dir}{ptable_entry.mountpoint}/' + ) + filesystem.create_on_device( + label=map_name.upper() + ) + filesystem_list.append(filesystem) + return filesystem_list + def _build_spare_filesystem(self, device_map: Dict) -> Optional[FileSystemBase]: if 'spare' in device_map and self.spare_part_fs: spare_part_data_path = None @@ -697,7 +736,9 @@ class DiskBuilder: system_boot = filesystem return system_boot, system_efi - def _build_and_map_disk_partitions(self, disk: Disk, disksize_mbytes: float) -> Dict: + def _build_and_map_disk_partitions( + self, disk: Disk, disksize_mbytes: float + ) -> Dict: disk.wipe() disksize_used_mbytes = 0 if self.firmware.legacy_bios_mode(): @@ -740,6 +781,14 @@ class DiskBuilder: ) disksize_used_mbytes += self.swap_mbytes + if self.custom_partitions: + log.info( + '--> creating custom partition(s): {0}'.format( + sorted(self.custom_partitions.keys()) + ) + ) + disk.create_custom_partitions(self.custom_partitions) + if self.spare_part_mbsize and not self.spare_part_is_last: log.info('--> creating spare partition') disk.create_spare_partition( @@ -914,6 +963,13 @@ class DiskBuilder: self._add_fstab_entry( device_map['swap'].get_device(), 'swap' ) + if self.custom_partitions: + for map_name in sorted(self.custom_partitions.keys()): + if device_map.get(map_name): + self._add_fstab_entry( + device_map[map_name].get_device(), + self.custom_partitions[map_name].mountpoint + ) setup.create_fstab( self.fstab ) @@ -1040,12 +1096,18 @@ class DiskBuilder: self, device_map: Dict, system: Any, system_boot: Optional[FileSystemBase], system_efi: Optional[FileSystemBase], - system_spare: Optional[FileSystemBase] + system_spare: Optional[FileSystemBase], + system_custom_parts: List[FileSystemBase] ) -> None: log.info('Syncing system to image') if system_spare: + log.info('--> Syncing spare partition data') system_spare.sync_data() + for system_custom_part in system_custom_parts: + log.info('--> Syncing custom partition(s) data') + system_custom_part.sync_data() + if system_efi: log.info('--> Syncing EFI boot data to EFI partition') system_efi.sync_data() diff --git a/kiwi/exceptions.py b/kiwi/exceptions.py index 5ccb2069..e0220d76 100644 --- a/kiwi/exceptions.py +++ b/kiwi/exceptions.py @@ -814,3 +814,10 @@ class KiwiUmountBusyError(KiwiError): """ Exception raised if the attempt to umount a resource has failed """ + + +class KiwiCustomPartitionConflictError(KiwiError): + """ + Exception raised if the entry in a custom partition setup + conflicts with an existing partition table layout setting + """ diff --git a/kiwi/schema/kiwi.rnc b/kiwi/schema/kiwi.rnc index edf3fb5d..05f71890 100644 --- a/kiwi/schema/kiwi.rnc +++ b/kiwi/schema/kiwi.rnc @@ -847,38 +847,6 @@ div { } } -#========================================== -# common element -# -div { - k.partition.type.attribute = - ## Partition Type identifier, see parted for details - attribute type { text } - k.partition.number.attribute = - ## Partition ID - attribute number { text } - k.partition.size.attribute = k.size.attribute - k.partition.mountpoint.attribute = - ## Mount path for this partition - attribute mountpoint { text } - k.partition.target.attribute = - ## Is a real target or not which means is part of - ## the /etc/fstab file or not - attribute target { xsd:boolean } - k.partition.attlist = - k.partition.type.attribute & - k.partition.number.attribute & - k.partition.size.attribute? & - k.partition.mountpoint.attribute? & - k.partition.target.attribute? - k.partition = - ## A Partition - element partition { - k.partition.attlist, - empty - } -} - #========================================== # common element # @@ -1914,6 +1882,7 @@ div { k.oemconfig? & k.size? & k.systemdisk? & + k.partitions? & k.vagrantconfig* & k.installmedia? } @@ -2087,6 +2056,69 @@ div { } } +#========================================== +# common element +# +div { + sch:pattern [ + abstract = "true" + id = "partition_name_type" + sch:rule [ + context = "partition[@name]" + sch:assert [ + test = "not(contains('$reserved', @name))" + "partition(name) is reserved " + "Reserved names are '$reserved'" + ] + ] + ] + k.partition.name.attribute = + ## Partition map name. The name of the partition as handled + ## by KIWI. Note, that there are the following reserved + ## names which cannot be used because they are already + ## represented by existing KIWI attributes: root, readonly, + ## boot, prep, spare, swap, efi_csm and efi. The filesystem + ## created on the partition will also use this name in + ## uppercase as its label + attribute name { text } + >> sch:pattern [ id = "partition_name" is-a = "partition_name_type" + sch:param [ name = "reserved" value = "root readonly boot prep spare swap efi_csm efi" ] + ] + k.partition.size.attribute = + ## Absolute size of the partition. + ## The value is used as MB by default but you can + ## add "M" and/or "G" as postfix + attribute size { partition-size-type } + k.partition.partition_type.attribute = + ## Partition type name in the context of kiwi + ## Allowed values are: t.linux + attribute partition_type { "t.linux" | "t.raid" } + k.partition.partition_name.attribute = + ## Partition name as it appears in the table + attribute partition_name { safe-posix-short-name } + k.partition.mountpoint.attribute = + ## Mountpoint below which this partition should be mounted to + attribute mountpoint { text } + k.partition.filesystem.attribute = + ## Filesystem which should be created on the partition + attribute filesystem { + "btrfs" | "ext2" | "ext3" | "ext4" | "squashfs" | "xfs" + } + k.partition.attlist = + k.partition.name.attribute & + k.partition.size.attribute & + k.partition.partition_name.attribute? & + k.partition.partition_type.attribute? & + k.partition.mountpoint.attribute & + k.partition.filesystem.attribute + k.partition = + ## Specify custom partition in the partition table + element partition { + k.partition.attlist, + empty + } +} + #========================================== # common element # @@ -2535,6 +2567,20 @@ div { } } +#========================================== +# main block: +# +div { + k.partitions.attlist = empty + k.partitions = + ## Partition table entries within the custom area + ## of the storage device + element partitions { + k.partitions.attlist & + k.partition+ + } +} + #========================================== # main block: # diff --git a/kiwi/schema/kiwi.rng b/kiwi/schema/kiwi.rng index eeefe345..fc57c426 100644 --- a/kiwi/schema/kiwi.rng +++ b/kiwi/schema/kiwi.rng @@ -1300,60 +1300,6 @@ the device is looked up in /dev/disk/by-* and /dev/mapper/* - -
- - - Partition Type identifier, see parted for details - - - - - Partition ID - - - - - - - - Mount path for this partition - - - - - Is a real target or not which means is part of -the /etc/fstab file or not - - - - - - - - - - - - - - - - - - - - - A Partition - - - - -
+
+ + + partition(name) is reserved Reserved names are '$reserved' + + + + + Partition map name. The name of the partition as handled +by KIWI. Note, that there are the following reserved +names which cannot be used because they are already +represented by existing KIWI attributes: root, readonly, +boot, prep, spare, swap, efi_csm and efi. The filesystem +created on the partition will also use this name in +uppercase as its label + + + + + + + + Absolute size of the partition. +The value is used as MB by default but you can +add "M" and/or "G" as postfix + + + + + + Partition type name in the context of kiwi +Allowed values are: t.linux + + t.linux + t.raid + + + + + + Partition name as it appears in the table + + + + + + Mountpoint below which this partition should be mounted to + + + + + Filesystem which should be created on the partition + + btrfs + ext2 + ext3 + ext4 + squashfs + xfs + + + + + + + + + + + + + + + + + + + + Specify custom partition in the partition table + + + + +
+
+ + + + + + Partition table entries within the custom area +of the storage device + + + + + + + + +