Add buildah tool support for OCI and Docker types

This commit adds buildah tool support for OCI and Docker types. It
requires buildah and skopeo to be installed in the host. The use of
umoci (kept as default choice) or buildah is configured using the
runtime configuration file; consider the following structure:

```
oci:
    - archive_tool: buildah
```

kiwi/container/oci.py

@@ -131,7 +131,7 @@ class ContainerImageOCI(object):
         )
 
         oci.unpack()
-        oci.sync_rootfs(''.join([self.root_dir, os.sep]), exclude_list)
+        oci.sync_rootfs(self.root_dir, exclude_list)
         oci.repack(self.oci_config)
         oci.set_config(self.oci_config)
         oci.post_process()
@@ -142,7 +142,7 @@ class ContainerImageOCI(object):
                 self.oci_config['container_tag']
             )
             additional_refs = []
-            if self.oci_config.get('additional_tags'):
+            if 'additional_tags' in self.oci_config:
                 additional_refs = []
                 for tag in self.oci_config['additional_tags']:
                     additional_refs.append('{0}:{1}'.format(

kiwi/exceptions.py

@@ -780,6 +780,12 @@ class KiwiDecodingError(KiwiError):
     """
 
 
+class KiwiBuildahError(KiwiError):
+    """
+    Exception raised on inconsistent buildah class calls
+    """
+
+
 class KiwiFileAccessError(KiwiError):
     """
     Exception raised if accessing a file or its metadata failed

kiwi/oci_tools/__init__.py

@@ -17,6 +17,7 @@
 #
 # project
 from kiwi.oci_tools.umoci import OCIUmoci
+from kiwi.oci_tools.buildah import OCIBuildah
 from kiwi.runtime_config import RuntimeConfig
 
 from kiwi.exceptions import (
@@ -33,6 +34,8 @@ class OCI(object):
         tool_name = runtime_config.get_oci_archive_tool()
         if tool_name == 'umoci':
             return OCIUmoci()
+        elif tool_name == 'buildah':
+            return OCIBuildah()
         else:
             raise KiwiOCIArchiveToolError(
                 'No support for {0} tool available'.format(tool_name)

kiwi/oci_tools/buildah.py

@@ -0,0 +1,326 @@
+# Copyright (c) 2019 SUSE Linux GmbH.  All rights reserved.
+#
+# This file is part of kiwi.
+#
+# kiwi is free software: you can redistribute it and/or modify
+# it under the terms owf the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# kiwi is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with kiwi.  If not, see <http://www.gnu.org/licenses/>
+#
+import os
+import random
+import string
+
+# project
+from kiwi.oci_tools.base import OCIBase
+from kiwi.command import Command
+from kiwi.path import Path
+from kiwi.defaults import Defaults
+from kiwi.logger import log
+from kiwi.exceptions import KiwiBuildahError
+
+
+class OCIBuildah(OCIBase):
+    """
+    **Open Container Operations using buildah**
+    """
+    def post_init(self):
+        """
+        Initializes some default parameters
+        """
+        self.working_image = None
+        self.imported_image = None
+        self.working_container = None
+
+    def import_container_image(self, container_image_ref):
+        """
+        Imports container image reference to the OCI containers storage.
+
+        :param str container_image_ref: container image reference
+        """
+        if not self.imported_image:
+            self.imported_image = 'kiwi-image-{0}:{1}'.format(
+                self._random_string_generator(),
+                Defaults.get_container_base_image_tag()
+            )
+        else:
+            raise KiwiBuildahError(
+                "Image already imported, called: '{0}'".format(
+                    self.imported_image
+                )
+            )
+
+        # We are making use of skopeo instead of only calling 'buildah from'
+        # because we want to control the image name loaded into the containers
+        # storage. This way we are certain to not leave any left over after the
+        # build.
+        Command.run(
+            [
+                'skopeo', 'copy', container_image_ref,
+                'containers-storage:{0}'.format(self.imported_image)
+            ]
+        )
+
+        if not self.working_container:
+            self.working_container = 'kiwi-container-{0}'.format(
+                self._random_string_generator()
+            )
+        else:
+            raise KiwiBuildahError(
+                "Container already initated, called: '{0}'".format(
+                    self.working_container
+                )
+            )
+
+        Command.run(
+            [
+                'buildah', 'from', '--name', self.working_container,
+                'containers-storage:{0}'.format(self.imported_image)
+            ]
+        )
+
+    def export_container_image(
+        self, filename, transport, image_ref, additional_refs=None
+    ):
+        """
+        Exports the working container to a container image archive
+
+        :param str filename: The resulting filename
+        :param str transport: The archive format
+        :param str image_name: Name of the exported image
+        :param str image_tag: Tag of the exported image
+        :param list additional_tags: List of additional references
+        """
+        extra_tags_opt = []
+        if additional_refs:
+            for ref in additional_refs:
+                extra_tags_opt.extend(['--additional-tag', ref])
+
+        # make sure the target tar file does not exist
+        # skopeo doesn't support force overwrite
+        Path.wipe(filename)
+        if self.working_image:
+            export_image = self.working_image
+        elif self.imported_image:
+            export_image = self.imported_image
+        else:
+            raise KiwiBuildahError("There is no image to export defined")
+
+        # we are using 'skopeo copy' to export images instead of 'buildah push'
+        # because buildah does not support multiple tags
+        Command.run([
+            'skopeo', 'copy', 'containers-storage:{0}'.format(export_image),
+            '{0}:{1}:{2}'.format(transport, filename, image_ref)
+        ] + extra_tags_opt)
+
+    def init_container(self):
+        """
+        Initialize a new container in OCI containers storage
+        """
+        if not self.working_container:
+            self.working_container = 'kiwi-container-{0}'.format(
+                self._random_string_generator()
+            )
+        else:
+            raise KiwiBuildahError(
+                "Image already imported or initated at '{0}' container".format(
+                    self.working_container
+                )
+            )
+
+        Command.run(
+            ['buildah', 'from', '--name', self.working_container, 'scratch']
+        )
+
+    def unpack(self):
+        """
+        Mounts current container root data to a directory
+        """
+        cmd = Command.run(
+            ['buildah', 'mount', self.working_container]
+        )
+        self.oci_root_dir = cmd.output.rstrip()
+
+    def sync_rootfs(self, root_dir, exclude_list=None):
+        """
+        Synchronizes the image root with the rootfs of the container
+
+        :param string root_dir: root directory of the prepare step
+        :param list exclude_list: list of paths to exclude
+        """
+        self._sync_data(
+            ''.join([root_dir, os.sep]), self.oci_root_dir,
+            exclude_list=exclude_list,
+            options=['-a', '-H', '-X', '-A', '--delete']
+        )
+
+    def import_rootfs(self, root_dir, exclude_list=None):
+        """
+        Synchronizes the container rootfs with the root tree of the build
+
+        :param string root_dir: root directory used in prepare step
+        :param list exclude_list: list of paths to exclude
+        """
+        self._sync_data(
+            os.sep.join([self.oci_root_dir, '']), root_dir,
+            exclude_list=exclude_list, options=['-a', '-H', '-X', '-A']
+        )
+
+    def repack(self, oci_config):
+        """
+        Pack root data directory into container image
+
+        :param list oci_config: unused parameter
+        """
+        Command.run(
+            ['buildah', 'umount', self.working_container]
+        )
+
+    def set_config(self, oci_config):
+        """
+        Set list of meta data information such as entry_point,
+        maintainer, etc... to the container.
+
+        :param list oci_config: meta data list
+        :param bool base_image: True|False
+        """
+        config_args = self._process_oci_config_to_arguments(oci_config)
+        Command.run(
+            ['buildah', 'config'] + config_args + [self.working_container]
+        )
+
+    def post_process(self):
+        """
+        Commits the OCI container into an OCI image
+        """
+        if not self.working_image and self.working_container:
+            self.working_image = 'kiwi-image-{0}:{1}'.format(
+                self._random_string_generator(), 'tag-{0}'.format(
+                    self._random_string_generator()
+                )
+            )
+        else:
+            raise KiwiBuildahError(
+                "No container to commit or container already committed"
+            )
+
+        output = Command.run(
+            [
+                'buildah', 'commit', '--rm', '--format', 'oci',
+                self.working_container, self.working_image
+            ]
+        )
+        self.working_image = output.output.rstrip()
+        self.working_container = None
+
+    @classmethod                                                # noqa:C901
+    def _process_oci_config_to_arguments(self, oci_config):
+        """
+        Process the oci configuration dictionary into a list of arguments
+        for the 'buildah config' command
+
+        :param list oci_config: meta data list
+
+        :return: List of buildah config arguments
+
+        :rtype: list
+        """
+        arguments = []
+        if 'maintainer' in oci_config:
+            arguments.append(
+                '--author={0}'.format(oci_config['maintainer'])
+            )
+
+        if 'user' in oci_config:
+            arguments.append(
+                '--user={0}'.format(oci_config['user'])
+            )
+
+        if 'workingdir' in oci_config:
+            arguments.append(
+                '--workingdir={0}'.format(oci_config['workingdir'])
+            )
+
+        if 'entry_command' in oci_config:
+            arguments.append('--entrypoint=[{0}]'.format(
+                ','.join(
+                    ['"{0}"'.format(x) for x in oci_config['entry_command']]
+                )
+            ))
+
+        if 'entry_subcommand' in oci_config:
+            arguments.append('--cmd={0}'.format(
+                ' '.join(oci_config['entry_subcommand'])
+            ))
+
+        if 'volumes' in oci_config:
+            for vol in oci_config['volumes']:
+                arguments.append('--volume={0}'.format(vol))
+
+        if 'expose_ports' in oci_config:
+            for port in oci_config['expose_ports']:
+                arguments.append('--port={0}'.format(port))
+
+        if 'environment' in oci_config:
+            for name in sorted(oci_config['environment']):
+                arguments.append('--env={0}={1}'.format(
+                    name, oci_config['environment'][name]
+                ))
+
+        if 'labels' in oci_config:
+            for name in sorted(oci_config['labels']):
+                arguments.append('--label={0}={1}'.format(
+                    name, oci_config['labels'][name]
+                ))
+
+        if 'history' in oci_config:
+            if 'comment' in oci_config['history']:
+                arguments.append('--history-comment={0}'.format(
+                    oci_config['history']['comment']
+                ))
+            if 'created_by' in oci_config['history']:
+                arguments.append('--created-by={0}'.format(
+                    oci_config['history']['created_by']
+                ))
+            if 'author' in oci_config['history']:
+                log.warning('Author field in history is ignored using buildah')
+
+        return arguments
+
+    @classmethod
+    def _random_string_generator(
+        cls, chars_num=6, allchars=string.ascii_lowercase + string.digits
+    ):
+        """
+        Creates a random string with the given length of characters choosen
+        randomly from a given list of possible characters.
+
+        Buildah makes use of the hosts configured containers storage of OCI
+        images. This method is used to avoid name collisions with any
+        previous image or container present in the host.
+
+        :param int chars_num: Lenght of the generated random string
+        :param list allchars: List of possible characters
+
+        :return: generated random string
+
+        :rtype: string
+        """
+        return "".join(random.choice(allchars) for x in range(chars_num))
+
+    def __del__(self):
+        if self.working_container:
+            Command.run(['buildah', 'umount', self.working_container])
+            Command.run(['buildah', 'rm', self.working_container])
+        if self.working_image:
+            Command.run(['buildah', 'rmi', self.working_image])
+        if self.imported_image:
+            Command.run(['buildah', 'rmi', self.imported_image])

kiwi/runtime_checker.py

@@ -26,6 +26,7 @@ from .system.uri import Uri
 from .defaults import Defaults
 from .path import Path
 from .utils.command_capabilities import CommandCapabilities
+from .runtime_config import RuntimeConfig
 from .exceptions import (
     KiwiRuntimeError
 )
@@ -211,8 +212,14 @@ class RuntimeChecker(object):
 
         expected_version = (0, 1, 0)
 
-        if self.xml_state.get_build_type_name() == 'docker':
-            for tool in ['umoci', 'skopeo']:
+        if self.xml_state.get_build_type_name() in ['docker', 'oci']:
+            runtime_config = RuntimeConfig()
+            tool_name = runtime_config.get_oci_archive_tool()
+            if tool_name == 'buildah':
+                oci_tools = ['buildah', 'skopeo']
+            else:
+                oci_tools = ['umoci', 'skopeo']
+            for tool in oci_tools:
                 if not Path.which(filename=tool, access_mode=os.X_OK):
                     raise KiwiRuntimeError(
                         message_tool_not_found.format(name=tool)

test/unit/container_image_oci_test.py

@@ -104,7 +104,7 @@ class TestContainerImageOCI(object):
         mock_oci.init_container.assert_called_once_with()
         mock_oci.unpack.assert_called_once_with()
         mock_oci.sync_rootfs.assert_called_once_with(
-            'root_dir/', [
+            'root_dir', [
                 'image', '.profile', '.kconfig', '.buildenv',
                 'var/cache/kiwi', 'boot', 'dev', 'sys', 'proc'
             ]
@@ -145,7 +145,7 @@ class TestContainerImageOCI(object):
         )
         mock_oci.unpack.assert_called_once_with()
         mock_oci.sync_rootfs.assert_called_once_with(
-            'root_dir/', [
+            'root_dir', [
                 'image', '.profile', '.kconfig', '.buildenv',
                 'var/cache/kiwi', 'boot', 'dev', 'sys', 'proc'
             ]

test/unit/oci_tools_buildah_test.py

@@ -0,0 +1,213 @@
+from mock import (
+    Mock, patch, call
+)
+from pytest import raises
+from collections import namedtuple
+
+from kiwi.oci_tools.buildah import OCIBuildah
+from kiwi.exceptions import KiwiBuildahError
+
+
+class TestOCIBuildah(object):
+    @patch('kiwi.oci_tools.base.datetime')
+    def setup(self, mock_datetime):
+        strftime = Mock()
+        strftime.strftime = Mock(return_value='current_date')
+        mock_datetime.utcnow = Mock(
+            return_value=strftime
+        )
+        self.oci = OCIBuildah()
+
+    @patch('kiwi.oci_tools.umoci.Command.run')
+    def teardown(self, mock_cmd_run):
+        del self.oci
+        mock_cmd_run.reset_mock()
+
+    @patch('kiwi.oci_tools.buildah.random.choice')
+    @patch('kiwi.oci_tools.buildah.Command.run')
+    def test_init_container(self, mock_Command_run, mock_choice):
+        mock_choice.return_value = 'x'
+        self.oci.init_container()
+        mock_Command_run.assert_called_once_with(
+            ['buildah', 'from', '--name', 'kiwi-container-xxxxxx', 'scratch']
+        )
+
+    def test_init_container_fail(self):
+        with raises(KiwiBuildahError):
+            self.oci.working_container = 'initialized'
+            self.oci.init_container()
+
+    @patch('kiwi.oci_tools.umoci.Command.run')
+    def test_unpack(self, mock_cmd_run):
+        command_type = namedtuple('command', ['output'])
+        mock_cmd_run.return_value = command_type(output='mountpoint-dir')
+        self.oci.working_container = 'kiwi-working'
+        self.oci.unpack()
+        mock_cmd_run.assert_called_once_with(
+            ['buildah', 'mount', 'kiwi-working']
+        )
+        assert self.oci.oci_root_dir == 'mountpoint-dir'
+
+    @patch('kiwi.oci_tools.base.DataSync')
+    def test_sync_rootfs(self, mock_sync):
+        sync = Mock()
+        mock_sync.return_value = sync
+        self.oci.oci_root_dir = 'oci_root'
+        self.oci.sync_rootfs('root_dir', exclude_list=['/dev', '/proc'])
+        mock_sync.assert_called_once_with(
+            'root_dir/', 'oci_root'
+        )
+        sync.sync_data.assert_called_once_with(
+            exclude=['/dev', '/proc'],
+            options=['-a', '-H', '-X', '-A', '--delete']
+        )
+
+    @patch('kiwi.oci_tools.base.DataSync')
+    def test_import_rootfs(self, mock_sync):
+        sync = Mock()
+        mock_sync.return_value = sync
+        self.oci.oci_root_dir = 'oci_root'
+        self.oci.import_rootfs('root_dir', exclude_list=['/dev', '/proc'])
+        mock_sync.assert_called_once_with(
+            'oci_root/', 'root_dir'
+        )
+        sync.sync_data.assert_called_once_with(
+            exclude=['/dev', '/proc'],
+            options=['-a', '-H', '-X', '-A']
+        )
+
+    @patch('kiwi.oci_tools.umoci.Command.run')
+    def test_repack(self, mock_cmd_run):
+        self.oci.working_container = 'kiwi-working'
+        self.oci.repack({})
+        mock_cmd_run.assert_called_once_with(
+            ['buildah', 'umount', 'kiwi-working']
+        )
+
+    @patch('kiwi.logger.log.warning')
+    @patch('kiwi.oci_tools.umoci.Command.run')
+    def test_set_config(self, mock_cmd_run, mock_warn):
+        oci_config = {
+            'entry_command': ['/bin/bash', '-x'],
+            'entry_subcommand': ['ls', '-l'],
+            'maintainer': 'tux',
+            'user': 'root',
+            'workingdir': '/root',
+            'expose_ports': ['80', '42'],
+            'volumes': ['/var/log', '/tmp'],
+            'environment': {'FOO': 'bar', 'PATH': '/bin'},
+            'labels': {'a': 'value', 'b': 'value'},
+            'history': {
+                'author': 'history author',
+                'comment': 'This is a comment',
+                'created_by': 'created by text'
+            }
+        }
+        self.oci.working_container = 'kiwi-working'
+        self.oci.set_config(oci_config)
+        mock_cmd_run.assert_called_once_with([
+            'buildah', 'config', '--author=tux', '--user=root',
+            '--workingdir=/root', '--entrypoint=["/bin/bash","-x"]',
+            '--cmd=ls -l',
+            '--volume=/var/log', '--volume=/tmp', '--port=80', '--port=42',
+            '--env=FOO=bar', '--env=PATH=/bin', '--label=a=value',
+            '--label=b=value', '--history-comment=This is a comment',
+            '--created-by=created by text', 'kiwi-working'
+        ])
+        assert mock_warn.called
+
+    @patch('kiwi.oci_tools.buildah.random.choice')
+    @patch('kiwi.oci_tools.umoci.Command.run')
+    def test_post_process(self, mock_cmd_run, mock_choice):
+        mock_choice.return_value = 'x'
+        self.oci.working_container = 'kiwi-container'
+        self.oci.post_process()
+        mock_cmd_run.assert_called_once_with([
+            'buildah', 'commit', '--rm', '--format', 'oci',
+            'kiwi-container', 'kiwi-image-xxxxxx:tag-xxxxxx'
+        ])
+
+    def test_post_processi_fail(self):
+        with raises(KiwiBuildahError):
+            self.oci.working_container = None
+            self.oci.post_process()
+
+    @patch('kiwi.oci_tools.buildah.random.choice')
+    @patch('kiwi.oci_tools.buildah.Command.run')
+    def test_import_container_image(self, mock_Command_run, mock_choice):
+        mock_choice.return_value = 'x'
+        self.oci.import_container_image('oci-archive:image.tar')
+        assert mock_Command_run.call_args_list == [
+            call([
+                'skopeo', 'copy', 'oci-archive:image.tar',
+                'containers-storage:kiwi-image-xxxxxx:base_layer'
+            ]),
+            call([
+                'buildah', 'from', '--name', 'kiwi-container-xxxxxx',
+                'containers-storage:kiwi-image-xxxxxx:base_layer'
+            ])
+        ]
+
+    @patch('kiwi.oci_tools.buildah.random.choice')
+    @patch('kiwi.oci_tools.buildah.Command.run')
+    def test_import_container_image_fail_create_container(
+        self, mock_Command_run, mock_choice
+    ):
+        with raises(KiwiBuildahError):
+            mock_choice.return_value = 'x'
+            self.oci.working_container = 'initialized'
+            self.oci.import_container_image('oci-archive:image.tar')
+            mock_Command_run.assert_called_once_with(
+                [
+                    'skopeo', 'copy', 'oci-archive:image.tar',
+                    'containers-storage:kiwi-image-xxxxxx:base_layer'
+                ]
+            )
+
+    def test_import_container_image_fail_load_image(self):
+        with raises(KiwiBuildahError):
+            self.oci.imported_image = 'initialized'
+            self.oci.import_container_image('oci-archive:image.tar')
+
+    @patch('kiwi.oci_tools.buildah.Path.wipe')
+    @patch('kiwi.oci_tools.buildah.Command.run')
+    def test_export_container_image(self, mock_Command_run, mock_wipe):
+        self.oci.working_image = 'kiwi-image:tag'
+        self.oci.export_container_image(
+            'image.tar', 'docker-archive', 'myimage:tag',
+            ['myimage:tag2', 'myimage:tag3']
+        )
+        mock_Command_run.assert_called_once_with([
+            'skopeo', 'copy', 'containers-storage:kiwi-image:tag',
+            'docker-archive:image.tar:myimage:tag', '--additional-tag',
+            'myimage:tag2', '--additional-tag', 'myimage:tag3'
+        ])
+        mock_wipe.assert_called_once_with('image.tar')
+
+    @patch('kiwi.oci_tools.buildah.Path.wipe')
+    @patch('kiwi.oci_tools.buildah.Command.run')
+    def test_export_container_image_imported(self, mock_Command_run, mock_wipe):
+        self.oci.working_image = None
+        self.oci.imported_image = 'kiwi-image:base_layer'
+        self.oci.export_container_image(
+            'image.tar', 'docker-archive', 'myimage:tag',
+            ['myimage:tag2', 'myimage:tag3']
+        )
+        mock_Command_run.assert_called_once_with([
+            'skopeo', 'copy', 'containers-storage:kiwi-image:base_layer',
+            'docker-archive:image.tar:myimage:tag', '--additional-tag',
+            'myimage:tag2', '--additional-tag', 'myimage:tag3'
+        ])
+        mock_wipe.assert_called_once_with('image.tar')
+
+    @patch('kiwi.oci_tools.buildah.Path.wipe')
+    @patch('kiwi.oci_tools.buildah.Command.run')
+    def test_export_container_image_fail(self, mock_Command_run, mock_wipe):
+        with raises(KiwiBuildahError):
+            self.oci.working_image = None
+            self.oci.imported_image = None
+            self.oci.export_container_image(
+                'image.tar', 'docker-archive', 'myimage:tag',
+                ['myimage:tag2', 'myimage:tag3']
+            )
+            mock_wipe.assert_called_once_with('image.tar')

test/unit/oci_tools_test.py

@@ -25,6 +25,16 @@ class TestOCI(object):
         OCI()
         mock_OCIUmoci.assert_called_once_with()
 
+    @patch('kiwi.oci_tools.OCIBuildah')
+    @patch('kiwi.oci_tools.RuntimeConfig')
+    def test_oci_tool_buildah(
+        self, mock_RuntimeConfig, mock_OCIBuildah
+    ):
+        self.runtime_config.get_oci_archive_tool.return_value = 'buildah'
+        mock_RuntimeConfig.return_value = self.runtime_config
+        OCI()
+        mock_OCIBuildah.assert_called_once_with()
+
     @patch('kiwi.oci_tools.RuntimeConfig')
     def test_oci_tool_not_supported(self, mock_RuntimeConfig):
         self.runtime_config.get_oci_archive_tool.return_value = 'foo'

test/unit/runtime_checker_test.py

@@ -82,6 +82,20 @@ class TestRuntimeChecker(object):
         runtime_checker = RuntimeChecker(xml_state)
         runtime_checker.check_docker_tool_chain_installed()
 
+    @patch('kiwi.runtime_checker.RuntimeConfig.get_oci_archive_tool')
+    @patch('kiwi.runtime_checker.Path.which')
+    @raises(KiwiRuntimeError)
+    def test_check_docker_tool_chain_installed_buildah(
+        self, mock_which, mock_oci_tool
+    ):
+        mock_oci_tool.return_value = 'buildah'
+        mock_which.return_value = False
+        xml_state = XMLState(
+            self.description.load(), ['docker'], 'docker'
+        )
+        runtime_checker = RuntimeChecker(xml_state)
+        runtime_checker.check_docker_tool_chain_installed()
+
     @patch('kiwi.runtime_checker.Path.which')
     @patch('kiwi.runtime_checker.CommandCapabilities.check_version')
     @raises(KiwiRuntimeError)