diff --git a/kiwi/container/oci.py b/kiwi/container/oci.py index 7e4ffe07..27ec04e4 100644 --- a/kiwi/container/oci.py +++ b/kiwi/container/oci.py @@ -131,7 +131,7 @@ class ContainerImageOCI(object): ) oci.unpack() - oci.sync_rootfs(''.join([self.root_dir, os.sep]), exclude_list) + oci.sync_rootfs(self.root_dir, exclude_list) oci.repack(self.oci_config) oci.set_config(self.oci_config) oci.post_process() @@ -142,7 +142,7 @@ class ContainerImageOCI(object): self.oci_config['container_tag'] ) additional_refs = [] - if self.oci_config.get('additional_tags'): + if 'additional_tags' in self.oci_config: additional_refs = [] for tag in self.oci_config['additional_tags']: additional_refs.append('{0}:{1}'.format( diff --git a/kiwi/exceptions.py b/kiwi/exceptions.py index b873716c..5014d4f8 100644 --- a/kiwi/exceptions.py +++ b/kiwi/exceptions.py @@ -780,6 +780,12 @@ class KiwiDecodingError(KiwiError): """ +class KiwiBuildahError(KiwiError): + """ + Exception raised on inconsistent buildah class calls + """ + + class KiwiFileAccessError(KiwiError): """ Exception raised if accessing a file or its metadata failed diff --git a/kiwi/oci_tools/__init__.py b/kiwi/oci_tools/__init__.py index 732d985f..90e6da24 100644 --- a/kiwi/oci_tools/__init__.py +++ b/kiwi/oci_tools/__init__.py @@ -17,6 +17,7 @@ # # project from kiwi.oci_tools.umoci import OCIUmoci +from kiwi.oci_tools.buildah import OCIBuildah from kiwi.runtime_config import RuntimeConfig from kiwi.exceptions import ( @@ -33,6 +34,8 @@ class OCI(object): tool_name = runtime_config.get_oci_archive_tool() if tool_name == 'umoci': return OCIUmoci() + elif tool_name == 'buildah': + return OCIBuildah() else: raise KiwiOCIArchiveToolError( 'No support for {0} tool available'.format(tool_name) diff --git a/kiwi/oci_tools/buildah.py b/kiwi/oci_tools/buildah.py new file mode 100644 index 00000000..b6f7aa91 --- /dev/null +++ b/kiwi/oci_tools/buildah.py @@ -0,0 +1,326 @@ +# Copyright (c) 2019 SUSE Linux GmbH. All rights reserved. +# +# This file is part of kiwi. +# +# kiwi is free software: you can redistribute it and/or modify +# it under the terms owf the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# kiwi is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with kiwi. If not, see +# +import os +import random +import string + +# project +from kiwi.oci_tools.base import OCIBase +from kiwi.command import Command +from kiwi.path import Path +from kiwi.defaults import Defaults +from kiwi.logger import log +from kiwi.exceptions import KiwiBuildahError + + +class OCIBuildah(OCIBase): + """ + **Open Container Operations using buildah** + """ + def post_init(self): + """ + Initializes some default parameters + """ + self.working_image = None + self.imported_image = None + self.working_container = None + + def import_container_image(self, container_image_ref): + """ + Imports container image reference to the OCI containers storage. + + :param str container_image_ref: container image reference + """ + if not self.imported_image: + self.imported_image = 'kiwi-image-{0}:{1}'.format( + self._random_string_generator(), + Defaults.get_container_base_image_tag() + ) + else: + raise KiwiBuildahError( + "Image already imported, called: '{0}'".format( + self.imported_image + ) + ) + + # We are making use of skopeo instead of only calling 'buildah from' + # because we want to control the image name loaded into the containers + # storage. This way we are certain to not leave any left over after the + # build. + Command.run( + [ + 'skopeo', 'copy', container_image_ref, + 'containers-storage:{0}'.format(self.imported_image) + ] + ) + + if not self.working_container: + self.working_container = 'kiwi-container-{0}'.format( + self._random_string_generator() + ) + else: + raise KiwiBuildahError( + "Container already initated, called: '{0}'".format( + self.working_container + ) + ) + + Command.run( + [ + 'buildah', 'from', '--name', self.working_container, + 'containers-storage:{0}'.format(self.imported_image) + ] + ) + + def export_container_image( + self, filename, transport, image_ref, additional_refs=None + ): + """ + Exports the working container to a container image archive + + :param str filename: The resulting filename + :param str transport: The archive format + :param str image_name: Name of the exported image + :param str image_tag: Tag of the exported image + :param list additional_tags: List of additional references + """ + extra_tags_opt = [] + if additional_refs: + for ref in additional_refs: + extra_tags_opt.extend(['--additional-tag', ref]) + + # make sure the target tar file does not exist + # skopeo doesn't support force overwrite + Path.wipe(filename) + if self.working_image: + export_image = self.working_image + elif self.imported_image: + export_image = self.imported_image + else: + raise KiwiBuildahError("There is no image to export defined") + + # we are using 'skopeo copy' to export images instead of 'buildah push' + # because buildah does not support multiple tags + Command.run([ + 'skopeo', 'copy', 'containers-storage:{0}'.format(export_image), + '{0}:{1}:{2}'.format(transport, filename, image_ref) + ] + extra_tags_opt) + + def init_container(self): + """ + Initialize a new container in OCI containers storage + """ + if not self.working_container: + self.working_container = 'kiwi-container-{0}'.format( + self._random_string_generator() + ) + else: + raise KiwiBuildahError( + "Image already imported or initated at '{0}' container".format( + self.working_container + ) + ) + + Command.run( + ['buildah', 'from', '--name', self.working_container, 'scratch'] + ) + + def unpack(self): + """ + Mounts current container root data to a directory + """ + cmd = Command.run( + ['buildah', 'mount', self.working_container] + ) + self.oci_root_dir = cmd.output.rstrip() + + def sync_rootfs(self, root_dir, exclude_list=None): + """ + Synchronizes the image root with the rootfs of the container + + :param string root_dir: root directory of the prepare step + :param list exclude_list: list of paths to exclude + """ + self._sync_data( + ''.join([root_dir, os.sep]), self.oci_root_dir, + exclude_list=exclude_list, + options=['-a', '-H', '-X', '-A', '--delete'] + ) + + def import_rootfs(self, root_dir, exclude_list=None): + """ + Synchronizes the container rootfs with the root tree of the build + + :param string root_dir: root directory used in prepare step + :param list exclude_list: list of paths to exclude + """ + self._sync_data( + os.sep.join([self.oci_root_dir, '']), root_dir, + exclude_list=exclude_list, options=['-a', '-H', '-X', '-A'] + ) + + def repack(self, oci_config): + """ + Pack root data directory into container image + + :param list oci_config: unused parameter + """ + Command.run( + ['buildah', 'umount', self.working_container] + ) + + def set_config(self, oci_config): + """ + Set list of meta data information such as entry_point, + maintainer, etc... to the container. + + :param list oci_config: meta data list + :param bool base_image: True|False + """ + config_args = self._process_oci_config_to_arguments(oci_config) + Command.run( + ['buildah', 'config'] + config_args + [self.working_container] + ) + + def post_process(self): + """ + Commits the OCI container into an OCI image + """ + if not self.working_image and self.working_container: + self.working_image = 'kiwi-image-{0}:{1}'.format( + self._random_string_generator(), 'tag-{0}'.format( + self._random_string_generator() + ) + ) + else: + raise KiwiBuildahError( + "No container to commit or container already committed" + ) + + output = Command.run( + [ + 'buildah', 'commit', '--rm', '--format', 'oci', + self.working_container, self.working_image + ] + ) + self.working_image = output.output.rstrip() + self.working_container = None + + @classmethod # noqa:C901 + def _process_oci_config_to_arguments(self, oci_config): + """ + Process the oci configuration dictionary into a list of arguments + for the 'buildah config' command + + :param list oci_config: meta data list + + :return: List of buildah config arguments + + :rtype: list + """ + arguments = [] + if 'maintainer' in oci_config: + arguments.append( + '--author={0}'.format(oci_config['maintainer']) + ) + + if 'user' in oci_config: + arguments.append( + '--user={0}'.format(oci_config['user']) + ) + + if 'workingdir' in oci_config: + arguments.append( + '--workingdir={0}'.format(oci_config['workingdir']) + ) + + if 'entry_command' in oci_config: + arguments.append('--entrypoint=[{0}]'.format( + ','.join( + ['"{0}"'.format(x) for x in oci_config['entry_command']] + ) + )) + + if 'entry_subcommand' in oci_config: + arguments.append('--cmd={0}'.format( + ' '.join(oci_config['entry_subcommand']) + )) + + if 'volumes' in oci_config: + for vol in oci_config['volumes']: + arguments.append('--volume={0}'.format(vol)) + + if 'expose_ports' in oci_config: + for port in oci_config['expose_ports']: + arguments.append('--port={0}'.format(port)) + + if 'environment' in oci_config: + for name in sorted(oci_config['environment']): + arguments.append('--env={0}={1}'.format( + name, oci_config['environment'][name] + )) + + if 'labels' in oci_config: + for name in sorted(oci_config['labels']): + arguments.append('--label={0}={1}'.format( + name, oci_config['labels'][name] + )) + + if 'history' in oci_config: + if 'comment' in oci_config['history']: + arguments.append('--history-comment={0}'.format( + oci_config['history']['comment'] + )) + if 'created_by' in oci_config['history']: + arguments.append('--created-by={0}'.format( + oci_config['history']['created_by'] + )) + if 'author' in oci_config['history']: + log.warning('Author field in history is ignored using buildah') + + return arguments + + @classmethod + def _random_string_generator( + cls, chars_num=6, allchars=string.ascii_lowercase + string.digits + ): + """ + Creates a random string with the given length of characters choosen + randomly from a given list of possible characters. + + Buildah makes use of the hosts configured containers storage of OCI + images. This method is used to avoid name collisions with any + previous image or container present in the host. + + :param int chars_num: Lenght of the generated random string + :param list allchars: List of possible characters + + :return: generated random string + + :rtype: string + """ + return "".join(random.choice(allchars) for x in range(chars_num)) + + def __del__(self): + if self.working_container: + Command.run(['buildah', 'umount', self.working_container]) + Command.run(['buildah', 'rm', self.working_container]) + if self.working_image: + Command.run(['buildah', 'rmi', self.working_image]) + if self.imported_image: + Command.run(['buildah', 'rmi', self.imported_image]) diff --git a/kiwi/runtime_checker.py b/kiwi/runtime_checker.py index 4e96dabb..52eb9014 100644 --- a/kiwi/runtime_checker.py +++ b/kiwi/runtime_checker.py @@ -26,6 +26,7 @@ from .system.uri import Uri from .defaults import Defaults from .path import Path from .utils.command_capabilities import CommandCapabilities +from .runtime_config import RuntimeConfig from .exceptions import ( KiwiRuntimeError ) @@ -211,8 +212,14 @@ class RuntimeChecker(object): expected_version = (0, 1, 0) - if self.xml_state.get_build_type_name() == 'docker': - for tool in ['umoci', 'skopeo']: + if self.xml_state.get_build_type_name() in ['docker', 'oci']: + runtime_config = RuntimeConfig() + tool_name = runtime_config.get_oci_archive_tool() + if tool_name == 'buildah': + oci_tools = ['buildah', 'skopeo'] + else: + oci_tools = ['umoci', 'skopeo'] + for tool in oci_tools: if not Path.which(filename=tool, access_mode=os.X_OK): raise KiwiRuntimeError( message_tool_not_found.format(name=tool) diff --git a/test/unit/container_image_oci_test.py b/test/unit/container_image_oci_test.py index 88a9df02..c7a199b8 100644 --- a/test/unit/container_image_oci_test.py +++ b/test/unit/container_image_oci_test.py @@ -104,7 +104,7 @@ class TestContainerImageOCI(object): mock_oci.init_container.assert_called_once_with() mock_oci.unpack.assert_called_once_with() mock_oci.sync_rootfs.assert_called_once_with( - 'root_dir/', [ + 'root_dir', [ 'image', '.profile', '.kconfig', '.buildenv', 'var/cache/kiwi', 'boot', 'dev', 'sys', 'proc' ] @@ -145,7 +145,7 @@ class TestContainerImageOCI(object): ) mock_oci.unpack.assert_called_once_with() mock_oci.sync_rootfs.assert_called_once_with( - 'root_dir/', [ + 'root_dir', [ 'image', '.profile', '.kconfig', '.buildenv', 'var/cache/kiwi', 'boot', 'dev', 'sys', 'proc' ] diff --git a/test/unit/oci_tools_buildah_test.py b/test/unit/oci_tools_buildah_test.py new file mode 100644 index 00000000..29b79e3c --- /dev/null +++ b/test/unit/oci_tools_buildah_test.py @@ -0,0 +1,213 @@ +from mock import ( + Mock, patch, call +) +from pytest import raises +from collections import namedtuple + +from kiwi.oci_tools.buildah import OCIBuildah +from kiwi.exceptions import KiwiBuildahError + + +class TestOCIBuildah(object): + @patch('kiwi.oci_tools.base.datetime') + def setup(self, mock_datetime): + strftime = Mock() + strftime.strftime = Mock(return_value='current_date') + mock_datetime.utcnow = Mock( + return_value=strftime + ) + self.oci = OCIBuildah() + + @patch('kiwi.oci_tools.umoci.Command.run') + def teardown(self, mock_cmd_run): + del self.oci + mock_cmd_run.reset_mock() + + @patch('kiwi.oci_tools.buildah.random.choice') + @patch('kiwi.oci_tools.buildah.Command.run') + def test_init_container(self, mock_Command_run, mock_choice): + mock_choice.return_value = 'x' + self.oci.init_container() + mock_Command_run.assert_called_once_with( + ['buildah', 'from', '--name', 'kiwi-container-xxxxxx', 'scratch'] + ) + + def test_init_container_fail(self): + with raises(KiwiBuildahError): + self.oci.working_container = 'initialized' + self.oci.init_container() + + @patch('kiwi.oci_tools.umoci.Command.run') + def test_unpack(self, mock_cmd_run): + command_type = namedtuple('command', ['output']) + mock_cmd_run.return_value = command_type(output='mountpoint-dir') + self.oci.working_container = 'kiwi-working' + self.oci.unpack() + mock_cmd_run.assert_called_once_with( + ['buildah', 'mount', 'kiwi-working'] + ) + assert self.oci.oci_root_dir == 'mountpoint-dir' + + @patch('kiwi.oci_tools.base.DataSync') + def test_sync_rootfs(self, mock_sync): + sync = Mock() + mock_sync.return_value = sync + self.oci.oci_root_dir = 'oci_root' + self.oci.sync_rootfs('root_dir', exclude_list=['/dev', '/proc']) + mock_sync.assert_called_once_with( + 'root_dir/', 'oci_root' + ) + sync.sync_data.assert_called_once_with( + exclude=['/dev', '/proc'], + options=['-a', '-H', '-X', '-A', '--delete'] + ) + + @patch('kiwi.oci_tools.base.DataSync') + def test_import_rootfs(self, mock_sync): + sync = Mock() + mock_sync.return_value = sync + self.oci.oci_root_dir = 'oci_root' + self.oci.import_rootfs('root_dir', exclude_list=['/dev', '/proc']) + mock_sync.assert_called_once_with( + 'oci_root/', 'root_dir' + ) + sync.sync_data.assert_called_once_with( + exclude=['/dev', '/proc'], + options=['-a', '-H', '-X', '-A'] + ) + + @patch('kiwi.oci_tools.umoci.Command.run') + def test_repack(self, mock_cmd_run): + self.oci.working_container = 'kiwi-working' + self.oci.repack({}) + mock_cmd_run.assert_called_once_with( + ['buildah', 'umount', 'kiwi-working'] + ) + + @patch('kiwi.logger.log.warning') + @patch('kiwi.oci_tools.umoci.Command.run') + def test_set_config(self, mock_cmd_run, mock_warn): + oci_config = { + 'entry_command': ['/bin/bash', '-x'], + 'entry_subcommand': ['ls', '-l'], + 'maintainer': 'tux', + 'user': 'root', + 'workingdir': '/root', + 'expose_ports': ['80', '42'], + 'volumes': ['/var/log', '/tmp'], + 'environment': {'FOO': 'bar', 'PATH': '/bin'}, + 'labels': {'a': 'value', 'b': 'value'}, + 'history': { + 'author': 'history author', + 'comment': 'This is a comment', + 'created_by': 'created by text' + } + } + self.oci.working_container = 'kiwi-working' + self.oci.set_config(oci_config) + mock_cmd_run.assert_called_once_with([ + 'buildah', 'config', '--author=tux', '--user=root', + '--workingdir=/root', '--entrypoint=["/bin/bash","-x"]', + '--cmd=ls -l', + '--volume=/var/log', '--volume=/tmp', '--port=80', '--port=42', + '--env=FOO=bar', '--env=PATH=/bin', '--label=a=value', + '--label=b=value', '--history-comment=This is a comment', + '--created-by=created by text', 'kiwi-working' + ]) + assert mock_warn.called + + @patch('kiwi.oci_tools.buildah.random.choice') + @patch('kiwi.oci_tools.umoci.Command.run') + def test_post_process(self, mock_cmd_run, mock_choice): + mock_choice.return_value = 'x' + self.oci.working_container = 'kiwi-container' + self.oci.post_process() + mock_cmd_run.assert_called_once_with([ + 'buildah', 'commit', '--rm', '--format', 'oci', + 'kiwi-container', 'kiwi-image-xxxxxx:tag-xxxxxx' + ]) + + def test_post_processi_fail(self): + with raises(KiwiBuildahError): + self.oci.working_container = None + self.oci.post_process() + + @patch('kiwi.oci_tools.buildah.random.choice') + @patch('kiwi.oci_tools.buildah.Command.run') + def test_import_container_image(self, mock_Command_run, mock_choice): + mock_choice.return_value = 'x' + self.oci.import_container_image('oci-archive:image.tar') + assert mock_Command_run.call_args_list == [ + call([ + 'skopeo', 'copy', 'oci-archive:image.tar', + 'containers-storage:kiwi-image-xxxxxx:base_layer' + ]), + call([ + 'buildah', 'from', '--name', 'kiwi-container-xxxxxx', + 'containers-storage:kiwi-image-xxxxxx:base_layer' + ]) + ] + + @patch('kiwi.oci_tools.buildah.random.choice') + @patch('kiwi.oci_tools.buildah.Command.run') + def test_import_container_image_fail_create_container( + self, mock_Command_run, mock_choice + ): + with raises(KiwiBuildahError): + mock_choice.return_value = 'x' + self.oci.working_container = 'initialized' + self.oci.import_container_image('oci-archive:image.tar') + mock_Command_run.assert_called_once_with( + [ + 'skopeo', 'copy', 'oci-archive:image.tar', + 'containers-storage:kiwi-image-xxxxxx:base_layer' + ] + ) + + def test_import_container_image_fail_load_image(self): + with raises(KiwiBuildahError): + self.oci.imported_image = 'initialized' + self.oci.import_container_image('oci-archive:image.tar') + + @patch('kiwi.oci_tools.buildah.Path.wipe') + @patch('kiwi.oci_tools.buildah.Command.run') + def test_export_container_image(self, mock_Command_run, mock_wipe): + self.oci.working_image = 'kiwi-image:tag' + self.oci.export_container_image( + 'image.tar', 'docker-archive', 'myimage:tag', + ['myimage:tag2', 'myimage:tag3'] + ) + mock_Command_run.assert_called_once_with([ + 'skopeo', 'copy', 'containers-storage:kiwi-image:tag', + 'docker-archive:image.tar:myimage:tag', '--additional-tag', + 'myimage:tag2', '--additional-tag', 'myimage:tag3' + ]) + mock_wipe.assert_called_once_with('image.tar') + + @patch('kiwi.oci_tools.buildah.Path.wipe') + @patch('kiwi.oci_tools.buildah.Command.run') + def test_export_container_image_imported(self, mock_Command_run, mock_wipe): + self.oci.working_image = None + self.oci.imported_image = 'kiwi-image:base_layer' + self.oci.export_container_image( + 'image.tar', 'docker-archive', 'myimage:tag', + ['myimage:tag2', 'myimage:tag3'] + ) + mock_Command_run.assert_called_once_with([ + 'skopeo', 'copy', 'containers-storage:kiwi-image:base_layer', + 'docker-archive:image.tar:myimage:tag', '--additional-tag', + 'myimage:tag2', '--additional-tag', 'myimage:tag3' + ]) + mock_wipe.assert_called_once_with('image.tar') + + @patch('kiwi.oci_tools.buildah.Path.wipe') + @patch('kiwi.oci_tools.buildah.Command.run') + def test_export_container_image_fail(self, mock_Command_run, mock_wipe): + with raises(KiwiBuildahError): + self.oci.working_image = None + self.oci.imported_image = None + self.oci.export_container_image( + 'image.tar', 'docker-archive', 'myimage:tag', + ['myimage:tag2', 'myimage:tag3'] + ) + mock_wipe.assert_called_once_with('image.tar') diff --git a/test/unit/oci_tools_test.py b/test/unit/oci_tools_test.py index 7da9fbd6..c279223a 100644 --- a/test/unit/oci_tools_test.py +++ b/test/unit/oci_tools_test.py @@ -25,6 +25,16 @@ class TestOCI(object): OCI() mock_OCIUmoci.assert_called_once_with() + @patch('kiwi.oci_tools.OCIBuildah') + @patch('kiwi.oci_tools.RuntimeConfig') + def test_oci_tool_buildah( + self, mock_RuntimeConfig, mock_OCIBuildah + ): + self.runtime_config.get_oci_archive_tool.return_value = 'buildah' + mock_RuntimeConfig.return_value = self.runtime_config + OCI() + mock_OCIBuildah.assert_called_once_with() + @patch('kiwi.oci_tools.RuntimeConfig') def test_oci_tool_not_supported(self, mock_RuntimeConfig): self.runtime_config.get_oci_archive_tool.return_value = 'foo' diff --git a/test/unit/runtime_checker_test.py b/test/unit/runtime_checker_test.py index 1d0bfb77..d54ac26e 100644 --- a/test/unit/runtime_checker_test.py +++ b/test/unit/runtime_checker_test.py @@ -82,6 +82,20 @@ class TestRuntimeChecker(object): runtime_checker = RuntimeChecker(xml_state) runtime_checker.check_docker_tool_chain_installed() + @patch('kiwi.runtime_checker.RuntimeConfig.get_oci_archive_tool') + @patch('kiwi.runtime_checker.Path.which') + @raises(KiwiRuntimeError) + def test_check_docker_tool_chain_installed_buildah( + self, mock_which, mock_oci_tool + ): + mock_oci_tool.return_value = 'buildah' + mock_which.return_value = False + xml_state = XMLState( + self.description.load(), ['docker'], 'docker' + ) + runtime_checker = RuntimeChecker(xml_state) + runtime_checker.check_docker_tool_chain_installed() + @patch('kiwi.runtime_checker.Path.which') @patch('kiwi.runtime_checker.CommandCapabilities.check_version') @raises(KiwiRuntimeError)