From 80fffdecc2b883ea41b5bde1edd3c38f0c86ba46 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marcus=20Sch=C3=A4fer?= Date: Wed, 30 Nov 2022 15:18:12 +0100 Subject: [PATCH] Support repo URI's with credentials on cmdline Specifying a repository as part of the image description allows for credentials via the username and password attributes. Howver, repositories can also be specified on the commandline via the --set-repo / --add-repo options. The options on the commandline did not allow to specify credentials so far. This commit adds the commandline options --set-repo-credentials and --add-repo-credentials to support them --- doc/source/commands/system_build.rst | 26 +++++++++--- doc/source/commands/system_prepare.rst | 14 +++++++ kiwi/tasks/system_build.py | 34 +++++++++++++-- kiwi/tasks/system_prepare.py | 34 +++++++++++++-- test/unit/cli_test.py | 2 + test/unit/tasks/system_build_test.py | 51 ++++++++++++++++++++--- test/unit/tasks/system_prepare_test.py | 57 ++++++++++++++++++++++---- 7 files changed, 191 insertions(+), 27 deletions(-) diff --git a/doc/source/commands/system_build.rst b/doc/source/commands/system_build.rst index aa89c0cb..465eff98 100644 --- a/doc/source/commands/system_build.rst +++ b/doc/source/commands/system_build.rst @@ -19,7 +19,9 @@ SYNOPSIS [--ignore-repos] [--ignore-repos-used-for-build] [--set-repo=] + [--set-repo-credentials=] [--add-repo=...] + [--add-repo-credentials=...] [--add-package=...] [--add-bootstrap-package=...] [--delete-package=...] @@ -69,6 +71,13 @@ OPTIONS For details about the provided option values see the **--set-repo** information below +--add-repo-credentials= + + For **uri://user:pass@location** type repositories, set the user and + password connected with an add-repo specification. The first + add-repo-credentials is connected with the first add-repo + specification and so on. + --allow-existing-root Allow to use an existing root directory from an earlier @@ -168,17 +177,22 @@ OPTIONS Set to either **true** or **false** to specify if this repository should validate the repository signature. +--set-repo-credentials= + + For **uri://user:pass@location** type repositories, set the user and + password connected to the set-repo specification + --set-container-derived-from= - overwrite the source location of the base container for the selected - image type. The setting is only effective if the configured image type - is setup with an initial derived_from reference + Overwrite the source location of the base container for the selected + image type. The setting is only effective if the configured image type + is setup with an initial derived_from reference --set-container-tag= - overwrite the container tag in the container configuration. - The setting is only effective if the container configuraiton - provides an initial tag value + Overwrite the container tag in the container configuration. + The setting is only effective if the container configuraiton + provides an initial tag value --signing-key= diff --git a/doc/source/commands/system_prepare.rst b/doc/source/commands/system_prepare.rst index 28e4dae3..13a7d87e 100644 --- a/doc/source/commands/system_prepare.rst +++ b/doc/source/commands/system_prepare.rst @@ -17,7 +17,9 @@ SYNOPSIS [--ignore-repos] [--ignore-repos-used-for-build] [--set-repo=] + [--set-repo-credentials=] [--add-repo=...] + [--add-repo-credentials=...] [--add-package=...] [--add-bootstrap-package=...] [--delete-package=...] @@ -69,6 +71,13 @@ OPTIONS For details about the provided option values see the **--set-repo** information below +--add-repo-credentials= + + For **uri://user:pass@location** type repositories, set the user and + password connected with an add-repo specification. The first + add-repo-credentials is connected with the first add-repo + specification and so on. + --allow-existing-root allow to re-use an existing image root directory @@ -169,6 +178,11 @@ OPTIONS Set to either **true** or **false** to specify if this repository should validate the repository signature. +--set-repo-credentials= + + For **uri://user:pass@location** type repositories, set the user and + password connected to the set-repo specification + --set-container-derived-from= overwrite the source location of the base container for the selected diff --git a/kiwi/tasks/system_build.py b/kiwi/tasks/system_build.py index a41654cc..fa9e77ce 100644 --- a/kiwi/tasks/system_build.py +++ b/kiwi/tasks/system_build.py @@ -23,7 +23,9 @@ usage: kiwi-ng system build -h | --help [--ignore-repos] [--ignore-repos-used-for-build] [--set-repo=] + [--set-repo-credentials=] [--add-repo=...] + [--add-repo-credentials=...] [--add-package=...] [--add-bootstrap-package=...] [--delete-package=...] @@ -52,6 +54,11 @@ options: component list for debian based repos as string delimited by a space, main distribution name for debian based repos and repo_gpgcheck(true|false) + --add-repo-credentials= + for uri://user:pass@location type repositories, set the user and + password connected with an add-repo specification. The first + add-repo-credentials is connected with the first add-repo + specification and so on. --allow-existing-root allow to use an existing root directory from an earlier build attempt. Use with caution this could cause an inconsistent @@ -90,6 +97,9 @@ options: component list for debian based repos as string delimited by a space, main distribution name for debian based repos and repo_gpgcheck(true|false) + --set-repo-credentials= + for uri://user:pass@location type repositories, set the user and + password connected to the set-repo specification --signing-key= includes the key-file as a trusted key for package manager validations --target-dir= @@ -97,6 +107,8 @@ options: """ import os import logging +from itertools import zip_longest +from urllib.parse import urlparse # project from kiwi.tasks.base import CliTask @@ -164,13 +176,19 @@ class SystemBuildTask(CliTask): if self.command_args['--set-repo']: self.xml_state.set_repository( - *self._get_repo_parameters(self.command_args['--set-repo']) + *self._get_repo_parameters( + self.command_args['--set-repo'], + self.command_args['--set-repo-credentials'] + ) ) if self.command_args['--add-repo']: - for add_repo in self.command_args['--add-repo']: + for add_repo, add_credentials in zip_longest( + self.command_args['--add-repo'], + self.command_args['--add-repo-credentials'] + ): self.xml_state.add_repository( - *self._get_repo_parameters(add_repo) + *self._get_repo_parameters(add_repo, add_credentials) ) if self.command_args['--set-container-tag']: @@ -311,10 +329,18 @@ class SystemBuildTask(CliTask): return False return self.manual - def _get_repo_parameters(self, tokens): + def _get_repo_parameters(self, tokens, credentials): parameters = self.tentuple_token(tokens) signing_keys_index = 6 + repo_source_index = 0 if not parameters[signing_keys_index]: # make sure to pass empty list for signing_keys param parameters[signing_keys_index] = [] + if credentials: + repo_source = parameters[repo_source_index] + repo_scheme = urlparse(repo_source).scheme + if repo_scheme: + repo_source = repo_source.replace(f'{repo_scheme}://', '') + repo_source = f'{repo_scheme}://{credentials}@{repo_source}' + parameters[repo_source_index] = repo_source return parameters diff --git a/kiwi/tasks/system_prepare.py b/kiwi/tasks/system_prepare.py index 758490be..26061d14 100644 --- a/kiwi/tasks/system_prepare.py +++ b/kiwi/tasks/system_prepare.py @@ -23,7 +23,9 @@ usage: kiwi-ng system prepare -h | --help [--ignore-repos] [--ignore-repos-used-for-build] [--set-repo=] + [--set-repo-credentials=] [--add-repo=...] + [--add-repo-credentials=...] [--add-package=...] [--add-bootstrap-package=...] [--delete-package=...] @@ -51,6 +53,11 @@ options: component list for debian based repos as string delimited by a space, main distribution name for debian based repos and repo_gpgcheck(true|false) + --add-repo-credentials= + for uri://user:pass@location type repositories, set the user and + password connected with an add-repo specification. The first + add-repo-credentials is connected with the first add-repo + specification and so on. --allow-existing-root allow to use an existing root directory. Use with caution this could cause an inconsistent root tree if the existing @@ -90,11 +97,16 @@ options: component list for debian based repos as string delimited by a space, main distribution name for debian based repos and repo_gpgcheck(true|false) + --set-repo-credentials= + for uri://user:pass@location type repositories, set the user and + password connected to the set-repo specification --signing-key= includes the key-file as a trusted key for package manager validations """ import os import logging +from itertools import zip_longest +from urllib.parse import urlparse # project from kiwi.tasks.base import CliTask @@ -149,13 +161,19 @@ class SystemPrepareTask(CliTask): if self.command_args['--set-repo']: self.xml_state.set_repository( - *self._get_repo_parameters(self.command_args['--set-repo']) + *self._get_repo_parameters( + self.command_args['--set-repo'], + self.command_args['--set-repo-credentials'] + ) ) if self.command_args['--add-repo']: - for add_repo in self.command_args['--add-repo']: + for add_repo, add_credentials in zip_longest( + self.command_args['--add-repo'], + self.command_args['--add-repo-credentials'] + ): self.xml_state.add_repository( - *self._get_repo_parameters(add_repo) + *self._get_repo_parameters(add_repo, add_credentials) ) if self.command_args['--set-container-tag']: @@ -287,10 +305,18 @@ class SystemPrepareTask(CliTask): return False return self.manual - def _get_repo_parameters(self, tokens): + def _get_repo_parameters(self, tokens, credentials): parameters = self.tentuple_token(tokens) signing_keys_index = 6 + repo_source_index = 0 if not parameters[signing_keys_index]: # make sure to pass empty list for signing_keys param parameters[signing_keys_index] = [] + if credentials: + repo_source = parameters[repo_source_index] + repo_scheme = urlparse(repo_source).scheme + if repo_scheme: + repo_source = repo_source.replace(f'{repo_scheme}://', '') + repo_source = f'{repo_scheme}://{credentials}@{repo_source}' + parameters[repo_source_index] = repo_source return parameters diff --git a/test/unit/cli_test.py b/test/unit/cli_test.py index ebfaec89..8c4e1654 100644 --- a/test/unit/cli_test.py +++ b/test/unit/cli_test.py @@ -51,6 +51,7 @@ class TestCli: } self.command_args = { '--add-repo': [], + '--add-repo-credentials': [], '--allow-existing-root': False, '--description': 'description', '--help': False, @@ -59,6 +60,7 @@ class TestCli: '--clear-cache': False, '--root': 'directory', '--set-repo': None, + '--set-repo-credentials': None, '--add-package': [], '--add-bootstrap-package': [], '--delete-package': [], diff --git a/test/unit/tasks/system_build_test.py b/test/unit/tasks/system_build_test.py index 98ffb4a1..9c036ffa 100644 --- a/test/unit/tasks/system_build_test.py +++ b/test/unit/tasks/system_build_test.py @@ -92,7 +92,9 @@ class TestSystemBuildTask: self.task.command_args['--description'] = '../data/description' self.task.command_args['--target-dir'] = 'some-target' self.task.command_args['--set-repo'] = None + self.task.command_args['--set-repo-credentials'] = None self.task.command_args['--add-repo'] = [] + self.task.command_args['--add-repo-credentials'] = [] self.task.command_args['--add-package'] = [] self.task.command_args['--add-bootstrap-package'] = [] self.task.command_args['--delete-package'] = [] @@ -289,6 +291,13 @@ class TestSystemBuildTask: 'http://example.com', 'yast2', 'alias', None, None, None, [], None, None, None ) + self.task.command_args['--set-repo-credentials'] = 'user:pass' + mock_set_repo.reset_mock() + self.task.process() + mock_set_repo.assert_called_once_with( + 'http://user:pass@example.com', 'yast2', 'alias', + None, None, None, [], None, None, None + ) @patch('kiwi.xml_state.XMLState.add_repository') @patch('kiwi.logger.Logger.set_logfile') @@ -297,13 +306,45 @@ class TestSystemBuildTask: ): self._init_command_args() self.task.command_args['--add-repo'] = [ - 'http://example.com,yast2,alias,99,false,true' + 'http://example1.com,yast2,alias,99,false,true', + 'http://example2.com,yast2,alias,99,false,true', + 'http://example3.com,yast2,alias,99,false,true' ] self.task.process() - mock_add_repo.assert_called_once_with( - 'http://example.com', 'yast2', 'alias', '99', - False, True, [], None, None, None - ) + assert mock_add_repo.call_args_list == [ + call( + 'http://example1.com', 'yast2', 'alias', '99', + False, True, [], None, None, None + ), + call( + 'http://example2.com', 'yast2', 'alias', '99', + False, True, [], None, None, None + ), + call( + 'http://example3.com', 'yast2', 'alias', '99', + False, True, [], None, None, None + ) + ] + self.task.command_args['--add-repo-credentials'] = [ + 'user1:pass1', + 'user2:pass2' + ] + mock_add_repo.reset_mock() + self.task.process() + assert mock_add_repo.call_args_list == [ + call( + 'http://user1:pass1@example1.com', 'yast2', 'alias', '99', + False, True, [], None, None, None + ), + call( + 'http://user2:pass2@example2.com', 'yast2', 'alias', '99', + False, True, [], None, None, None + ), + call( + 'http://example3.com', 'yast2', 'alias', '99', + False, True, [], None, None, None + ) + ] def test_process_system_build_help(self): self._init_command_args() diff --git a/test/unit/tasks/system_prepare_test.py b/test/unit/tasks/system_prepare_test.py index 5e095270..cec7c1de 100644 --- a/test/unit/tasks/system_prepare_test.py +++ b/test/unit/tasks/system_prepare_test.py @@ -85,7 +85,9 @@ class TestSystemPrepareTask: self.task.command_args['--root'] = '../data/root-dir' self.task.command_args['--allow-existing-root'] = False self.task.command_args['--set-repo'] = None + self.task.command_args['--set-repo-credentials'] = None self.task.command_args['--add-repo'] = [] + self.task.command_args['--add-repo-credentials'] = [] self.task.command_args['--add-package'] = [] self.task.command_args['--add-bootstrap-package'] = [] self.task.command_args['--delete-package'] = [] @@ -270,26 +272,65 @@ class TestSystemPrepareTask: ) @patch('kiwi.xml_state.XMLState.set_repository') - def test_process_system_prepare_set_repo(self, mock_state): + def test_process_system_prepare_set_repo(self, mock_set_repo): self._init_command_args() self.task.command_args['--set-repo'] = 'http://example.com,yast2,alias' self.task.process() - mock_state.assert_called_once_with( + mock_set_repo.assert_called_once_with( 'http://example.com', 'yast2', 'alias', None, None, None, [], None, None, None ) + self.task.command_args['--set-repo-credentials'] = 'user:pass' + mock_set_repo.reset_mock() + self.task.process() + mock_set_repo.assert_called_once_with( + 'http://user:pass@example.com', 'yast2', 'alias', + None, None, None, [], None, None, None + ) @patch('kiwi.xml_state.XMLState.add_repository') - def test_process_system_prepare_add_repo(self, mock_state): + def test_process_system_prepare_add_repo(self, mock_add_repo): self._init_command_args() self.task.command_args['--add-repo'] = [ - 'http://example.com,yast2,alias,99,true' + 'http://example1.com,yast2,alias,99,true', + 'http://example2.com,yast2,alias,99,false,true', + 'http://example3.com,yast2,alias,99,false,true' ] self.task.process() - mock_state.assert_called_once_with( - 'http://example.com', 'yast2', 'alias', '99', - True, None, [], None, None, None - ) + assert mock_add_repo.call_args_list == [ + call( + 'http://example1.com', 'yast2', 'alias', '99', + True, None, [], None, None, None + ), + call( + 'http://example2.com', 'yast2', 'alias', '99', + False, True, [], None, None, None + ), + call( + 'http://example3.com', 'yast2', 'alias', '99', + False, True, [], None, None, None + ) + ] + self.task.command_args['--add-repo-credentials'] = [ + 'user1:pass1', + 'user2:pass2' + ] + mock_add_repo.reset_mock() + self.task.process() + assert mock_add_repo.call_args_list == [ + call( + 'http://user1:pass1@example1.com', 'yast2', 'alias', '99', + True, None, [], None, None, None + ), + call( + 'http://user2:pass2@example2.com', 'yast2', 'alias', '99', + False, True, [], None, None, None + ), + call( + 'http://example3.com', 'yast2', 'alias', '99', + False, True, [], None, None, None + ) + ] def test_process_system_prepare_help(self): self._init_command_args()