From 5d241d316a0cb89e8c7bf900f121c08deeba5026 Mon Sep 17 00:00:00 2001 From: Qixiang Wan Date: Fri, 13 Jan 2017 16:57:30 +0800 Subject: [PATCH] ostree-installer: Allow multiple repos in ostree installer Add new key 'repo' to allow specifying multiple repos as the source repositories. And change 'source_repo_from' to allow specifying multiple vairant names to use variant repos as source repositories. Doc of 'source_repo_from' is updated to not mention URL is supported, though we still support that in code. User should add url of repos in 'repo' key instead of 'source_repo_from'. Fixes: #508 Signed-off-by: Qixiang Wan --- doc/configuration.rst | 11 ++- pungi/checks.py | 3 +- pungi/phases/ostree_installer.py | 7 +- tests/test_ostree_installer_phase.py | 117 ++++++++++++++++++++++++--- 4 files changed, 120 insertions(+), 18 deletions(-) diff --git a/doc/configuration.rst b/doc/configuration.rst index 0a1cc54a..a2db4e73 100644 --- a/doc/configuration.rst +++ b/doc/configuration.rst @@ -1155,11 +1155,12 @@ an OSTree repository. This always runs in Koji as a ``runroot`` task. The configuration dict for each variant arch pair must have this key: - * ``source_repo_from`` -- (*str*) Name of variant serving as source - repository or a URL pointing the the repo. + * ``source_repo_from`` -- (*str|[str]*) Name of variant or a name list of + variants serving as source repositories. These keys are optional: + * ``repo`` -- (*str|[str]*) URL of a repo or a list of urls. * ``release`` -- (*str*) Release value to set for the installer image. Set to ``None`` to generate the value :ref:`automatically `. * ``failable`` -- (*[str]*) List of architectures for which this @@ -1205,6 +1206,12 @@ Example config ] 'template_repo': 'https://git.fedorahosted.org/git/spin-kickstarts.git', 'template_branch': 'f24', + + # optional + "repo": [ + "https://example.com/extra-repo1.repo", + "https://example.com/extra-repo2.repo", + ], } }) ] diff --git a/pungi/checks.py b/pungi/checks.py index 3c9de33a..7830721f 100644 --- a/pungi/checks.py +++ b/pungi/checks.py @@ -713,7 +713,8 @@ def _make_schema(): "ostree_installer": _variant_arch_mapping({ "type": "object", "properties": { - "source_repo_from": {"type": "string"}, + "repo": {"$ref": "#/definitions/strings"}, + "source_repo_from": {"$ref": "#/definitions/strings"}, "release": {"$ref": "#/definitions/optional_string"}, "failable": {"$ref": "#/definitions/list_of_strings"}, "installpkgs": {"$ref": "#/definitions/list_of_strings"}, diff --git a/pungi/phases/ostree_installer.py b/pungi/phases/ostree_installer.py index 9c8f8e7f..c9297593 100644 --- a/pungi/phases/ostree_installer.py +++ b/pungi/phases/ostree_installer.py @@ -46,7 +46,10 @@ class OstreeInstallerThread(WorkerThread): self.pool.log_info('[BEGIN] %s' % msg) self.logdir = compose.paths.log.topdir('%s/ostree_installer' % arch) - source_repo = self._get_source_repo(compose, arch, config['source_repo_from']) + source_from_repos = [self._get_source_repo(compose, arch, v) + for v in shortcuts.force_list(config['source_repo_from'])] + repos = shortcuts.force_list(config.pop('repo', [])) + source_repos = source_from_repos + repos output_dir = os.path.join(compose.paths.work.topdir(arch), variant.uid, 'ostree_installer') util.makedirs(os.path.dirname(output_dir)) @@ -55,7 +58,7 @@ class OstreeInstallerThread(WorkerThread): disc_type = compose.conf['disc_types'].get('ostree', 'ostree') volid = get_volid(compose, arch, variant, disc_type=disc_type) - self._run_ostree_cmd(compose, variant, arch, config, source_repo, output_dir, volid) + self._run_ostree_cmd(compose, variant, arch, config, source_repos, output_dir, volid) filename = compose.get_image_name(arch, variant, disc_type=disc_type) self._copy_image(compose, variant, arch, filename, output_dir) diff --git a/tests/test_ostree_installer_phase.py b/tests/test_ostree_installer_phase.py index 4698760f..6aeecdb6 100644 --- a/tests/test_ostree_installer_phase.py +++ b/tests/test_ostree_installer_phase.py @@ -8,6 +8,8 @@ import mock import os import sys +from kobo.shortcuts import force_list + sys.path.insert(0, os.path.join(os.path.dirname(__file__), '..')) from tests import helpers @@ -70,21 +72,33 @@ class OstreeThreadTest(helpers.PungiTestCase): self.assertEqual(compose.im.add.mock_calls, [mock.call('Everything', 'x86_64', image)]) - def assertRunrootCall(self, koji, source, release, isfinal=False, extra=[]): - final = ['--isfinal'] if isfinal else [] + def assertRunrootCall(self, koji, sources, release, isfinal=False, extra=[]): + lorax_cmd = [ + 'lorax', + '--product=Fedora', + '--version=Rawhide', + '--release=%s' % release, + ] + + for s in force_list(sources): + lorax_cmd.append('--source=%s' % s) + + lorax_cmd.append('--variant=Everything') + lorax_cmd.append('--nomacboot') + + if isfinal: + lorax_cmd.append('--isfinal') + + lorax_cmd.append('--volid=test-Everything-x86_64') + + if extra: + lorax_cmd.extend(extra) + + lorax_cmd.append(self.topdir + '/work/x86_64/Everything/ostree_installer') + self.assertEqual(koji.get_runroot_cmd.call_args_list, [mock.call('rrt', 'x86_64', - ['lorax', - '--product=Fedora', - '--version=Rawhide', - '--release=%s' % release, - '--source=%s' % source, - '--variant=Everything', - '--nomacboot'] + - final + - ['--volid=test-Everything-x86_64'] + - extra + - [self.topdir + '/work/x86_64/Everything/ostree_installer'], + lorax_cmd, channel=None, mounts=[self.topdir], packages=['pungi', 'lorax', 'ostree'], task_id=True, use_shell=True)]) @@ -178,6 +192,83 @@ class OstreeThreadTest(helpers.PungiTestCase): self.assertImageAdded(self.compose, ImageCls, iso) self.assertAllCopied(run) + @mock.patch('kobo.shortcuts.run') + @mock.patch('productmd.images.Image') + @mock.patch('pungi.util.get_mtime') + @mock.patch('pungi.util.get_file_size') + @mock.patch('pungi.phases.ostree_installer.iso') + @mock.patch('os.link') + @mock.patch('pungi.wrappers.kojiwrapper.KojiWrapper') + def test_run_with_repo_key(self, KojiWrapper, link, iso, + get_file_size, get_mtime, ImageCls, run): + pool = mock.Mock() + cfg = { + 'source_repo_from': 'Everything', + 'release': '20160321.n.0', + 'repo': [ + 'https://example.com/extra-repo1.repo', + 'https://example.com/extra-repo2.repo', + ], + } + koji = KojiWrapper.return_value + koji.run_runroot_cmd.return_value = { + 'task_id': 1234, + 'retcode': 0, + 'output': 'Foo bar\n', + } + + t = ostree.OstreeInstallerThread(pool) + + t.process((self.compose, self.compose.variants['Everything'], 'x86_64', cfg), 1) + + sources = [ + 'file://%s/compose/Everything/x86_64/os' % self.topdir, + 'https://example.com/extra-repo1.repo', + 'https://example.com/extra-repo2.repo' + ] + + self.assertRunrootCall(koji, sources, cfg['release'], isfinal=True, + extra=['--logfile=%s/logs/x86_64/ostree_installer/lorax.log' % self.topdir]) + + @mock.patch('kobo.shortcuts.run') + @mock.patch('productmd.images.Image') + @mock.patch('pungi.util.get_mtime') + @mock.patch('pungi.util.get_file_size') + @mock.patch('pungi.phases.ostree_installer.iso') + @mock.patch('os.link') + @mock.patch('pungi.wrappers.kojiwrapper.KojiWrapper') + def test_run_with_multiple_variant_repos(self, KojiWrapper, link, iso, + get_file_size, get_mtime, ImageCls, run): + pool = mock.Mock() + cfg = { + 'source_repo_from': ['Everything', 'Server'], + 'release': '20160321.n.0', + 'repo': [ + 'https://example.com/extra-repo1.repo', + 'https://example.com/extra-repo2.repo', + ], + } + koji = KojiWrapper.return_value + koji.run_runroot_cmd.return_value = { + 'task_id': 1234, + 'retcode': 0, + 'output': 'Foo bar\n', + } + + t = ostree.OstreeInstallerThread(pool) + + t.process((self.compose, self.compose.variants['Everything'], 'x86_64', cfg), 1) + + sources = [ + 'file://%s/compose/Everything/x86_64/os' % self.topdir, + 'file://%s/compose/Server/x86_64/os' % self.topdir, + 'https://example.com/extra-repo1.repo', + 'https://example.com/extra-repo2.repo' + ] + + self.assertRunrootCall(koji, sources, cfg['release'], isfinal=True, + extra=['--logfile=%s/logs/x86_64/ostree_installer/lorax.log' % self.topdir]) + @mock.patch('kobo.shortcuts.run') @mock.patch('productmd.images.Image') @mock.patch('pungi.util.get_mtime')