ostree-installer: Allow multiple repos in ostree installer

Add new key 'repo' to allow specifying multiple repos as the source
repositories. And change 'source_repo_from' to allow specifying multiple
vairant names to use variant repos as source repositories.

Doc of 'source_repo_from' is updated to not mention URL is supported,
though we still support that in code. User should add url of repos in
'repo' key instead of 'source_repo_from'.

Fixes: #508

Signed-off-by: Qixiang Wan <qwan@redhat.com>

doc/configuration.rst

@@ -1155,11 +1155,12 @@ an OSTree repository. This always runs in Koji as a ``runroot`` task.
 
     The configuration dict for each variant arch pair must have this key:
 
-    * ``source_repo_from`` -- (*str*) Name of variant serving as source
+    * ``source_repo_from`` -- (*str|[str]*) Name of variant or a name list of
-      repository or a URL pointing the the repo.
+      variants serving as source repositories.
 
     These keys are optional:
 
+    * ``repo`` -- (*str|[str]*) URL of a repo or a list of urls.
     * ``release`` -- (*str*) Release value to set for the installer image. Set
       to ``None`` to generate the value :ref:`automatically <auto_release>`.
     * ``failable`` -- (*[str]*) List of architectures for which this
@@ -1205,6 +1206,12 @@ Example config
                 ]
                 'template_repo': 'https://git.fedorahosted.org/git/spin-kickstarts.git',
                 'template_branch': 'f24',
+
+                # optional
+                "repo": [
+                    "https://example.com/extra-repo1.repo",
+                    "https://example.com/extra-repo2.repo",
+                ],
             }
         })
     ]

pungi/checks.py

@@ -713,7 +713,8 @@ def _make_schema():
             "ostree_installer": _variant_arch_mapping({
                 "type": "object",
                 "properties": {
-                    "source_repo_from": {"type": "string"},
+                    "repo": {"$ref": "#/definitions/strings"},
+                    "source_repo_from": {"$ref": "#/definitions/strings"},
                     "release": {"$ref": "#/definitions/optional_string"},
                     "failable": {"$ref": "#/definitions/list_of_strings"},
                     "installpkgs": {"$ref": "#/definitions/list_of_strings"},

pungi/phases/ostree_installer.py

@@ -46,7 +46,10 @@ class OstreeInstallerThread(WorkerThread):
         self.pool.log_info('[BEGIN] %s' % msg)
         self.logdir = compose.paths.log.topdir('%s/ostree_installer' % arch)
 
-        source_repo = self._get_source_repo(compose, arch, config['source_repo_from'])
+        source_from_repos = [self._get_source_repo(compose, arch, v)
+                             for v in shortcuts.force_list(config['source_repo_from'])]
+        repos = shortcuts.force_list(config.pop('repo', []))
+        source_repos = source_from_repos + repos
         output_dir = os.path.join(compose.paths.work.topdir(arch), variant.uid, 'ostree_installer')
         util.makedirs(os.path.dirname(output_dir))
 
@@ -55,7 +58,7 @@ class OstreeInstallerThread(WorkerThread):
         disc_type = compose.conf['disc_types'].get('ostree', 'ostree')
 
         volid = get_volid(compose, arch, variant, disc_type=disc_type)
-        self._run_ostree_cmd(compose, variant, arch, config, source_repo, output_dir, volid)
+        self._run_ostree_cmd(compose, variant, arch, config, source_repos, output_dir, volid)
 
         filename = compose.get_image_name(arch, variant, disc_type=disc_type)
         self._copy_image(compose, variant, arch, filename, output_dir)

tests/test_ostree_installer_phase.py

@@ -8,6 +8,8 @@ import mock
 import os
 import sys
 
+from kobo.shortcuts import force_list
+
 sys.path.insert(0, os.path.join(os.path.dirname(__file__), '..'))
 
 from tests import helpers
@@ -70,21 +72,33 @@ class OstreeThreadTest(helpers.PungiTestCase):
         self.assertEqual(compose.im.add.mock_calls,
                          [mock.call('Everything', 'x86_64', image)])
 
-    def assertRunrootCall(self, koji, source, release, isfinal=False, extra=[]):
+    def assertRunrootCall(self, koji, sources, release, isfinal=False, extra=[]):
-        final = ['--isfinal'] if isfinal else []
+        lorax_cmd = [
+            'lorax',
+            '--product=Fedora',
+            '--version=Rawhide',
+            '--release=%s' % release,
+        ]
+
+        for s in force_list(sources):
+            lorax_cmd.append('--source=%s' % s)
+
+        lorax_cmd.append('--variant=Everything')
+        lorax_cmd.append('--nomacboot')
+
+        if isfinal:
+            lorax_cmd.append('--isfinal')
+
+        lorax_cmd.append('--volid=test-Everything-x86_64')
+
+        if extra:
+            lorax_cmd.extend(extra)
+
+        lorax_cmd.append(self.topdir + '/work/x86_64/Everything/ostree_installer')
+
         self.assertEqual(koji.get_runroot_cmd.call_args_list,
                          [mock.call('rrt', 'x86_64',
-                                    ['lorax',
+                                    lorax_cmd,
-                                     '--product=Fedora',
-                                     '--version=Rawhide',
-                                     '--release=%s' % release,
-                                     '--source=%s' % source,
-                                     '--variant=Everything',
-                                     '--nomacboot'] +
-                                    final +
-                                    ['--volid=test-Everything-x86_64'] +
-                                    extra +
-                                    [self.topdir + '/work/x86_64/Everything/ostree_installer'],
                                     channel=None, mounts=[self.topdir],
                                     packages=['pungi', 'lorax', 'ostree'],
                                     task_id=True, use_shell=True)])
@@ -178,6 +192,83 @@ class OstreeThreadTest(helpers.PungiTestCase):
         self.assertImageAdded(self.compose, ImageCls, iso)
         self.assertAllCopied(run)
 
+    @mock.patch('kobo.shortcuts.run')
+    @mock.patch('productmd.images.Image')
+    @mock.patch('pungi.util.get_mtime')
+    @mock.patch('pungi.util.get_file_size')
+    @mock.patch('pungi.phases.ostree_installer.iso')
+    @mock.patch('os.link')
+    @mock.patch('pungi.wrappers.kojiwrapper.KojiWrapper')
+    def test_run_with_repo_key(self, KojiWrapper, link, iso,
+                               get_file_size, get_mtime, ImageCls, run):
+        pool = mock.Mock()
+        cfg = {
+            'source_repo_from': 'Everything',
+            'release': '20160321.n.0',
+            'repo': [
+                'https://example.com/extra-repo1.repo',
+                'https://example.com/extra-repo2.repo',
+            ],
+        }
+        koji = KojiWrapper.return_value
+        koji.run_runroot_cmd.return_value = {
+            'task_id': 1234,
+            'retcode': 0,
+            'output': 'Foo bar\n',
+        }
+
+        t = ostree.OstreeInstallerThread(pool)
+
+        t.process((self.compose, self.compose.variants['Everything'], 'x86_64', cfg), 1)
+
+        sources = [
+            'file://%s/compose/Everything/x86_64/os' % self.topdir,
+            'https://example.com/extra-repo1.repo',
+            'https://example.com/extra-repo2.repo'
+        ]
+
+        self.assertRunrootCall(koji, sources, cfg['release'], isfinal=True,
+                               extra=['--logfile=%s/logs/x86_64/ostree_installer/lorax.log' % self.topdir])
+
+    @mock.patch('kobo.shortcuts.run')
+    @mock.patch('productmd.images.Image')
+    @mock.patch('pungi.util.get_mtime')
+    @mock.patch('pungi.util.get_file_size')
+    @mock.patch('pungi.phases.ostree_installer.iso')
+    @mock.patch('os.link')
+    @mock.patch('pungi.wrappers.kojiwrapper.KojiWrapper')
+    def test_run_with_multiple_variant_repos(self, KojiWrapper, link, iso,
+                                             get_file_size, get_mtime, ImageCls, run):
+        pool = mock.Mock()
+        cfg = {
+            'source_repo_from': ['Everything', 'Server'],
+            'release': '20160321.n.0',
+            'repo': [
+                'https://example.com/extra-repo1.repo',
+                'https://example.com/extra-repo2.repo',
+            ],
+        }
+        koji = KojiWrapper.return_value
+        koji.run_runroot_cmd.return_value = {
+            'task_id': 1234,
+            'retcode': 0,
+            'output': 'Foo bar\n',
+        }
+
+        t = ostree.OstreeInstallerThread(pool)
+
+        t.process((self.compose, self.compose.variants['Everything'], 'x86_64', cfg), 1)
+
+        sources = [
+            'file://%s/compose/Everything/x86_64/os' % self.topdir,
+            'file://%s/compose/Server/x86_64/os' % self.topdir,
+            'https://example.com/extra-repo1.repo',
+            'https://example.com/extra-repo2.repo'
+        ]
+
+        self.assertRunrootCall(koji, sources, cfg['release'], isfinal=True,
+                               extra=['--logfile=%s/logs/x86_64/ostree_installer/lorax.log' % self.topdir])
+
     @mock.patch('kobo.shortcuts.run')
     @mock.patch('productmd.images.Image')
     @mock.patch('pungi.util.get_mtime')