Merge #495 osbs: Enable specifying extra repos

doc/configuration.rst

@@ -1244,7 +1244,9 @@ they are not scratch builds).
     This includes ``name``, ``version``, ``scratch`` and ``priority``.
 
     A value for ``yum_repourls`` will be created automatically and point at a
-    repository in the current compose.
+    repository in the current compose. You can add extra repositories with
+    ``repo`` key having a list of urls pointing to ``.repo`` files or
+    ``repo_from`` as a list of variants in current compose.
 
 
 Example config

pungi/checks.py

@@ -828,6 +828,8 @@ def _make_schema():
                             "version": {"type": "string"},
                             "scratch": {"type": "boolean"},
                             "priority": {"type": "number"},
+                            "repo": {"$ref": "#/definitions/strings"},
+                            "repo_from": {"$ref": "#/definitions/strings"},
                         },
                         "required": ["url", "target"]
                     }

pungi/phases/osbs.py

@@ -3,6 +3,7 @@
 import json
 import os
 from kobo.threads import ThreadPool, WorkerThread
+from kobo import shortcuts
 
 from .base import ConfigGuardedPhase, PhaseLoggerMixin
 from .. import util
@@ -50,15 +51,15 @@ class OSBSThread(WorkerThread):
         koji.login()
 
         # Start task
-        try:
+        source = util.resolve_git_url(config.pop('url'))
-            source = util.resolve_git_url(config.pop('url'))
+        target = config.pop('target')
-            target = config.pop('target')
-        except KeyError as exc:
-            raise RuntimeError('OSBS: missing config key %s for %s'
-                               % (exc, variant.uid))
         priority = config.pop('priority', None)
+        repos = shortcuts.force_list(config.pop('repo', []))
+        compose_repos = [self._get_repo(compose, v)
+                         for v in [variant.uid] + shortcuts.force_list(
+                             config.pop('repo_from', []))]
 
-        config['yum_repourls'] = [self._get_repo(compose, variant)]
+        config['yum_repourls'] = compose_repos + repos
 
         task_id = koji.koji_proxy.buildContainer(source, target, config,
                                                  priority=priority)
@@ -106,11 +107,17 @@ class OSBSThread(WorkerThread):
             self.pool.metadata.setdefault(
                 variant.uid, {}).setdefault(arch, []).append(data)
 
-    def _get_repo(self, compose, variant):
+    def _get_repo(self, compose, variant_uid):
         """
         Write a .repo file pointing to current variant and return URL to the
         file.
         """
+        try:
+            variant = compose.all_variants[variant_uid]
+        except KeyError:
+            raise RuntimeError(
+                'There is no variant %s to get repo from to pass to OSBS.'
+                % (variant_uid))
         os_tree = compose.paths.compose.os_tree('$basearch', variant,
                                                 create_dir=False)
         repo_file = os.path.join(compose.paths.work.tmp_dir(None, variant),

tests/test_osbs_phase.py

@@ -8,12 +8,14 @@ except ImportError:
 import mock
 import json
 
+import copy
 import os
 import sys
 
 sys.path.insert(0, os.path.join(os.path.dirname(__file__), '..'))
 
 from tests import helpers
+from pungi import checks
 from pungi.phases import osbs
 
 
@@ -192,10 +194,28 @@ class OSBSThreadTest(helpers.PungiTestCase):
                 mock.call.koji_proxy.getBuild(54321),
                 mock.call.koji_proxy.listArchives(54321)])
 
-    def _assertRepoFile(self):
+    def _assertRepoFile(self, variants=None):
-        with open(self.topdir + '/work/global/tmp-Server/compose-rpms-1.repo') as f:
+        variants = variants or ['Server']
-            lines = f.read().split('\n')
+        for variant in variants:
-            self.assertIn('baseurl=http://root/compose/Server/$baseurl/os', lines)
+            with open(self.topdir + '/work/global/tmp-%s/compose-rpms-1.repo' % variant) as f:
+                lines = f.read().split('\n')
+                self.assertIn('baseurl=http://root/compose/%s/$basearch/os' % variant, lines)
+
+    def _assertConfigCorrect(self, cfg):
+        config = copy.deepcopy(self.compose.conf)
+        config['osbs'] = {
+            '^Server$': cfg
+        }
+        self.assertEqual(([], []), checks.validate(config))
+
+    def _assertConfigMissing(self, cfg, key):
+        config = copy.deepcopy(self.compose.conf)
+        config['osbs'] = {
+            '^Server$': cfg
+        }
+        self.assertEqual(
+            (['Failed validation in osbs.^Server$: \'%s\' is a required property' % key], []),
+            checks.validate(config))
 
     @mock.patch('pungi.util.resolve_git_url')
     @mock.patch('pungi.phases.osbs.kojiwrapper.KojiWrapper')
@@ -205,11 +225,13 @@ class OSBSThreadTest(helpers.PungiTestCase):
             'target': 'f24-docker-candidate',
         }
         self._setupMock(KojiWrapper, resolve_git_url)
+        self._assertConfigCorrect(cfg)
 
         self.t.process((self.compose, self.compose.variants['Server'], cfg), 1)
 
         self._assertCorrectCalls({})
         self._assertCorrectMetadata()
+        self._assertRepoFile()
 
     @mock.patch('pungi.util.resolve_git_url')
     @mock.patch('pungi.phases.osbs.kojiwrapper.KojiWrapper')
@@ -220,11 +242,13 @@ class OSBSThreadTest(helpers.PungiTestCase):
             'failable': ['*']
         }
         self._setupMock(KojiWrapper, resolve_git_url)
+        self._assertConfigCorrect(cfg)
 
         self.t.process((self.compose, self.compose.variants['Server'], cfg), 1)
 
         self._assertCorrectCalls({})
         self._assertCorrectMetadata()
+        self._assertRepoFile()
 
     @mock.patch('pungi.util.resolve_git_url')
     @mock.patch('pungi.phases.osbs.kojiwrapper.KojiWrapper')
@@ -236,39 +260,104 @@ class OSBSThreadTest(helpers.PungiTestCase):
             'version': '1.0',
         }
         self._setupMock(KojiWrapper, resolve_git_url)
+        self._assertConfigCorrect(cfg)
 
         self.t.process((self.compose, self.compose.variants['Server'], cfg), 1)
 
         self._assertCorrectCalls({'name': 'my-name', 'version': '1.0'})
         self._assertCorrectMetadata()
+        self._assertRepoFile()
 
     @mock.patch('pungi.util.resolve_git_url')
     @mock.patch('pungi.phases.osbs.kojiwrapper.KojiWrapper')
-    def test_run_with_missing_url(self, KojiWrapper, resolve_git_url):
+    def test_run_with_extra_repos(self, KojiWrapper, resolve_git_url):
+        cfg = {
+            'url': 'git://example.com/repo?#HEAD',
+            'target': 'f24-docker-candidate',
+            'name': 'my-name',
+            'version': '1.0',
+            'repo': 'http://pkgs.example.com/my.repo',
+            'repo_from': 'Everything',
+        }
+        self._setupMock(KojiWrapper, resolve_git_url)
+        self._assertConfigCorrect(cfg)
+
+        self.t.process((self.compose, self.compose.variants['Server'], cfg), 1)
+
+        options = {
+            'name': 'my-name',
+            'version': '1.0',
+            'yum_repourls': [
+                'http://root/work/global/tmp-Server/compose-rpms-1.repo',
+                'http://root/work/global/tmp-Everything/compose-rpms-1.repo',
+                'http://pkgs.example.com/my.repo',
+            ]
+        }
+        self._assertCorrectCalls(options)
+        self._assertCorrectMetadata()
+        self._assertRepoFile(['Server', 'Everything'])
+
+    @mock.patch('pungi.util.resolve_git_url')
+    @mock.patch('pungi.phases.osbs.kojiwrapper.KojiWrapper')
+    def test_run_with_extra_repos_in_list(self, KojiWrapper, resolve_git_url):
+        cfg = {
+            'url': 'git://example.com/repo?#HEAD',
+            'target': 'f24-docker-candidate',
+            'name': 'my-name',
+            'version': '1.0',
+            'repo': ['http://pkgs.example.com/my.repo'],
+            'repo_from': ['Everything', 'Client'],
+        }
+        self._assertConfigCorrect(cfg)
+        self._setupMock(KojiWrapper, resolve_git_url)
+
+        self.t.process((self.compose, self.compose.variants['Server'], cfg), 1)
+
+        options = {
+            'name': 'my-name',
+            'version': '1.0',
+            'yum_repourls': [
+                'http://root/work/global/tmp-Server/compose-rpms-1.repo',
+                'http://root/work/global/tmp-Everything/compose-rpms-1.repo',
+                'http://root/work/global/tmp-Client/compose-rpms-1.repo',
+                'http://pkgs.example.com/my.repo',
+            ]
+        }
+        self._assertCorrectCalls(options)
+        self._assertCorrectMetadata()
+        self._assertRepoFile(['Server', 'Everything', 'Client'])
+
+    @mock.patch('pungi.util.resolve_git_url')
+    @mock.patch('pungi.phases.osbs.kojiwrapper.KojiWrapper')
+    def test_run_with_extra_repos_missing_variant(self, KojiWrapper, resolve_git_url):
+        cfg = {
+            'url': 'git://example.com/repo?#HEAD',
+            'target': 'f24-docker-candidate',
+            'name': 'my-name',
+            'version': '1.0',
+            'repo_from': 'Gold',
+        }
+        self._assertConfigCorrect(cfg)
+        self._setupMock(KojiWrapper, resolve_git_url)
+
+        with self.assertRaises(RuntimeError) as ctx:
+            self.t.process((self.compose, self.compose.variants['Server'], cfg), 1)
+
+        self.assertIn('no variant Gold', str(ctx.exception))
+
+    def test_run_with_missing_url(self):
         cfg = {
             'target': 'f24-docker-candidate',
             'name': 'my-name',
         }
-        self._setupMock(KojiWrapper, resolve_git_url)
+        self._assertConfigMissing(cfg, 'url')
 
-        with self.assertRaises(RuntimeError) as ctx:
+    def test_run_with_missing_target(self):
-            self.t.process((self.compose, self.compose.variants['Server'], cfg), 1)
-
-        self.assertIn("missing config key 'url' for Server", str(ctx.exception))
-
-    @mock.patch('pungi.util.resolve_git_url')
-    @mock.patch('pungi.phases.osbs.kojiwrapper.KojiWrapper')
-    def test_run_with_missing_target(self, KojiWrapper, resolve_git_url):
         cfg = {
             'url': 'git://example.com/repo?#HEAD',
             'name': 'my-name',
         }
-        self._setupMock(KojiWrapper, resolve_git_url)
+        self._assertConfigMissing(cfg, 'target')
-
-        with self.assertRaises(RuntimeError) as ctx:
-            self.t.process((self.compose, self.compose.variants['Server'], cfg), 1)
-
-        self.assertIn("missing config key 'target' for Server", str(ctx.exception))
 
     @mock.patch('pungi.util.resolve_git_url')
     @mock.patch('pungi.phases.osbs.kojiwrapper.KojiWrapper')
@@ -277,6 +366,7 @@ class OSBSThreadTest(helpers.PungiTestCase):
             'url': 'git://example.com/repo?#HEAD',
             'target': 'fedora-24-docker-candidate',
         }
+        self._assertConfigCorrect(cfg)
         self._setupMock(KojiWrapper, resolve_git_url)
         self.wrapper.watch_task.return_value = 1
 
@@ -293,6 +383,7 @@ class OSBSThreadTest(helpers.PungiTestCase):
             'target': 'fedora-24-docker-candidate',
             'failable': ['*']
         }
+        self._assertConfigCorrect(cfg)
         self._setupMock(KojiWrapper, resolve_git_url)
         self.wrapper.watch_task.return_value = 1
 
@@ -306,6 +397,7 @@ class OSBSThreadTest(helpers.PungiTestCase):
             'target': 'fedora-24-docker-candidate',
             'scratch': True,
         }
+        self._assertConfigCorrect(cfg)
         self._setupMock(KojiWrapper, resolve_git_url)
 
         self.t.process((self.compose, self.compose.variants['Server'], cfg), 1)