Merge #463 [ostree] Allow extra repos to get packages for composing OSTree repository

doc/configuration.rst

@@ -1087,6 +1087,19 @@ a new commit.
 
     These keys are optional:
 
+    * ``extra_source_repos`` -- (*[dict]*) Extra source repos to get packages
+      while composing the OSTree repository. Each dict represents a yum repo.
+      The allowed keys are:
+
+      * ``name`` (required)
+      * ``baseurl`` (required) -- URL of external repo or variant UID, in the case
+        of variant UID, url to variant repo will be built automatically.
+      * ``gpgcheck`` (optional)
+      * ``exclude`` (optional)
+
+    * ``keep_original_sources`` -- (*bool*) Keep the existing source repos in
+      the tree config file. If not enabled, all the original source repos will
+      be removed from the tree config file.
     * ``config_branch`` -- (*str*) Git branch of the repo to use. Defaults to
       ``master``.
     * ``failable`` -- (*[str]*) List of architectures for which this
@@ -1107,7 +1120,20 @@ Example config
             "x86_64": {
                 "treefile": "fedora-atomic-docker-host.json",
                 "config_url": "https://git.fedorahosted.org/git/fedora-atomic.git",
-                "source_repo_from": "Everything",
+                "source_repo_from": "Server",
+                "extra_source_repos": [
+                    {
+                        "name": "repo_a",
+                        "baseurl": "http://example.com/repo/x86_64/os",
+                        "exclude": "systemd-container",
+                        "gpgcheck": False
+                    },
+                    {
+                        "name": "Everything",
+                        "baseurl": "Everything",
+                    }
+                ],
+                "keep_original_sources": True,
                 "ostree_repo": "/mnt/koji/compose/atomic/Rawhide/",
                 "update_summary": True,
                 "version": "24"

pungi/checks.py

@@ -318,11 +318,27 @@ def _make_schema():
                 ]
             },
 
+            "source_repo_dict": {
+                "type": "object",
+                "properties": {
+                    "name": {"type": "string"},
+                    "baseurl": {"type": "string"},
+                    "exclude": {"type": "string"},
+                    "gpgcheck": {"type": "boolean"},
+                },
+                "additionalProperties": False,
+            },
+
             "list_of_strings": {
                 "type": "array",
                 "items": {"type": "string"},
             },
 
+            "list_of_source_repo_dicts": {
+                "type": "array",
+                "items": {"$ref": "#/definitions/source_repo_dict"},
+            },
+
             "strings": {
                 "anyOf": [
                     {"type": "string"},
@@ -675,6 +691,8 @@ def _make_schema():
                     "treefile": {"type": "string"},
                     "config_url": {"type": "string"},
                     "source_repo_from": {"type": "string"},
+                    "extra_source_repos": {"$ref": "#/definitions/list_of_source_repo_dicts"},
+                    "keep_original_sources": {"type": "boolean"},
                     "ostree_repo": {"type": "string"},
                     "failable": {"$ref": "#/definitions/list_of_strings"},
                     "update_summary": {"type": "boolean"},

pungi/phases/ostree.py

@@ -1,9 +1,9 @@
 # -*- coding: utf-8 -*-
 
+import datetime
 import json
 import os
 from kobo.threads import ThreadPool, WorkerThread
-import re
 
 from .base import ConfigGuardedPhase
 from .. import util
@@ -52,7 +52,29 @@ class OSTreeThread(WorkerThread):
                                                                       create_dir=False))
 
         self._clone_repo(repodir, config['config_url'], config.get('config_branch', 'master'))
-        self._tweak_mirrorlist(repodir, source_repo)
+
+        treeconf = os.path.join(repodir, config['treefile'])
+        source_repos = [{'name': '%s-%s' % (compose.compose_id, config['source_repo_from']),
+                         'baseurl': source_repo}]
+
+        extra_source_repos = config.get('extra_source_repos', None)
+        if extra_source_repos:
+            for extra in extra_source_repos:
+                baseurl = extra['baseurl']
+                if "://" not in baseurl:
+                    # it's variant UID, translate to url
+                    variant = compose.variants[baseurl]
+                    url = translate_path(compose,
+                                         compose.paths.compose.repository('$basearch',
+                                                                          variant,
+                                                                          create_dir=False))
+                    extra['baseurl'] = url
+
+            source_repos = source_repos + extra_source_repos
+
+        keep_original_sources = config.get('keep_original_sources', False)
+        self._tweak_treeconf(treeconf, source_repos=source_repos,
+                             keep_original_sources=keep_original_sources)
 
         # Ensure target directory exists, otherwise Koji task will fail to
         # mount it.
@@ -136,23 +158,42 @@ class OSTreeThread(WorkerThread):
         scm.get_dir_from_scm({'scm': 'git', 'repo': url, 'branch': branch, 'dir': '.'},
                              repodir, logger=self.pool._logger)
 
-    def _tweak_mirrorlist(self, repodir, source_repo):
+    def _tweak_treeconf(self, treeconf, source_repos, keep_original_sources=False):
-        for file in os.listdir(repodir):
+        """
-            if file.endswith('.repo'):
+        Update tree config file by adding new repos and remove existing repos
-                tweak_file(os.path.join(repodir, file), source_repo)
+        from the tree config file if 'keep_original_sources' is not enabled.
+        """
+        # add this timestamp to repo name to get unique repo filename and repo name
+        # should be safe enough
+        time = datetime.datetime.now().strftime("%Y%m%d%H%M%S")
 
+        treeconf_dir = os.path.dirname(treeconf)
+        with open(treeconf, 'r') as f:
+            treeconf_content = json.load(f)
 
-def tweak_file(path, source_repo):
+        # backup the old tree config
-    """
+        os.rename(treeconf, '%s.%s.bak' % (treeconf, time))
-    Ensure a given .repo file points to `source_repo`.
 
-    This function replaces all lines starting with `mirrorlist`, `metalink` or
+        repos = []
-    `baseurl` with `baseurl` set to requested repository.
+        for repo in source_repos:
-    """
+            name = "%s-%s" % (repo['name'], time)
-    with open(path, 'r') as f:
+            with open("%s/%s.repo" % (treeconf_dir, name), 'w') as f:
-        contents = f.read()
+                f.write("[%s]\n" % name)
-    replacement = 'baseurl=%s' % source_repo
+                f.write("name=%s\n" % name)
-    exp = re.compile(r'^(mirrorlist|metalink|baseurl)=.*$', re.MULTILINE)
+                f.write("baseurl=%s\n" % repo['baseurl'])
-    contents = exp.sub(replacement, contents)
+                exclude = repo.get('exclude', None)
-    with open(path, 'w') as f:
+                if exclude:
-        f.write(contents)
+                    f.write("exclude=%s\n" % exclude)
+                gpgcheck = '1' if repo.get('gpgcheck', False) else '0'
+                f.write("gpgcheck=%s\n" % gpgcheck)
+            repos.append(name)
+
+        original_repos = treeconf_content.get('repos', [])
+        if keep_original_sources:
+            treeconf_content['repos'] = original_repos + repos
+        else:
+            treeconf_content['repos'] = repos
+
+        # update tree config to add new repos
+        with open(treeconf, 'w') as f:
+            json.dump(treeconf_content, f, indent=4)

tests/test_ostree_phase.py

@@ -69,13 +69,14 @@ class OSTreeThreadTest(helpers.PungiTestCase):
     def _dummy_config_repo(self, scm_dict, target, logger=None):
         os.makedirs(target)
         helpers.touch(os.path.join(target, 'fedora-atomic-docker-host.json'),
-                      json.dumps({'ref': 'fedora-atomic/25/x86_64'}))
+                      json.dumps({'ref': 'fedora-atomic/25/x86_64',
+                                  'repos': ['fedora-rawhide', 'fedora-24', 'fedora-23']}))
         helpers.touch(os.path.join(target, 'fedora-rawhide.repo'),
-                      'mirrorlist=mirror-mirror-on-the-wall')
+                      '[fedora-rawhide]\nmirrorlist=mirror-mirror-on-the-wall')
         helpers.touch(os.path.join(target, 'fedora-24.repo'),
-                      'metalink=who-is-the-fairest-of-them-all')
+                      '[fedora-24]\nmetalink=who-is-the-fairest-of-them-all')
         helpers.touch(os.path.join(target, 'fedora-23.repo'),
-                      'baseurl=why-not-zoidberg?')
+                      '[fedora-23]\nbaseurl=why-not-zoidberg?')
 
     def _mock_runroot(self, retcode, writefiles=None):
         """Pretend to run a task in runroot, creating a log file with given line
@@ -120,10 +121,17 @@ class OSTreeThreadTest(helpers.PungiTestCase):
                          [mock.call(koji.get_runroot_cmd.return_value,
                                     log_file=self.topdir + '/logs/x86_64/Everything/ostree-1/runroot.log')])
 
-        for fp in ['fedora-rawhide.repo', 'fedora-24.repo', 'fedora-24.repo']:
+        repo_files = []
-            with open(os.path.join(self.topdir, 'work/ostree-1/config_repo', fp)) as f:
+        for fp in os.listdir(os.path.join(self.topdir, 'work/ostree-1/config_repo')):
-                self.assertIn('baseurl=http://example.com/Everything/$basearch/os',
+            if fp.endswith('.repo'):
-                              f.read())
+                repo_files.append(fp)
+
+                if fp not in ['fedora-rawhide.repo', 'fedora-24.repo', 'fedora-23.repo']:
+                    with open(os.path.join(self.topdir, 'work/ostree-1/config_repo', fp)) as f:
+                        self.assertIn('baseurl=http://example.com/Everything/$basearch/os', f.read())
+        # test a new repo file created
+        self.assertEqual(len(repo_files), 4)
+
         self.assertTrue(os.path.isdir(self.repo))
 
     @mock.patch('pungi.wrappers.scm.get_dir_from_scm')
@@ -257,12 +265,6 @@ class OSTreeThreadTest(helpers.PungiTestCase):
                          [mock.call(koji.get_runroot_cmd.return_value,
                                     log_file=self.topdir + '/logs/x86_64/Everything/ostree-1/runroot.log')])
 
-        for fp in ['fedora-rawhide.repo', 'fedora-24.repo', 'fedora-24.repo']:
-            with open(os.path.join(self.topdir, 'work/ostree-1/config_repo', fp)) as f:
-                self.assertIn('baseurl=http://example.com/Everything/$basearch/os',
-                              f.read())
-        self.assertTrue(os.path.isdir(self.repo))
-
     @mock.patch('pungi.wrappers.scm.get_dir_from_scm')
     @mock.patch('pungi.wrappers.kojiwrapper.KojiWrapper')
     def test_run_with_versioning_metadata(self, KojiWrapper, get_dir_from_scm):
@@ -295,5 +297,94 @@ class OSTreeThreadTest(helpers.PungiTestCase):
                          [mock.call(koji.get_runroot_cmd.return_value,
                                     log_file=self.topdir + '/logs/x86_64/Everything/ostree-1/runroot.log')])
 
+    @mock.patch('pungi.wrappers.scm.get_dir_from_scm')
+    @mock.patch('pungi.wrappers.kojiwrapper.KojiWrapper')
+    def test_run_with_extra_source_repos(self, KojiWrapper, get_dir_from_scm):
+        get_dir_from_scm.side_effect = self._dummy_config_repo
+
+        koji = KojiWrapper.return_value
+        koji.run_runroot_cmd.side_effect = self._mock_runroot(0)
+
+        cfg = {
+            'source_repo_from': 'Everything',
+            'extra_source_repos': [
+                {
+                    'name': 'repo_a',
+                    'baseurl': 'http://url/to/repo/a',
+                    'exclude': 'systemd-container'
+                },
+                {
+                    'name': 'Server',
+                    'baseurl': 'Server',
+                    'exclude': 'systemd-container'
+                }
+            ],
+            'config_url': 'https://git.fedorahosted.org/git/fedora-atomic.git',
+            'config_branch': 'f24',
+            'treefile': 'fedora-atomic-docker-host.json',
+            'ostree_repo': self.repo
+        }
+
+        t = ostree.OSTreeThread(self.pool)
+
+        t.process((self.compose, self.compose.variants['Everything'], 'x86_64', cfg), 1)
+
+        repo_files = []
+        for fp in os.listdir(os.path.join(self.topdir, 'work/ostree-1/config_repo')):
+            if fp.endswith('.repo'):
+                repo_files.append(fp)
+
+                if fp not in ['fedora-rawhide.repo', 'fedora-24.repo', 'fedora-23.repo']:
+                    if fp.startswith('repo_a'):
+                        with open(os.path.join(self.topdir, 'work/ostree-1/config_repo', fp)) as f:
+                            # ignore timestamp in repo name while checking
+                            content = f.read()
+                            self.assertIn('[repo_a', content)
+                            self.assertIn('name=repo_a', content)
+                            self.assertIn('baseurl=http://url/to/repo/a', content)
+                            self.assertIn('exclude=systemd-container', content)
+                            self.assertIn('gpgcheck=0', content)
+                    elif fp.startswith('Server'):
+                        with open(os.path.join(self.topdir, 'work/ostree-1/config_repo', fp)) as f:
+                            content = f.read()
+                            self.assertIn('[Server', content)
+                            self.assertIn('baseurl=http://example.com/Server/$basearch/os', content)
+                            self.assertIn('exclude=systemd-container', content)
+                            self.assertIn('gpgcheck=0', content)
+                    else:
+                        # this is the Everything repo (source_repo_from)
+                        with open(os.path.join(self.topdir, 'work/ostree-1/config_repo', fp)) as f:
+                            self.assertIn('baseurl=http://example.com/Everything/$basearch/os', f.read())
+        # test new repos files created
+        self.assertEqual(len(repo_files), 3 + 1 + len(cfg['extra_source_repos']))
+
+    @mock.patch('pungi.wrappers.scm.get_dir_from_scm')
+    @mock.patch('pungi.wrappers.kojiwrapper.KojiWrapper')
+    def test_run_with_keep_original_source_repos(self, KojiWrapper, get_dir_from_scm):
+        get_dir_from_scm.side_effect = self._dummy_config_repo
+
+        koji = KojiWrapper.return_value
+        koji.run_runroot_cmd.side_effect = self._mock_runroot(0)
+
+        cfg = {
+            'source_repo_from': 'Everything',
+            'keep_original_sources': True,
+            'config_url': 'https://git.fedorahosted.org/git/fedora-atomic.git',
+            'config_branch': 'f24',
+            'treefile': 'fedora-atomic-docker-host.json',
+            'ostree_repo': self.repo
+        }
+
+        t = ostree.OSTreeThread(self.pool)
+
+        t.process((self.compose, self.compose.variants['Everything'], 'x86_64', cfg), 1)
+
+        treeconf_content = json.load(open(os.path.join(self.topdir,
+                                                       'work/ostree-1/config_repo',
+                                                       cfg['treefile'])))
+
+        # added 1 repo (Everything), have 4 (3 + 1) repos now
+        self.assertEqual(len(treeconf_content['repos']), 4)
+
 if __name__ == '__main__':
     unittest.main()