[ostree] Allow extra repos to get packages for composing OSTree repository

Sometimes addtional repos are required to get necessary packages for
composing OSTree repository. For example, RHEL doesn't have an 'Everyting'
variant, so composing OSTree repository from any of the RHEL variants
won't work, addtional source repos need to be enabled to achieve that.
The new option "extra_source_repos" enable the ability of allowing extra
source repos.

And a new option 'keep_original_sources' is introduced to keep the
original repos found in tree config file, if this is enabled, Pungi
will not remove the existing source repos from the tree config file,
just add new repos of "source_repo_from" + "extra_source_repos" to
the existing repos.

Signed-off-by: Qixiang Wan <qwan@redhat.com>

doc/configuration.rst

@@ -1087,6 +1087,19 @@ a new commit.
 
     These keys are optional:
 
+    * ``extra_source_repos`` -- (*[dict]*) Extra source repos to get packages
+      while composing the OSTree repository. Each dict represents a yum repo.
+      The allowed keys are:
+
+      * ``name`` (required)
+      * ``baseurl`` (required) -- URL of external repo or variant UID, in the case
+        of variant UID, url to variant repo will be built automatically.
+      * ``gpgcheck`` (optional)
+      * ``exclude`` (optional)
+
+    * ``keep_original_sources`` -- (*bool*) Keep the existing source repos in
+      the tree config file. If not enabled, all the original source repos will
+      be removed from the tree config file.
     * ``config_branch`` -- (*str*) Git branch of the repo to use. Defaults to
       ``master``.
     * ``failable`` -- (*[str]*) List of architectures for which this
@@ -1107,7 +1120,20 @@ Example config
             "x86_64": {
                 "treefile": "fedora-atomic-docker-host.json",
                 "config_url": "https://git.fedorahosted.org/git/fedora-atomic.git",
-                "source_repo_from": "Everything",
+                "source_repo_from": "Server",
+                "extra_source_repos": [
+                    {
+                        "name": "repo_a",
+                        "baseurl": "http://example.com/repo/x86_64/os",
+                        "exclude": "systemd-container",
+                        "gpgcheck": False
+                    },
+                    {
+                        "name": "Everything",
+                        "baseurl": "Everything",
+                    }
+                ],
+                "keep_original_sources": True,
                 "ostree_repo": "/mnt/koji/compose/atomic/Rawhide/",
                 "update_summary": True,
                 "version": "24"

pungi/checks.py

@@ -318,11 +318,27 @@ def _make_schema():
                 ]
             },
 
+            "source_repo_dict": {
+                "type": "object",
+                "properties": {
+                    "name": {"type": "string"},
+                    "baseurl": {"type": "string"},
+                    "exclude": {"type": "string"},
+                    "gpgcheck": {"type": "boolean"},
+                },
+                "additionalProperties": False,
+            },
+
             "list_of_strings": {
                 "type": "array",
                 "items": {"type": "string"},
             },
 
+            "list_of_source_repo_dicts": {
+                "type": "array",
+                "items": {"$ref": "#/definitions/source_repo_dict"},
+            },
+
             "strings": {
                 "anyOf": [
                     {"type": "string"},
@@ -675,6 +691,8 @@ def _make_schema():
                     "treefile": {"type": "string"},
                     "config_url": {"type": "string"},
                     "source_repo_from": {"type": "string"},
+                    "extra_source_repos": {"$ref": "#/definitions/list_of_source_repo_dicts"},
+                    "keep_original_sources": {"type": "boolean"},
                     "ostree_repo": {"type": "string"},
                     "failable": {"$ref": "#/definitions/list_of_strings"},
                     "update_summary": {"type": "boolean"},

pungi/phases/ostree.py

@@ -1,9 +1,9 @@
 # -*- coding: utf-8 -*-
 
+import datetime
 import json
 import os
 from kobo.threads import ThreadPool, WorkerThread
-import re
 
 from .base import ConfigGuardedPhase
 from .. import util
@@ -52,7 +52,29 @@ class OSTreeThread(WorkerThread):
                                                                       create_dir=False))
 
         self._clone_repo(repodir, config['config_url'], config.get('config_branch', 'master'))
-        self._tweak_mirrorlist(repodir, source_repo)
+
+        treeconf = os.path.join(repodir, config['treefile'])
+        source_repos = [{'name': '%s-%s' % (compose.compose_id, config['source_repo_from']),
+                         'baseurl': source_repo}]
+
+        extra_source_repos = config.get('extra_source_repos', None)
+        if extra_source_repos:
+            for extra in extra_source_repos:
+                baseurl = extra['baseurl']
+                if "://" not in baseurl:
+                    # it's variant UID, translate to url
+                    variant = compose.variants[baseurl]
+                    url = translate_path(compose,
+                                         compose.paths.compose.repository('$basearch',
+                                                                          variant,
+                                                                          create_dir=False))
+                    extra['baseurl'] = url
+
+            source_repos = source_repos + extra_source_repos
+
+        keep_original_sources = config.get('keep_original_sources', False)
+        self._tweak_treeconf(treeconf, source_repos=source_repos,
+                             keep_original_sources=keep_original_sources)
 
         # Ensure target directory exists, otherwise Koji task will fail to
         # mount it.
@@ -136,23 +158,42 @@ class OSTreeThread(WorkerThread):
         scm.get_dir_from_scm({'scm': 'git', 'repo': url, 'branch': branch, 'dir': '.'},
                              repodir, logger=self.pool._logger)
 
-    def _tweak_mirrorlist(self, repodir, source_repo):
+    def _tweak_treeconf(self, treeconf, source_repos, keep_original_sources=False):
-        for file in os.listdir(repodir):
+        """
-            if file.endswith('.repo'):
+        Update tree config file by adding new repos and remove existing repos
-                tweak_file(os.path.join(repodir, file), source_repo)
+        from the tree config file if 'keep_original_sources' is not enabled.
+        """
+        # add this timestamp to repo name to get unique repo filename and repo name
+        # should be safe enough
+        time = datetime.datetime.now().strftime("%Y%m%d%H%M%S")
 
+        treeconf_dir = os.path.dirname(treeconf)
+        with open(treeconf, 'r') as f:
+            treeconf_content = json.load(f)
 
-def tweak_file(path, source_repo):
+        # backup the old tree config
-    """
+        os.rename(treeconf, '%s.%s.bak' % (treeconf, time))
-    Ensure a given .repo file points to `source_repo`.
 
-    This function replaces all lines starting with `mirrorlist`, `metalink` or
+        repos = []
-    `baseurl` with `baseurl` set to requested repository.
+        for repo in source_repos:
-    """
+            name = "%s-%s" % (repo['name'], time)
-    with open(path, 'r') as f:
+            with open("%s/%s.repo" % (treeconf_dir, name), 'w') as f:
-        contents = f.read()
+                f.write("[%s]\n" % name)
-    replacement = 'baseurl=%s' % source_repo
+                f.write("name=%s\n" % name)
-    exp = re.compile(r'^(mirrorlist|metalink|baseurl)=.*$', re.MULTILINE)
+                f.write("baseurl=%s\n" % repo['baseurl'])
-    contents = exp.sub(replacement, contents)
+                exclude = repo.get('exclude', None)
-    with open(path, 'w') as f:
+                if exclude:
-        f.write(contents)
+                    f.write("exclude=%s\n" % exclude)
+                gpgcheck = '1' if repo.get('gpgcheck', False) else '0'
+                f.write("gpgcheck=%s\n" % gpgcheck)
+            repos.append(name)
+
+        original_repos = treeconf_content.get('repos', [])
+        if keep_original_sources:
+            treeconf_content['repos'] = original_repos + repos
+        else:
+            treeconf_content['repos'] = repos
+
+        # update tree config to add new repos
+        with open(treeconf, 'w') as f:
+            json.dump(treeconf_content, f, indent=4)

tests/test_ostree_phase.py

@@ -69,13 +69,14 @@ class OSTreeThreadTest(helpers.PungiTestCase):
     def _dummy_config_repo(self, scm_dict, target, logger=None):
         os.makedirs(target)
         helpers.touch(os.path.join(target, 'fedora-atomic-docker-host.json'),
-                      json.dumps({'ref': 'fedora-atomic/25/x86_64'}))
+                      json.dumps({'ref': 'fedora-atomic/25/x86_64',
+                                  'repos': ['fedora-rawhide', 'fedora-24', 'fedora-23']}))
         helpers.touch(os.path.join(target, 'fedora-rawhide.repo'),
-                      'mirrorlist=mirror-mirror-on-the-wall')
+                      '[fedora-rawhide]\nmirrorlist=mirror-mirror-on-the-wall')
         helpers.touch(os.path.join(target, 'fedora-24.repo'),
-                      'metalink=who-is-the-fairest-of-them-all')
+                      '[fedora-24]\nmetalink=who-is-the-fairest-of-them-all')
         helpers.touch(os.path.join(target, 'fedora-23.repo'),
-                      'baseurl=why-not-zoidberg?')
+                      '[fedora-23]\nbaseurl=why-not-zoidberg?')
 
     def _mock_runroot(self, retcode, writefiles=None):
         """Pretend to run a task in runroot, creating a log file with given line
@@ -120,10 +121,17 @@ class OSTreeThreadTest(helpers.PungiTestCase):
                          [mock.call(koji.get_runroot_cmd.return_value,
                                     log_file=self.topdir + '/logs/x86_64/Everything/ostree-1/runroot.log')])
 
-        for fp in ['fedora-rawhide.repo', 'fedora-24.repo', 'fedora-24.repo']:
+        repo_files = []
-            with open(os.path.join(self.topdir, 'work/ostree-1/config_repo', fp)) as f:
+        for fp in os.listdir(os.path.join(self.topdir, 'work/ostree-1/config_repo')):
-                self.assertIn('baseurl=http://example.com/Everything/$basearch/os',
+            if fp.endswith('.repo'):
-                              f.read())
+                repo_files.append(fp)
+
+                if fp not in ['fedora-rawhide.repo', 'fedora-24.repo', 'fedora-23.repo']:
+                    with open(os.path.join(self.topdir, 'work/ostree-1/config_repo', fp)) as f:
+                        self.assertIn('baseurl=http://example.com/Everything/$basearch/os', f.read())
+        # test a new repo file created
+        self.assertEqual(len(repo_files), 4)
+
         self.assertTrue(os.path.isdir(self.repo))
 
     @mock.patch('pungi.wrappers.scm.get_dir_from_scm')
@@ -257,12 +265,6 @@ class OSTreeThreadTest(helpers.PungiTestCase):
                          [mock.call(koji.get_runroot_cmd.return_value,
                                     log_file=self.topdir + '/logs/x86_64/Everything/ostree-1/runroot.log')])
 
-        for fp in ['fedora-rawhide.repo', 'fedora-24.repo', 'fedora-24.repo']:
-            with open(os.path.join(self.topdir, 'work/ostree-1/config_repo', fp)) as f:
-                self.assertIn('baseurl=http://example.com/Everything/$basearch/os',
-                              f.read())
-        self.assertTrue(os.path.isdir(self.repo))
-
     @mock.patch('pungi.wrappers.scm.get_dir_from_scm')
     @mock.patch('pungi.wrappers.kojiwrapper.KojiWrapper')
     def test_run_with_versioning_metadata(self, KojiWrapper, get_dir_from_scm):
@@ -295,5 +297,94 @@ class OSTreeThreadTest(helpers.PungiTestCase):
                          [mock.call(koji.get_runroot_cmd.return_value,
                                     log_file=self.topdir + '/logs/x86_64/Everything/ostree-1/runroot.log')])
 
+    @mock.patch('pungi.wrappers.scm.get_dir_from_scm')
+    @mock.patch('pungi.wrappers.kojiwrapper.KojiWrapper')
+    def test_run_with_extra_source_repos(self, KojiWrapper, get_dir_from_scm):
+        get_dir_from_scm.side_effect = self._dummy_config_repo
+
+        koji = KojiWrapper.return_value
+        koji.run_runroot_cmd.side_effect = self._mock_runroot(0)
+
+        cfg = {
+            'source_repo_from': 'Everything',
+            'extra_source_repos': [
+                {
+                    'name': 'repo_a',
+                    'baseurl': 'http://url/to/repo/a',
+                    'exclude': 'systemd-container'
+                },
+                {
+                    'name': 'Server',
+                    'baseurl': 'Server',
+                    'exclude': 'systemd-container'
+                }
+            ],
+            'config_url': 'https://git.fedorahosted.org/git/fedora-atomic.git',
+            'config_branch': 'f24',
+            'treefile': 'fedora-atomic-docker-host.json',
+            'ostree_repo': self.repo
+        }
+
+        t = ostree.OSTreeThread(self.pool)
+
+        t.process((self.compose, self.compose.variants['Everything'], 'x86_64', cfg), 1)
+
+        repo_files = []
+        for fp in os.listdir(os.path.join(self.topdir, 'work/ostree-1/config_repo')):
+            if fp.endswith('.repo'):
+                repo_files.append(fp)
+
+                if fp not in ['fedora-rawhide.repo', 'fedora-24.repo', 'fedora-23.repo']:
+                    if fp.startswith('repo_a'):
+                        with open(os.path.join(self.topdir, 'work/ostree-1/config_repo', fp)) as f:
+                            # ignore timestamp in repo name while checking
+                            content = f.read()
+                            self.assertIn('[repo_a', content)
+                            self.assertIn('name=repo_a', content)
+                            self.assertIn('baseurl=http://url/to/repo/a', content)
+                            self.assertIn('exclude=systemd-container', content)
+                            self.assertIn('gpgcheck=0', content)
+                    elif fp.startswith('Server'):
+                        with open(os.path.join(self.topdir, 'work/ostree-1/config_repo', fp)) as f:
+                            content = f.read()
+                            self.assertIn('[Server', content)
+                            self.assertIn('baseurl=http://example.com/Server/$basearch/os', content)
+                            self.assertIn('exclude=systemd-container', content)
+                            self.assertIn('gpgcheck=0', content)
+                    else:
+                        # this is the Everything repo (source_repo_from)
+                        with open(os.path.join(self.topdir, 'work/ostree-1/config_repo', fp)) as f:
+                            self.assertIn('baseurl=http://example.com/Everything/$basearch/os', f.read())
+        # test new repos files created
+        self.assertEqual(len(repo_files), 3 + 1 + len(cfg['extra_source_repos']))
+
+    @mock.patch('pungi.wrappers.scm.get_dir_from_scm')
+    @mock.patch('pungi.wrappers.kojiwrapper.KojiWrapper')
+    def test_run_with_keep_original_source_repos(self, KojiWrapper, get_dir_from_scm):
+        get_dir_from_scm.side_effect = self._dummy_config_repo
+
+        koji = KojiWrapper.return_value
+        koji.run_runroot_cmd.side_effect = self._mock_runroot(0)
+
+        cfg = {
+            'source_repo_from': 'Everything',
+            'keep_original_sources': True,
+            'config_url': 'https://git.fedorahosted.org/git/fedora-atomic.git',
+            'config_branch': 'f24',
+            'treefile': 'fedora-atomic-docker-host.json',
+            'ostree_repo': self.repo
+        }
+
+        t = ostree.OSTreeThread(self.pool)
+
+        t.process((self.compose, self.compose.variants['Everything'], 'x86_64', cfg), 1)
+
+        treeconf_content = json.load(open(os.path.join(self.topdir,
+                                                       'work/ostree-1/config_repo',
+                                                       cfg['treefile'])))
+
+        # added 1 repo (Everything), have 4 (3 + 1) repos now
+        self.assertEqual(len(treeconf_content['repos']), 4)
+
 if __name__ == '__main__':
     unittest.main()