diff --git a/doc/configuration.rst b/doc/configuration.rst index ed318207..e22ae445 100644 --- a/doc/configuration.rst +++ b/doc/configuration.rst @@ -1087,6 +1087,19 @@ a new commit. These keys are optional: + * ``extra_source_repos`` -- (*[dict]*) Extra source repos to get packages + while composing the OSTree repository. Each dict represents a yum repo. + The allowed keys are: + + * ``name`` (required) + * ``baseurl`` (required) -- URL of external repo or variant UID, in the case + of variant UID, url to variant repo will be built automatically. + * ``gpgcheck`` (optional) + * ``exclude`` (optional) + + * ``keep_original_sources`` -- (*bool*) Keep the existing source repos in + the tree config file. If not enabled, all the original source repos will + be removed from the tree config file. * ``config_branch`` -- (*str*) Git branch of the repo to use. Defaults to ``master``. * ``failable`` -- (*[str]*) List of architectures for which this @@ -1107,7 +1120,20 @@ Example config "x86_64": { "treefile": "fedora-atomic-docker-host.json", "config_url": "https://git.fedorahosted.org/git/fedora-atomic.git", - "source_repo_from": "Everything", + "source_repo_from": "Server", + "extra_source_repos": [ + { + "name": "repo_a", + "baseurl": "http://example.com/repo/x86_64/os", + "exclude": "systemd-container", + "gpgcheck": False + }, + { + "name": "Everything", + "baseurl": "Everything", + } + ], + "keep_original_sources": True, "ostree_repo": "/mnt/koji/compose/atomic/Rawhide/", "update_summary": True, "version": "24" diff --git a/pungi/checks.py b/pungi/checks.py index f3f0c2bf..edf70202 100644 --- a/pungi/checks.py +++ b/pungi/checks.py @@ -318,11 +318,27 @@ def _make_schema(): ] }, + "source_repo_dict": { + "type": "object", + "properties": { + "name": {"type": "string"}, + "baseurl": {"type": "string"}, + "exclude": {"type": "string"}, + "gpgcheck": {"type": "boolean"}, + }, + "additionalProperties": False, + }, + "list_of_strings": { "type": "array", "items": {"type": "string"}, }, + "list_of_source_repo_dicts": { + "type": "array", + "items": {"$ref": "#/definitions/source_repo_dict"}, + }, + "strings": { "anyOf": [ {"type": "string"}, @@ -675,6 +691,8 @@ def _make_schema(): "treefile": {"type": "string"}, "config_url": {"type": "string"}, "source_repo_from": {"type": "string"}, + "extra_source_repos": {"$ref": "#/definitions/list_of_source_repo_dicts"}, + "keep_original_sources": {"type": "boolean"}, "ostree_repo": {"type": "string"}, "failable": {"$ref": "#/definitions/list_of_strings"}, "update_summary": {"type": "boolean"}, diff --git a/pungi/phases/ostree.py b/pungi/phases/ostree.py index 4a52f2ff..b9c969d2 100644 --- a/pungi/phases/ostree.py +++ b/pungi/phases/ostree.py @@ -1,9 +1,9 @@ # -*- coding: utf-8 -*- +import datetime import json import os from kobo.threads import ThreadPool, WorkerThread -import re from .base import ConfigGuardedPhase from .. import util @@ -52,7 +52,29 @@ class OSTreeThread(WorkerThread): create_dir=False)) self._clone_repo(repodir, config['config_url'], config.get('config_branch', 'master')) - self._tweak_mirrorlist(repodir, source_repo) + + treeconf = os.path.join(repodir, config['treefile']) + source_repos = [{'name': '%s-%s' % (compose.compose_id, config['source_repo_from']), + 'baseurl': source_repo}] + + extra_source_repos = config.get('extra_source_repos', None) + if extra_source_repos: + for extra in extra_source_repos: + baseurl = extra['baseurl'] + if "://" not in baseurl: + # it's variant UID, translate to url + variant = compose.variants[baseurl] + url = translate_path(compose, + compose.paths.compose.repository('$basearch', + variant, + create_dir=False)) + extra['baseurl'] = url + + source_repos = source_repos + extra_source_repos + + keep_original_sources = config.get('keep_original_sources', False) + self._tweak_treeconf(treeconf, source_repos=source_repos, + keep_original_sources=keep_original_sources) # Ensure target directory exists, otherwise Koji task will fail to # mount it. @@ -136,23 +158,42 @@ class OSTreeThread(WorkerThread): scm.get_dir_from_scm({'scm': 'git', 'repo': url, 'branch': branch, 'dir': '.'}, repodir, logger=self.pool._logger) - def _tweak_mirrorlist(self, repodir, source_repo): - for file in os.listdir(repodir): - if file.endswith('.repo'): - tweak_file(os.path.join(repodir, file), source_repo) + def _tweak_treeconf(self, treeconf, source_repos, keep_original_sources=False): + """ + Update tree config file by adding new repos and remove existing repos + from the tree config file if 'keep_original_sources' is not enabled. + """ + # add this timestamp to repo name to get unique repo filename and repo name + # should be safe enough + time = datetime.datetime.now().strftime("%Y%m%d%H%M%S") + treeconf_dir = os.path.dirname(treeconf) + with open(treeconf, 'r') as f: + treeconf_content = json.load(f) -def tweak_file(path, source_repo): - """ - Ensure a given .repo file points to `source_repo`. + # backup the old tree config + os.rename(treeconf, '%s.%s.bak' % (treeconf, time)) - This function replaces all lines starting with `mirrorlist`, `metalink` or - `baseurl` with `baseurl` set to requested repository. - """ - with open(path, 'r') as f: - contents = f.read() - replacement = 'baseurl=%s' % source_repo - exp = re.compile(r'^(mirrorlist|metalink|baseurl)=.*$', re.MULTILINE) - contents = exp.sub(replacement, contents) - with open(path, 'w') as f: - f.write(contents) + repos = [] + for repo in source_repos: + name = "%s-%s" % (repo['name'], time) + with open("%s/%s.repo" % (treeconf_dir, name), 'w') as f: + f.write("[%s]\n" % name) + f.write("name=%s\n" % name) + f.write("baseurl=%s\n" % repo['baseurl']) + exclude = repo.get('exclude', None) + if exclude: + f.write("exclude=%s\n" % exclude) + gpgcheck = '1' if repo.get('gpgcheck', False) else '0' + f.write("gpgcheck=%s\n" % gpgcheck) + repos.append(name) + + original_repos = treeconf_content.get('repos', []) + if keep_original_sources: + treeconf_content['repos'] = original_repos + repos + else: + treeconf_content['repos'] = repos + + # update tree config to add new repos + with open(treeconf, 'w') as f: + json.dump(treeconf_content, f, indent=4) diff --git a/tests/test_ostree_phase.py b/tests/test_ostree_phase.py index 7d3a8389..0ef10194 100644 --- a/tests/test_ostree_phase.py +++ b/tests/test_ostree_phase.py @@ -69,13 +69,14 @@ class OSTreeThreadTest(helpers.PungiTestCase): def _dummy_config_repo(self, scm_dict, target, logger=None): os.makedirs(target) helpers.touch(os.path.join(target, 'fedora-atomic-docker-host.json'), - json.dumps({'ref': 'fedora-atomic/25/x86_64'})) + json.dumps({'ref': 'fedora-atomic/25/x86_64', + 'repos': ['fedora-rawhide', 'fedora-24', 'fedora-23']})) helpers.touch(os.path.join(target, 'fedora-rawhide.repo'), - 'mirrorlist=mirror-mirror-on-the-wall') + '[fedora-rawhide]\nmirrorlist=mirror-mirror-on-the-wall') helpers.touch(os.path.join(target, 'fedora-24.repo'), - 'metalink=who-is-the-fairest-of-them-all') + '[fedora-24]\nmetalink=who-is-the-fairest-of-them-all') helpers.touch(os.path.join(target, 'fedora-23.repo'), - 'baseurl=why-not-zoidberg?') + '[fedora-23]\nbaseurl=why-not-zoidberg?') def _mock_runroot(self, retcode, writefiles=None): """Pretend to run a task in runroot, creating a log file with given line @@ -120,10 +121,17 @@ class OSTreeThreadTest(helpers.PungiTestCase): [mock.call(koji.get_runroot_cmd.return_value, log_file=self.topdir + '/logs/x86_64/Everything/ostree-1/runroot.log')]) - for fp in ['fedora-rawhide.repo', 'fedora-24.repo', 'fedora-24.repo']: - with open(os.path.join(self.topdir, 'work/ostree-1/config_repo', fp)) as f: - self.assertIn('baseurl=http://example.com/Everything/$basearch/os', - f.read()) + repo_files = [] + for fp in os.listdir(os.path.join(self.topdir, 'work/ostree-1/config_repo')): + if fp.endswith('.repo'): + repo_files.append(fp) + + if fp not in ['fedora-rawhide.repo', 'fedora-24.repo', 'fedora-23.repo']: + with open(os.path.join(self.topdir, 'work/ostree-1/config_repo', fp)) as f: + self.assertIn('baseurl=http://example.com/Everything/$basearch/os', f.read()) + # test a new repo file created + self.assertEqual(len(repo_files), 4) + self.assertTrue(os.path.isdir(self.repo)) @mock.patch('pungi.wrappers.scm.get_dir_from_scm') @@ -257,12 +265,6 @@ class OSTreeThreadTest(helpers.PungiTestCase): [mock.call(koji.get_runroot_cmd.return_value, log_file=self.topdir + '/logs/x86_64/Everything/ostree-1/runroot.log')]) - for fp in ['fedora-rawhide.repo', 'fedora-24.repo', 'fedora-24.repo']: - with open(os.path.join(self.topdir, 'work/ostree-1/config_repo', fp)) as f: - self.assertIn('baseurl=http://example.com/Everything/$basearch/os', - f.read()) - self.assertTrue(os.path.isdir(self.repo)) - @mock.patch('pungi.wrappers.scm.get_dir_from_scm') @mock.patch('pungi.wrappers.kojiwrapper.KojiWrapper') def test_run_with_versioning_metadata(self, KojiWrapper, get_dir_from_scm): @@ -295,5 +297,94 @@ class OSTreeThreadTest(helpers.PungiTestCase): [mock.call(koji.get_runroot_cmd.return_value, log_file=self.topdir + '/logs/x86_64/Everything/ostree-1/runroot.log')]) + @mock.patch('pungi.wrappers.scm.get_dir_from_scm') + @mock.patch('pungi.wrappers.kojiwrapper.KojiWrapper') + def test_run_with_extra_source_repos(self, KojiWrapper, get_dir_from_scm): + get_dir_from_scm.side_effect = self._dummy_config_repo + + koji = KojiWrapper.return_value + koji.run_runroot_cmd.side_effect = self._mock_runroot(0) + + cfg = { + 'source_repo_from': 'Everything', + 'extra_source_repos': [ + { + 'name': 'repo_a', + 'baseurl': 'http://url/to/repo/a', + 'exclude': 'systemd-container' + }, + { + 'name': 'Server', + 'baseurl': 'Server', + 'exclude': 'systemd-container' + } + ], + 'config_url': 'https://git.fedorahosted.org/git/fedora-atomic.git', + 'config_branch': 'f24', + 'treefile': 'fedora-atomic-docker-host.json', + 'ostree_repo': self.repo + } + + t = ostree.OSTreeThread(self.pool) + + t.process((self.compose, self.compose.variants['Everything'], 'x86_64', cfg), 1) + + repo_files = [] + for fp in os.listdir(os.path.join(self.topdir, 'work/ostree-1/config_repo')): + if fp.endswith('.repo'): + repo_files.append(fp) + + if fp not in ['fedora-rawhide.repo', 'fedora-24.repo', 'fedora-23.repo']: + if fp.startswith('repo_a'): + with open(os.path.join(self.topdir, 'work/ostree-1/config_repo', fp)) as f: + # ignore timestamp in repo name while checking + content = f.read() + self.assertIn('[repo_a', content) + self.assertIn('name=repo_a', content) + self.assertIn('baseurl=http://url/to/repo/a', content) + self.assertIn('exclude=systemd-container', content) + self.assertIn('gpgcheck=0', content) + elif fp.startswith('Server'): + with open(os.path.join(self.topdir, 'work/ostree-1/config_repo', fp)) as f: + content = f.read() + self.assertIn('[Server', content) + self.assertIn('baseurl=http://example.com/Server/$basearch/os', content) + self.assertIn('exclude=systemd-container', content) + self.assertIn('gpgcheck=0', content) + else: + # this is the Everything repo (source_repo_from) + with open(os.path.join(self.topdir, 'work/ostree-1/config_repo', fp)) as f: + self.assertIn('baseurl=http://example.com/Everything/$basearch/os', f.read()) + # test new repos files created + self.assertEqual(len(repo_files), 3 + 1 + len(cfg['extra_source_repos'])) + + @mock.patch('pungi.wrappers.scm.get_dir_from_scm') + @mock.patch('pungi.wrappers.kojiwrapper.KojiWrapper') + def test_run_with_keep_original_source_repos(self, KojiWrapper, get_dir_from_scm): + get_dir_from_scm.side_effect = self._dummy_config_repo + + koji = KojiWrapper.return_value + koji.run_runroot_cmd.side_effect = self._mock_runroot(0) + + cfg = { + 'source_repo_from': 'Everything', + 'keep_original_sources': True, + 'config_url': 'https://git.fedorahosted.org/git/fedora-atomic.git', + 'config_branch': 'f24', + 'treefile': 'fedora-atomic-docker-host.json', + 'ostree_repo': self.repo + } + + t = ostree.OSTreeThread(self.pool) + + t.process((self.compose, self.compose.variants['Everything'], 'x86_64', cfg), 1) + + treeconf_content = json.load(open(os.path.join(self.topdir, + 'work/ostree-1/config_repo', + cfg['treefile']))) + + # added 1 repo (Everything), have 4 (3 + 1) repos now + self.assertEqual(len(treeconf_content['repos']), 4) + if __name__ == '__main__': unittest.main()