From 68e121e421ca2cc6d0502fb86d3aa7bb3a5cb9bf Mon Sep 17 00:00:00 2001 From: Qixiang Wan Date: Mon, 7 Nov 2016 13:23:13 +0800 Subject: [PATCH] [ostree] Allow extra repos to get packages for composing OSTree repository Sometimes addtional repos are required to get necessary packages for composing OSTree repository. For example, RHEL doesn't have an 'Everyting' variant, so composing OSTree repository from any of the RHEL variants won't work, addtional source repos need to be enabled to achieve that. The new option "extra_source_repos" enable the ability of allowing extra source repos. And a new option 'keep_original_sources' is introduced to keep the original repos found in tree config file, if this is enabled, Pungi will not remove the existing source repos from the tree config file, just add new repos of "source_repo_from" + "extra_source_repos" to the existing repos. Signed-off-by: Qixiang Wan --- doc/configuration.rst | 28 ++++++++- pungi/checks.py | 18 ++++++ pungi/phases/ostree.py | 79 ++++++++++++++++++------ tests/test_ostree_phase.py | 119 ++++++++++++++++++++++++++++++++----- 4 files changed, 210 insertions(+), 34 deletions(-) diff --git a/doc/configuration.rst b/doc/configuration.rst index ed318207..e22ae445 100644 --- a/doc/configuration.rst +++ b/doc/configuration.rst @@ -1087,6 +1087,19 @@ a new commit. These keys are optional: + * ``extra_source_repos`` -- (*[dict]*) Extra source repos to get packages + while composing the OSTree repository. Each dict represents a yum repo. + The allowed keys are: + + * ``name`` (required) + * ``baseurl`` (required) -- URL of external repo or variant UID, in the case + of variant UID, url to variant repo will be built automatically. + * ``gpgcheck`` (optional) + * ``exclude`` (optional) + + * ``keep_original_sources`` -- (*bool*) Keep the existing source repos in + the tree config file. If not enabled, all the original source repos will + be removed from the tree config file. * ``config_branch`` -- (*str*) Git branch of the repo to use. Defaults to ``master``. * ``failable`` -- (*[str]*) List of architectures for which this @@ -1107,7 +1120,20 @@ Example config "x86_64": { "treefile": "fedora-atomic-docker-host.json", "config_url": "https://git.fedorahosted.org/git/fedora-atomic.git", - "source_repo_from": "Everything", + "source_repo_from": "Server", + "extra_source_repos": [ + { + "name": "repo_a", + "baseurl": "http://example.com/repo/x86_64/os", + "exclude": "systemd-container", + "gpgcheck": False + }, + { + "name": "Everything", + "baseurl": "Everything", + } + ], + "keep_original_sources": True, "ostree_repo": "/mnt/koji/compose/atomic/Rawhide/", "update_summary": True, "version": "24" diff --git a/pungi/checks.py b/pungi/checks.py index f3f0c2bf..edf70202 100644 --- a/pungi/checks.py +++ b/pungi/checks.py @@ -318,11 +318,27 @@ def _make_schema(): ] }, + "source_repo_dict": { + "type": "object", + "properties": { + "name": {"type": "string"}, + "baseurl": {"type": "string"}, + "exclude": {"type": "string"}, + "gpgcheck": {"type": "boolean"}, + }, + "additionalProperties": False, + }, + "list_of_strings": { "type": "array", "items": {"type": "string"}, }, + "list_of_source_repo_dicts": { + "type": "array", + "items": {"$ref": "#/definitions/source_repo_dict"}, + }, + "strings": { "anyOf": [ {"type": "string"}, @@ -675,6 +691,8 @@ def _make_schema(): "treefile": {"type": "string"}, "config_url": {"type": "string"}, "source_repo_from": {"type": "string"}, + "extra_source_repos": {"$ref": "#/definitions/list_of_source_repo_dicts"}, + "keep_original_sources": {"type": "boolean"}, "ostree_repo": {"type": "string"}, "failable": {"$ref": "#/definitions/list_of_strings"}, "update_summary": {"type": "boolean"}, diff --git a/pungi/phases/ostree.py b/pungi/phases/ostree.py index 4a52f2ff..b9c969d2 100644 --- a/pungi/phases/ostree.py +++ b/pungi/phases/ostree.py @@ -1,9 +1,9 @@ # -*- coding: utf-8 -*- +import datetime import json import os from kobo.threads import ThreadPool, WorkerThread -import re from .base import ConfigGuardedPhase from .. import util @@ -52,7 +52,29 @@ class OSTreeThread(WorkerThread): create_dir=False)) self._clone_repo(repodir, config['config_url'], config.get('config_branch', 'master')) - self._tweak_mirrorlist(repodir, source_repo) + + treeconf = os.path.join(repodir, config['treefile']) + source_repos = [{'name': '%s-%s' % (compose.compose_id, config['source_repo_from']), + 'baseurl': source_repo}] + + extra_source_repos = config.get('extra_source_repos', None) + if extra_source_repos: + for extra in extra_source_repos: + baseurl = extra['baseurl'] + if "://" not in baseurl: + # it's variant UID, translate to url + variant = compose.variants[baseurl] + url = translate_path(compose, + compose.paths.compose.repository('$basearch', + variant, + create_dir=False)) + extra['baseurl'] = url + + source_repos = source_repos + extra_source_repos + + keep_original_sources = config.get('keep_original_sources', False) + self._tweak_treeconf(treeconf, source_repos=source_repos, + keep_original_sources=keep_original_sources) # Ensure target directory exists, otherwise Koji task will fail to # mount it. @@ -136,23 +158,42 @@ class OSTreeThread(WorkerThread): scm.get_dir_from_scm({'scm': 'git', 'repo': url, 'branch': branch, 'dir': '.'}, repodir, logger=self.pool._logger) - def _tweak_mirrorlist(self, repodir, source_repo): - for file in os.listdir(repodir): - if file.endswith('.repo'): - tweak_file(os.path.join(repodir, file), source_repo) + def _tweak_treeconf(self, treeconf, source_repos, keep_original_sources=False): + """ + Update tree config file by adding new repos and remove existing repos + from the tree config file if 'keep_original_sources' is not enabled. + """ + # add this timestamp to repo name to get unique repo filename and repo name + # should be safe enough + time = datetime.datetime.now().strftime("%Y%m%d%H%M%S") + treeconf_dir = os.path.dirname(treeconf) + with open(treeconf, 'r') as f: + treeconf_content = json.load(f) -def tweak_file(path, source_repo): - """ - Ensure a given .repo file points to `source_repo`. + # backup the old tree config + os.rename(treeconf, '%s.%s.bak' % (treeconf, time)) - This function replaces all lines starting with `mirrorlist`, `metalink` or - `baseurl` with `baseurl` set to requested repository. - """ - with open(path, 'r') as f: - contents = f.read() - replacement = 'baseurl=%s' % source_repo - exp = re.compile(r'^(mirrorlist|metalink|baseurl)=.*$', re.MULTILINE) - contents = exp.sub(replacement, contents) - with open(path, 'w') as f: - f.write(contents) + repos = [] + for repo in source_repos: + name = "%s-%s" % (repo['name'], time) + with open("%s/%s.repo" % (treeconf_dir, name), 'w') as f: + f.write("[%s]\n" % name) + f.write("name=%s\n" % name) + f.write("baseurl=%s\n" % repo['baseurl']) + exclude = repo.get('exclude', None) + if exclude: + f.write("exclude=%s\n" % exclude) + gpgcheck = '1' if repo.get('gpgcheck', False) else '0' + f.write("gpgcheck=%s\n" % gpgcheck) + repos.append(name) + + original_repos = treeconf_content.get('repos', []) + if keep_original_sources: + treeconf_content['repos'] = original_repos + repos + else: + treeconf_content['repos'] = repos + + # update tree config to add new repos + with open(treeconf, 'w') as f: + json.dump(treeconf_content, f, indent=4) diff --git a/tests/test_ostree_phase.py b/tests/test_ostree_phase.py index 7d3a8389..0ef10194 100644 --- a/tests/test_ostree_phase.py +++ b/tests/test_ostree_phase.py @@ -69,13 +69,14 @@ class OSTreeThreadTest(helpers.PungiTestCase): def _dummy_config_repo(self, scm_dict, target, logger=None): os.makedirs(target) helpers.touch(os.path.join(target, 'fedora-atomic-docker-host.json'), - json.dumps({'ref': 'fedora-atomic/25/x86_64'})) + json.dumps({'ref': 'fedora-atomic/25/x86_64', + 'repos': ['fedora-rawhide', 'fedora-24', 'fedora-23']})) helpers.touch(os.path.join(target, 'fedora-rawhide.repo'), - 'mirrorlist=mirror-mirror-on-the-wall') + '[fedora-rawhide]\nmirrorlist=mirror-mirror-on-the-wall') helpers.touch(os.path.join(target, 'fedora-24.repo'), - 'metalink=who-is-the-fairest-of-them-all') + '[fedora-24]\nmetalink=who-is-the-fairest-of-them-all') helpers.touch(os.path.join(target, 'fedora-23.repo'), - 'baseurl=why-not-zoidberg?') + '[fedora-23]\nbaseurl=why-not-zoidberg?') def _mock_runroot(self, retcode, writefiles=None): """Pretend to run a task in runroot, creating a log file with given line @@ -120,10 +121,17 @@ class OSTreeThreadTest(helpers.PungiTestCase): [mock.call(koji.get_runroot_cmd.return_value, log_file=self.topdir + '/logs/x86_64/Everything/ostree-1/runroot.log')]) - for fp in ['fedora-rawhide.repo', 'fedora-24.repo', 'fedora-24.repo']: - with open(os.path.join(self.topdir, 'work/ostree-1/config_repo', fp)) as f: - self.assertIn('baseurl=http://example.com/Everything/$basearch/os', - f.read()) + repo_files = [] + for fp in os.listdir(os.path.join(self.topdir, 'work/ostree-1/config_repo')): + if fp.endswith('.repo'): + repo_files.append(fp) + + if fp not in ['fedora-rawhide.repo', 'fedora-24.repo', 'fedora-23.repo']: + with open(os.path.join(self.topdir, 'work/ostree-1/config_repo', fp)) as f: + self.assertIn('baseurl=http://example.com/Everything/$basearch/os', f.read()) + # test a new repo file created + self.assertEqual(len(repo_files), 4) + self.assertTrue(os.path.isdir(self.repo)) @mock.patch('pungi.wrappers.scm.get_dir_from_scm') @@ -257,12 +265,6 @@ class OSTreeThreadTest(helpers.PungiTestCase): [mock.call(koji.get_runroot_cmd.return_value, log_file=self.topdir + '/logs/x86_64/Everything/ostree-1/runroot.log')]) - for fp in ['fedora-rawhide.repo', 'fedora-24.repo', 'fedora-24.repo']: - with open(os.path.join(self.topdir, 'work/ostree-1/config_repo', fp)) as f: - self.assertIn('baseurl=http://example.com/Everything/$basearch/os', - f.read()) - self.assertTrue(os.path.isdir(self.repo)) - @mock.patch('pungi.wrappers.scm.get_dir_from_scm') @mock.patch('pungi.wrappers.kojiwrapper.KojiWrapper') def test_run_with_versioning_metadata(self, KojiWrapper, get_dir_from_scm): @@ -295,5 +297,94 @@ class OSTreeThreadTest(helpers.PungiTestCase): [mock.call(koji.get_runroot_cmd.return_value, log_file=self.topdir + '/logs/x86_64/Everything/ostree-1/runroot.log')]) + @mock.patch('pungi.wrappers.scm.get_dir_from_scm') + @mock.patch('pungi.wrappers.kojiwrapper.KojiWrapper') + def test_run_with_extra_source_repos(self, KojiWrapper, get_dir_from_scm): + get_dir_from_scm.side_effect = self._dummy_config_repo + + koji = KojiWrapper.return_value + koji.run_runroot_cmd.side_effect = self._mock_runroot(0) + + cfg = { + 'source_repo_from': 'Everything', + 'extra_source_repos': [ + { + 'name': 'repo_a', + 'baseurl': 'http://url/to/repo/a', + 'exclude': 'systemd-container' + }, + { + 'name': 'Server', + 'baseurl': 'Server', + 'exclude': 'systemd-container' + } + ], + 'config_url': 'https://git.fedorahosted.org/git/fedora-atomic.git', + 'config_branch': 'f24', + 'treefile': 'fedora-atomic-docker-host.json', + 'ostree_repo': self.repo + } + + t = ostree.OSTreeThread(self.pool) + + t.process((self.compose, self.compose.variants['Everything'], 'x86_64', cfg), 1) + + repo_files = [] + for fp in os.listdir(os.path.join(self.topdir, 'work/ostree-1/config_repo')): + if fp.endswith('.repo'): + repo_files.append(fp) + + if fp not in ['fedora-rawhide.repo', 'fedora-24.repo', 'fedora-23.repo']: + if fp.startswith('repo_a'): + with open(os.path.join(self.topdir, 'work/ostree-1/config_repo', fp)) as f: + # ignore timestamp in repo name while checking + content = f.read() + self.assertIn('[repo_a', content) + self.assertIn('name=repo_a', content) + self.assertIn('baseurl=http://url/to/repo/a', content) + self.assertIn('exclude=systemd-container', content) + self.assertIn('gpgcheck=0', content) + elif fp.startswith('Server'): + with open(os.path.join(self.topdir, 'work/ostree-1/config_repo', fp)) as f: + content = f.read() + self.assertIn('[Server', content) + self.assertIn('baseurl=http://example.com/Server/$basearch/os', content) + self.assertIn('exclude=systemd-container', content) + self.assertIn('gpgcheck=0', content) + else: + # this is the Everything repo (source_repo_from) + with open(os.path.join(self.topdir, 'work/ostree-1/config_repo', fp)) as f: + self.assertIn('baseurl=http://example.com/Everything/$basearch/os', f.read()) + # test new repos files created + self.assertEqual(len(repo_files), 3 + 1 + len(cfg['extra_source_repos'])) + + @mock.patch('pungi.wrappers.scm.get_dir_from_scm') + @mock.patch('pungi.wrappers.kojiwrapper.KojiWrapper') + def test_run_with_keep_original_source_repos(self, KojiWrapper, get_dir_from_scm): + get_dir_from_scm.side_effect = self._dummy_config_repo + + koji = KojiWrapper.return_value + koji.run_runroot_cmd.side_effect = self._mock_runroot(0) + + cfg = { + 'source_repo_from': 'Everything', + 'keep_original_sources': True, + 'config_url': 'https://git.fedorahosted.org/git/fedora-atomic.git', + 'config_branch': 'f24', + 'treefile': 'fedora-atomic-docker-host.json', + 'ostree_repo': self.repo + } + + t = ostree.OSTreeThread(self.pool) + + t.process((self.compose, self.compose.variants['Everything'], 'x86_64', cfg), 1) + + treeconf_content = json.load(open(os.path.join(self.topdir, + 'work/ostree-1/config_repo', + cfg['treefile']))) + + # added 1 repo (Everything), have 4 (3 + 1) repos now + self.assertEqual(len(treeconf_content['repos']), 4) + if __name__ == '__main__': unittest.main()