[ostree] Allow extra repos to get packages for composing OSTree repository

Sometimes addtional repos are required to get necessary packages for
composing OSTree repository. For example, RHEL doesn't have an 'Everyting'
variant, so composing OSTree repository from any of the RHEL variants
won't work, addtional source repos need to be enabled to achieve that.
The new option "extra_source_repos" enable the ability of allowing extra
source repos.

And a new option 'keep_original_sources' is introduced to keep the
original repos found in tree config file, if this is enabled, Pungi
will not remove the existing source repos from the tree config file,
just add new repos of "source_repo_from" + "extra_source_repos" to
the existing repos.

Signed-off-by: Qixiang Wan <qwan@redhat.com>
This commit is contained in:
Qixiang Wan 2016-11-07 13:23:13 +08:00
parent 4427769f6a
commit 68e121e421
4 changed files with 210 additions and 34 deletions

View File

@ -1087,6 +1087,19 @@ a new commit.
These keys are optional: These keys are optional:
* ``extra_source_repos`` -- (*[dict]*) Extra source repos to get packages
while composing the OSTree repository. Each dict represents a yum repo.
The allowed keys are:
* ``name`` (required)
* ``baseurl`` (required) -- URL of external repo or variant UID, in the case
of variant UID, url to variant repo will be built automatically.
* ``gpgcheck`` (optional)
* ``exclude`` (optional)
* ``keep_original_sources`` -- (*bool*) Keep the existing source repos in
the tree config file. If not enabled, all the original source repos will
be removed from the tree config file.
* ``config_branch`` -- (*str*) Git branch of the repo to use. Defaults to * ``config_branch`` -- (*str*) Git branch of the repo to use. Defaults to
``master``. ``master``.
* ``failable`` -- (*[str]*) List of architectures for which this * ``failable`` -- (*[str]*) List of architectures for which this
@ -1107,7 +1120,20 @@ Example config
"x86_64": { "x86_64": {
"treefile": "fedora-atomic-docker-host.json", "treefile": "fedora-atomic-docker-host.json",
"config_url": "https://git.fedorahosted.org/git/fedora-atomic.git", "config_url": "https://git.fedorahosted.org/git/fedora-atomic.git",
"source_repo_from": "Everything", "source_repo_from": "Server",
"extra_source_repos": [
{
"name": "repo_a",
"baseurl": "http://example.com/repo/x86_64/os",
"exclude": "systemd-container",
"gpgcheck": False
},
{
"name": "Everything",
"baseurl": "Everything",
}
],
"keep_original_sources": True,
"ostree_repo": "/mnt/koji/compose/atomic/Rawhide/", "ostree_repo": "/mnt/koji/compose/atomic/Rawhide/",
"update_summary": True, "update_summary": True,
"version": "24" "version": "24"

View File

@ -318,11 +318,27 @@ def _make_schema():
] ]
}, },
"source_repo_dict": {
"type": "object",
"properties": {
"name": {"type": "string"},
"baseurl": {"type": "string"},
"exclude": {"type": "string"},
"gpgcheck": {"type": "boolean"},
},
"additionalProperties": False,
},
"list_of_strings": { "list_of_strings": {
"type": "array", "type": "array",
"items": {"type": "string"}, "items": {"type": "string"},
}, },
"list_of_source_repo_dicts": {
"type": "array",
"items": {"$ref": "#/definitions/source_repo_dict"},
},
"strings": { "strings": {
"anyOf": [ "anyOf": [
{"type": "string"}, {"type": "string"},
@ -675,6 +691,8 @@ def _make_schema():
"treefile": {"type": "string"}, "treefile": {"type": "string"},
"config_url": {"type": "string"}, "config_url": {"type": "string"},
"source_repo_from": {"type": "string"}, "source_repo_from": {"type": "string"},
"extra_source_repos": {"$ref": "#/definitions/list_of_source_repo_dicts"},
"keep_original_sources": {"type": "boolean"},
"ostree_repo": {"type": "string"}, "ostree_repo": {"type": "string"},
"failable": {"$ref": "#/definitions/list_of_strings"}, "failable": {"$ref": "#/definitions/list_of_strings"},
"update_summary": {"type": "boolean"}, "update_summary": {"type": "boolean"},

View File

@ -1,9 +1,9 @@
# -*- coding: utf-8 -*- # -*- coding: utf-8 -*-
import datetime
import json import json
import os import os
from kobo.threads import ThreadPool, WorkerThread from kobo.threads import ThreadPool, WorkerThread
import re
from .base import ConfigGuardedPhase from .base import ConfigGuardedPhase
from .. import util from .. import util
@ -52,7 +52,29 @@ class OSTreeThread(WorkerThread):
create_dir=False)) create_dir=False))
self._clone_repo(repodir, config['config_url'], config.get('config_branch', 'master')) self._clone_repo(repodir, config['config_url'], config.get('config_branch', 'master'))
self._tweak_mirrorlist(repodir, source_repo)
treeconf = os.path.join(repodir, config['treefile'])
source_repos = [{'name': '%s-%s' % (compose.compose_id, config['source_repo_from']),
'baseurl': source_repo}]
extra_source_repos = config.get('extra_source_repos', None)
if extra_source_repos:
for extra in extra_source_repos:
baseurl = extra['baseurl']
if "://" not in baseurl:
# it's variant UID, translate to url
variant = compose.variants[baseurl]
url = translate_path(compose,
compose.paths.compose.repository('$basearch',
variant,
create_dir=False))
extra['baseurl'] = url
source_repos = source_repos + extra_source_repos
keep_original_sources = config.get('keep_original_sources', False)
self._tweak_treeconf(treeconf, source_repos=source_repos,
keep_original_sources=keep_original_sources)
# Ensure target directory exists, otherwise Koji task will fail to # Ensure target directory exists, otherwise Koji task will fail to
# mount it. # mount it.
@ -136,23 +158,42 @@ class OSTreeThread(WorkerThread):
scm.get_dir_from_scm({'scm': 'git', 'repo': url, 'branch': branch, 'dir': '.'}, scm.get_dir_from_scm({'scm': 'git', 'repo': url, 'branch': branch, 'dir': '.'},
repodir, logger=self.pool._logger) repodir, logger=self.pool._logger)
def _tweak_mirrorlist(self, repodir, source_repo): def _tweak_treeconf(self, treeconf, source_repos, keep_original_sources=False):
for file in os.listdir(repodir):
if file.endswith('.repo'):
tweak_file(os.path.join(repodir, file), source_repo)
def tweak_file(path, source_repo):
""" """
Ensure a given .repo file points to `source_repo`. Update tree config file by adding new repos and remove existing repos
from the tree config file if 'keep_original_sources' is not enabled.
This function replaces all lines starting with `mirrorlist`, `metalink` or
`baseurl` with `baseurl` set to requested repository.
""" """
with open(path, 'r') as f: # add this timestamp to repo name to get unique repo filename and repo name
contents = f.read() # should be safe enough
replacement = 'baseurl=%s' % source_repo time = datetime.datetime.now().strftime("%Y%m%d%H%M%S")
exp = re.compile(r'^(mirrorlist|metalink|baseurl)=.*$', re.MULTILINE)
contents = exp.sub(replacement, contents) treeconf_dir = os.path.dirname(treeconf)
with open(path, 'w') as f: with open(treeconf, 'r') as f:
f.write(contents) treeconf_content = json.load(f)
# backup the old tree config
os.rename(treeconf, '%s.%s.bak' % (treeconf, time))
repos = []
for repo in source_repos:
name = "%s-%s" % (repo['name'], time)
with open("%s/%s.repo" % (treeconf_dir, name), 'w') as f:
f.write("[%s]\n" % name)
f.write("name=%s\n" % name)
f.write("baseurl=%s\n" % repo['baseurl'])
exclude = repo.get('exclude', None)
if exclude:
f.write("exclude=%s\n" % exclude)
gpgcheck = '1' if repo.get('gpgcheck', False) else '0'
f.write("gpgcheck=%s\n" % gpgcheck)
repos.append(name)
original_repos = treeconf_content.get('repos', [])
if keep_original_sources:
treeconf_content['repos'] = original_repos + repos
else:
treeconf_content['repos'] = repos
# update tree config to add new repos
with open(treeconf, 'w') as f:
json.dump(treeconf_content, f, indent=4)

View File

@ -69,13 +69,14 @@ class OSTreeThreadTest(helpers.PungiTestCase):
def _dummy_config_repo(self, scm_dict, target, logger=None): def _dummy_config_repo(self, scm_dict, target, logger=None):
os.makedirs(target) os.makedirs(target)
helpers.touch(os.path.join(target, 'fedora-atomic-docker-host.json'), helpers.touch(os.path.join(target, 'fedora-atomic-docker-host.json'),
json.dumps({'ref': 'fedora-atomic/25/x86_64'})) json.dumps({'ref': 'fedora-atomic/25/x86_64',
'repos': ['fedora-rawhide', 'fedora-24', 'fedora-23']}))
helpers.touch(os.path.join(target, 'fedora-rawhide.repo'), helpers.touch(os.path.join(target, 'fedora-rawhide.repo'),
'mirrorlist=mirror-mirror-on-the-wall') '[fedora-rawhide]\nmirrorlist=mirror-mirror-on-the-wall')
helpers.touch(os.path.join(target, 'fedora-24.repo'), helpers.touch(os.path.join(target, 'fedora-24.repo'),
'metalink=who-is-the-fairest-of-them-all') '[fedora-24]\nmetalink=who-is-the-fairest-of-them-all')
helpers.touch(os.path.join(target, 'fedora-23.repo'), helpers.touch(os.path.join(target, 'fedora-23.repo'),
'baseurl=why-not-zoidberg?') '[fedora-23]\nbaseurl=why-not-zoidberg?')
def _mock_runroot(self, retcode, writefiles=None): def _mock_runroot(self, retcode, writefiles=None):
"""Pretend to run a task in runroot, creating a log file with given line """Pretend to run a task in runroot, creating a log file with given line
@ -120,10 +121,17 @@ class OSTreeThreadTest(helpers.PungiTestCase):
[mock.call(koji.get_runroot_cmd.return_value, [mock.call(koji.get_runroot_cmd.return_value,
log_file=self.topdir + '/logs/x86_64/Everything/ostree-1/runroot.log')]) log_file=self.topdir + '/logs/x86_64/Everything/ostree-1/runroot.log')])
for fp in ['fedora-rawhide.repo', 'fedora-24.repo', 'fedora-24.repo']: repo_files = []
for fp in os.listdir(os.path.join(self.topdir, 'work/ostree-1/config_repo')):
if fp.endswith('.repo'):
repo_files.append(fp)
if fp not in ['fedora-rawhide.repo', 'fedora-24.repo', 'fedora-23.repo']:
with open(os.path.join(self.topdir, 'work/ostree-1/config_repo', fp)) as f: with open(os.path.join(self.topdir, 'work/ostree-1/config_repo', fp)) as f:
self.assertIn('baseurl=http://example.com/Everything/$basearch/os', self.assertIn('baseurl=http://example.com/Everything/$basearch/os', f.read())
f.read()) # test a new repo file created
self.assertEqual(len(repo_files), 4)
self.assertTrue(os.path.isdir(self.repo)) self.assertTrue(os.path.isdir(self.repo))
@mock.patch('pungi.wrappers.scm.get_dir_from_scm') @mock.patch('pungi.wrappers.scm.get_dir_from_scm')
@ -257,12 +265,6 @@ class OSTreeThreadTest(helpers.PungiTestCase):
[mock.call(koji.get_runroot_cmd.return_value, [mock.call(koji.get_runroot_cmd.return_value,
log_file=self.topdir + '/logs/x86_64/Everything/ostree-1/runroot.log')]) log_file=self.topdir + '/logs/x86_64/Everything/ostree-1/runroot.log')])
for fp in ['fedora-rawhide.repo', 'fedora-24.repo', 'fedora-24.repo']:
with open(os.path.join(self.topdir, 'work/ostree-1/config_repo', fp)) as f:
self.assertIn('baseurl=http://example.com/Everything/$basearch/os',
f.read())
self.assertTrue(os.path.isdir(self.repo))
@mock.patch('pungi.wrappers.scm.get_dir_from_scm') @mock.patch('pungi.wrappers.scm.get_dir_from_scm')
@mock.patch('pungi.wrappers.kojiwrapper.KojiWrapper') @mock.patch('pungi.wrappers.kojiwrapper.KojiWrapper')
def test_run_with_versioning_metadata(self, KojiWrapper, get_dir_from_scm): def test_run_with_versioning_metadata(self, KojiWrapper, get_dir_from_scm):
@ -295,5 +297,94 @@ class OSTreeThreadTest(helpers.PungiTestCase):
[mock.call(koji.get_runroot_cmd.return_value, [mock.call(koji.get_runroot_cmd.return_value,
log_file=self.topdir + '/logs/x86_64/Everything/ostree-1/runroot.log')]) log_file=self.topdir + '/logs/x86_64/Everything/ostree-1/runroot.log')])
@mock.patch('pungi.wrappers.scm.get_dir_from_scm')
@mock.patch('pungi.wrappers.kojiwrapper.KojiWrapper')
def test_run_with_extra_source_repos(self, KojiWrapper, get_dir_from_scm):
get_dir_from_scm.side_effect = self._dummy_config_repo
koji = KojiWrapper.return_value
koji.run_runroot_cmd.side_effect = self._mock_runroot(0)
cfg = {
'source_repo_from': 'Everything',
'extra_source_repos': [
{
'name': 'repo_a',
'baseurl': 'http://url/to/repo/a',
'exclude': 'systemd-container'
},
{
'name': 'Server',
'baseurl': 'Server',
'exclude': 'systemd-container'
}
],
'config_url': 'https://git.fedorahosted.org/git/fedora-atomic.git',
'config_branch': 'f24',
'treefile': 'fedora-atomic-docker-host.json',
'ostree_repo': self.repo
}
t = ostree.OSTreeThread(self.pool)
t.process((self.compose, self.compose.variants['Everything'], 'x86_64', cfg), 1)
repo_files = []
for fp in os.listdir(os.path.join(self.topdir, 'work/ostree-1/config_repo')):
if fp.endswith('.repo'):
repo_files.append(fp)
if fp not in ['fedora-rawhide.repo', 'fedora-24.repo', 'fedora-23.repo']:
if fp.startswith('repo_a'):
with open(os.path.join(self.topdir, 'work/ostree-1/config_repo', fp)) as f:
# ignore timestamp in repo name while checking
content = f.read()
self.assertIn('[repo_a', content)
self.assertIn('name=repo_a', content)
self.assertIn('baseurl=http://url/to/repo/a', content)
self.assertIn('exclude=systemd-container', content)
self.assertIn('gpgcheck=0', content)
elif fp.startswith('Server'):
with open(os.path.join(self.topdir, 'work/ostree-1/config_repo', fp)) as f:
content = f.read()
self.assertIn('[Server', content)
self.assertIn('baseurl=http://example.com/Server/$basearch/os', content)
self.assertIn('exclude=systemd-container', content)
self.assertIn('gpgcheck=0', content)
else:
# this is the Everything repo (source_repo_from)
with open(os.path.join(self.topdir, 'work/ostree-1/config_repo', fp)) as f:
self.assertIn('baseurl=http://example.com/Everything/$basearch/os', f.read())
# test new repos files created
self.assertEqual(len(repo_files), 3 + 1 + len(cfg['extra_source_repos']))
@mock.patch('pungi.wrappers.scm.get_dir_from_scm')
@mock.patch('pungi.wrappers.kojiwrapper.KojiWrapper')
def test_run_with_keep_original_source_repos(self, KojiWrapper, get_dir_from_scm):
get_dir_from_scm.side_effect = self._dummy_config_repo
koji = KojiWrapper.return_value
koji.run_runroot_cmd.side_effect = self._mock_runroot(0)
cfg = {
'source_repo_from': 'Everything',
'keep_original_sources': True,
'config_url': 'https://git.fedorahosted.org/git/fedora-atomic.git',
'config_branch': 'f24',
'treefile': 'fedora-atomic-docker-host.json',
'ostree_repo': self.repo
}
t = ostree.OSTreeThread(self.pool)
t.process((self.compose, self.compose.variants['Everything'], 'x86_64', cfg), 1)
treeconf_content = json.load(open(os.path.join(self.topdir,
'work/ostree-1/config_repo',
cfg['treefile'])))
# added 1 repo (Everything), have 4 (3 + 1) repos now
self.assertEqual(len(treeconf_content['repos']), 4)
if __name__ == '__main__': if __name__ == '__main__':
unittest.main() unittest.main()