[ostree] Allow extra repos to get packages for composing OSTree repository
Sometimes addtional repos are required to get necessary packages for composing OSTree repository. For example, RHEL doesn't have an 'Everyting' variant, so composing OSTree repository from any of the RHEL variants won't work, addtional source repos need to be enabled to achieve that. The new option "extra_source_repos" enable the ability of allowing extra source repos. And a new option 'keep_original_sources' is introduced to keep the original repos found in tree config file, if this is enabled, Pungi will not remove the existing source repos from the tree config file, just add new repos of "source_repo_from" + "extra_source_repos" to the existing repos. Signed-off-by: Qixiang Wan <qwan@redhat.com>
This commit is contained in:
Qixiang Wan
parent
4427769f6a
commit
68e121e421
@ -1087,6 +1087,19 @@ a new commit.
|
||||
|
||||
These keys are optional:
|
||||
|
||||
* ``extra_source_repos`` -- (*[dict]*) Extra source repos to get packages
|
||||
while composing the OSTree repository. Each dict represents a yum repo.
|
||||
The allowed keys are:
|
||||
|
||||
* ``name`` (required)
|
||||
* ``baseurl`` (required) -- URL of external repo or variant UID, in the case
|
||||
of variant UID, url to variant repo will be built automatically.
|
||||
* ``gpgcheck`` (optional)
|
||||
* ``exclude`` (optional)
|
||||
|
||||
* ``keep_original_sources`` -- (*bool*) Keep the existing source repos in
|
||||
the tree config file. If not enabled, all the original source repos will
|
||||
be removed from the tree config file.
|
||||
* ``config_branch`` -- (*str*) Git branch of the repo to use. Defaults to
|
||||
``master``.
|
||||
* ``failable`` -- (*[str]*) List of architectures for which this
|
||||
@ -1107,7 +1120,20 @@ Example config
|
||||
"x86_64": {
|
||||
"treefile": "fedora-atomic-docker-host.json",
|
||||
"config_url": "https://git.fedorahosted.org/git/fedora-atomic.git",
|
||||
"source_repo_from": "Everything",
|
||||
"source_repo_from": "Server",
|
||||
"extra_source_repos": [
|
||||
{
|
||||
"name": "repo_a",
|
||||
"baseurl": "http://example.com/repo/x86_64/os",
|
||||
"exclude": "systemd-container",
|
||||
"gpgcheck": False
|
||||
},
|
||||
{
|
||||
"name": "Everything",
|
||||
"baseurl": "Everything",
|
||||
}
|
||||
],
|
||||
"keep_original_sources": True,
|
||||
"ostree_repo": "/mnt/koji/compose/atomic/Rawhide/",
|
||||
"update_summary": True,
|
||||
"version": "24"
|
||||
|
||||
@ -318,11 +318,27 @@ def _make_schema():
|
||||
]
|
||||
},
|
||||
|
||||
"source_repo_dict": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"name": {"type": "string"},
|
||||
"baseurl": {"type": "string"},
|
||||
"exclude": {"type": "string"},
|
||||
"gpgcheck": {"type": "boolean"},
|
||||
},
|
||||
"additionalProperties": False,
|
||||
},
|
||||
|
||||
"list_of_strings": {
|
||||
"type": "array",
|
||||
"items": {"type": "string"},
|
||||
},
|
||||
|
||||
"list_of_source_repo_dicts": {
|
||||
"type": "array",
|
||||
"items": {"$ref": "#/definitions/source_repo_dict"},
|
||||
},
|
||||
|
||||
"strings": {
|
||||
"anyOf": [
|
||||
{"type": "string"},
|
||||
@ -675,6 +691,8 @@ def _make_schema():
|
||||
"treefile": {"type": "string"},
|
||||
"config_url": {"type": "string"},
|
||||
"source_repo_from": {"type": "string"},
|
||||
"extra_source_repos": {"$ref": "#/definitions/list_of_source_repo_dicts"},
|
||||
"keep_original_sources": {"type": "boolean"},
|
||||
"ostree_repo": {"type": "string"},
|
||||
"failable": {"$ref": "#/definitions/list_of_strings"},
|
||||
"update_summary": {"type": "boolean"},
|
||||
|
||||
@ -1,9 +1,9 @@
|
||||
# -*- coding: utf-8 -*-
|
||||
|
||||
import datetime
|
||||
import json
|
||||
import os
|
||||
from kobo.threads import ThreadPool, WorkerThread
|
||||
import re
|
||||
|
||||
from .base import ConfigGuardedPhase
|
||||
from .. import util
|
||||
@ -52,7 +52,29 @@ class OSTreeThread(WorkerThread):
|
||||
create_dir=False))
|
||||
|
||||
self._clone_repo(repodir, config['config_url'], config.get('config_branch', 'master'))
|
||||
self._tweak_mirrorlist(repodir, source_repo)
|
||||
|
||||
treeconf = os.path.join(repodir, config['treefile'])
|
||||
source_repos = [{'name': '%s-%s' % (compose.compose_id, config['source_repo_from']),
|
||||
'baseurl': source_repo}]
|
||||
|
||||
extra_source_repos = config.get('extra_source_repos', None)
|
||||
if extra_source_repos:
|
||||
for extra in extra_source_repos:
|
||||
baseurl = extra['baseurl']
|
||||
if "://" not in baseurl:
|
||||
# it's variant UID, translate to url
|
||||
variant = compose.variants[baseurl]
|
||||
url = translate_path(compose,
|
||||
compose.paths.compose.repository('$basearch',
|
||||
variant,
|
||||
create_dir=False))
|
||||
extra['baseurl'] = url
|
||||
|
||||
source_repos = source_repos + extra_source_repos
|
||||
|
||||
keep_original_sources = config.get('keep_original_sources', False)
|
||||
self._tweak_treeconf(treeconf, source_repos=source_repos,
|
||||
keep_original_sources=keep_original_sources)
|
||||
|
||||
# Ensure target directory exists, otherwise Koji task will fail to
|
||||
# mount it.
|
||||
@ -136,23 +158,42 @@ class OSTreeThread(WorkerThread):
|
||||
scm.get_dir_from_scm({'scm': 'git', 'repo': url, 'branch': branch, 'dir': '.'},
|
||||
repodir, logger=self.pool._logger)
|
||||
|
||||
def _tweak_mirrorlist(self, repodir, source_repo):
|
||||
for file in os.listdir(repodir):
|
||||
if file.endswith('.repo'):
|
||||
tweak_file(os.path.join(repodir, file), source_repo)
|
||||
|
||||
|
||||
def tweak_file(path, source_repo):
|
||||
def _tweak_treeconf(self, treeconf, source_repos, keep_original_sources=False):
|
||||
"""
|
||||
Ensure a given .repo file points to `source_repo`.
|
||||
|
||||
This function replaces all lines starting with `mirrorlist`, `metalink` or
|
||||
`baseurl` with `baseurl` set to requested repository.
|
||||
Update tree config file by adding new repos and remove existing repos
|
||||
from the tree config file if 'keep_original_sources' is not enabled.
|
||||
"""
|
||||
with open(path, 'r') as f:
|
||||
contents = f.read()
|
||||
replacement = 'baseurl=%s' % source_repo
|
||||
exp = re.compile(r'^(mirrorlist|metalink|baseurl)=.*$', re.MULTILINE)
|
||||
contents = exp.sub(replacement, contents)
|
||||
with open(path, 'w') as f:
|
||||
f.write(contents)
|
||||
# add this timestamp to repo name to get unique repo filename and repo name
|
||||
# should be safe enough
|
||||
time = datetime.datetime.now().strftime("%Y%m%d%H%M%S")
|
||||
|
||||
treeconf_dir = os.path.dirname(treeconf)
|
||||
with open(treeconf, 'r') as f:
|
||||
treeconf_content = json.load(f)
|
||||
|
||||
# backup the old tree config
|
||||
os.rename(treeconf, '%s.%s.bak' % (treeconf, time))
|
||||
|
||||
repos = []
|
||||
for repo in source_repos:
|
||||
name = "%s-%s" % (repo['name'], time)
|
||||
with open("%s/%s.repo" % (treeconf_dir, name), 'w') as f:
|
||||
f.write("[%s]\n" % name)
|
||||
f.write("name=%s\n" % name)
|
||||
f.write("baseurl=%s\n" % repo['baseurl'])
|
||||
exclude = repo.get('exclude', None)
|
||||
if exclude:
|
||||
f.write("exclude=%s\n" % exclude)
|
||||
gpgcheck = '1' if repo.get('gpgcheck', False) else '0'
|
||||
f.write("gpgcheck=%s\n" % gpgcheck)
|
||||
repos.append(name)
|
||||
|
||||
original_repos = treeconf_content.get('repos', [])
|
||||
if keep_original_sources:
|
||||
treeconf_content['repos'] = original_repos + repos
|
||||
else:
|
||||
treeconf_content['repos'] = repos
|
||||
|
||||
# update tree config to add new repos
|
||||
with open(treeconf, 'w') as f:
|
||||
json.dump(treeconf_content, f, indent=4)
|
||||
|
||||
@ -69,13 +69,14 @@ class OSTreeThreadTest(helpers.PungiTestCase):
|
||||
def _dummy_config_repo(self, scm_dict, target, logger=None):
|
||||
os.makedirs(target)
|
||||
helpers.touch(os.path.join(target, 'fedora-atomic-docker-host.json'),
|
||||
json.dumps({'ref': 'fedora-atomic/25/x86_64'}))
|
||||
json.dumps({'ref': 'fedora-atomic/25/x86_64',
|
||||
'repos': ['fedora-rawhide', 'fedora-24', 'fedora-23']}))
|
||||
helpers.touch(os.path.join(target, 'fedora-rawhide.repo'),
|
||||
'mirrorlist=mirror-mirror-on-the-wall')
|
||||
'[fedora-rawhide]\nmirrorlist=mirror-mirror-on-the-wall')
|
||||
helpers.touch(os.path.join(target, 'fedora-24.repo'),
|
||||
'metalink=who-is-the-fairest-of-them-all')
|
||||
'[fedora-24]\nmetalink=who-is-the-fairest-of-them-all')
|
||||
helpers.touch(os.path.join(target, 'fedora-23.repo'),
|
||||
'baseurl=why-not-zoidberg?')
|
||||
'[fedora-23]\nbaseurl=why-not-zoidberg?')
|
||||
|
||||
def _mock_runroot(self, retcode, writefiles=None):
|
||||
"""Pretend to run a task in runroot, creating a log file with given line
|
||||
@ -120,10 +121,17 @@ class OSTreeThreadTest(helpers.PungiTestCase):
|
||||
[mock.call(koji.get_runroot_cmd.return_value,
|
||||
log_file=self.topdir + '/logs/x86_64/Everything/ostree-1/runroot.log')])
|
||||
|
||||
for fp in ['fedora-rawhide.repo', 'fedora-24.repo', 'fedora-24.repo']:
|
||||
repo_files = []
|
||||
for fp in os.listdir(os.path.join(self.topdir, 'work/ostree-1/config_repo')):
|
||||
if fp.endswith('.repo'):
|
||||
repo_files.append(fp)
|
||||
|
||||
if fp not in ['fedora-rawhide.repo', 'fedora-24.repo', 'fedora-23.repo']:
|
||||
with open(os.path.join(self.topdir, 'work/ostree-1/config_repo', fp)) as f:
|
||||
self.assertIn('baseurl=http://example.com/Everything/$basearch/os',
|
||||
f.read())
|
||||
self.assertIn('baseurl=http://example.com/Everything/$basearch/os', f.read())
|
||||
# test a new repo file created
|
||||
self.assertEqual(len(repo_files), 4)
|
||||
|
||||
self.assertTrue(os.path.isdir(self.repo))
|
||||
|
||||
@mock.patch('pungi.wrappers.scm.get_dir_from_scm')
|
||||
@ -257,12 +265,6 @@ class OSTreeThreadTest(helpers.PungiTestCase):
|
||||
[mock.call(koji.get_runroot_cmd.return_value,
|
||||
log_file=self.topdir + '/logs/x86_64/Everything/ostree-1/runroot.log')])
|
||||
|
||||
for fp in ['fedora-rawhide.repo', 'fedora-24.repo', 'fedora-24.repo']:
|
||||
with open(os.path.join(self.topdir, 'work/ostree-1/config_repo', fp)) as f:
|
||||
self.assertIn('baseurl=http://example.com/Everything/$basearch/os',
|
||||
f.read())
|
||||
self.assertTrue(os.path.isdir(self.repo))
|
||||
|
||||
@mock.patch('pungi.wrappers.scm.get_dir_from_scm')
|
||||
@mock.patch('pungi.wrappers.kojiwrapper.KojiWrapper')
|
||||
def test_run_with_versioning_metadata(self, KojiWrapper, get_dir_from_scm):
|
||||
@ -295,5 +297,94 @@ class OSTreeThreadTest(helpers.PungiTestCase):
|
||||
[mock.call(koji.get_runroot_cmd.return_value,
|
||||
log_file=self.topdir + '/logs/x86_64/Everything/ostree-1/runroot.log')])
|
||||
|
||||
@mock.patch('pungi.wrappers.scm.get_dir_from_scm')
|
||||
@mock.patch('pungi.wrappers.kojiwrapper.KojiWrapper')
|
||||
def test_run_with_extra_source_repos(self, KojiWrapper, get_dir_from_scm):
|
||||
get_dir_from_scm.side_effect = self._dummy_config_repo
|
||||
|
||||
koji = KojiWrapper.return_value
|
||||
koji.run_runroot_cmd.side_effect = self._mock_runroot(0)
|
||||
|
||||
cfg = {
|
||||
'source_repo_from': 'Everything',
|
||||
'extra_source_repos': [
|
||||
{
|
||||
'name': 'repo_a',
|
||||
'baseurl': 'http://url/to/repo/a',
|
||||
'exclude': 'systemd-container'
|
||||
},
|
||||
{
|
||||
'name': 'Server',
|
||||
'baseurl': 'Server',
|
||||
'exclude': 'systemd-container'
|
||||
}
|
||||
],
|
||||
'config_url': 'https://git.fedorahosted.org/git/fedora-atomic.git',
|
||||
'config_branch': 'f24',
|
||||
'treefile': 'fedora-atomic-docker-host.json',
|
||||
'ostree_repo': self.repo
|
||||
}
|
||||
|
||||
t = ostree.OSTreeThread(self.pool)
|
||||
|
||||
t.process((self.compose, self.compose.variants['Everything'], 'x86_64', cfg), 1)
|
||||
|
||||
repo_files = []
|
||||
for fp in os.listdir(os.path.join(self.topdir, 'work/ostree-1/config_repo')):
|
||||
if fp.endswith('.repo'):
|
||||
repo_files.append(fp)
|
||||
|
||||
if fp not in ['fedora-rawhide.repo', 'fedora-24.repo', 'fedora-23.repo']:
|
||||
if fp.startswith('repo_a'):
|
||||
with open(os.path.join(self.topdir, 'work/ostree-1/config_repo', fp)) as f:
|
||||
# ignore timestamp in repo name while checking
|
||||
content = f.read()
|
||||
self.assertIn('[repo_a', content)
|
||||
self.assertIn('name=repo_a', content)
|
||||
self.assertIn('baseurl=http://url/to/repo/a', content)
|
||||
self.assertIn('exclude=systemd-container', content)
|
||||
self.assertIn('gpgcheck=0', content)
|
||||
elif fp.startswith('Server'):
|
||||
with open(os.path.join(self.topdir, 'work/ostree-1/config_repo', fp)) as f:
|
||||
content = f.read()
|
||||
self.assertIn('[Server', content)
|
||||
self.assertIn('baseurl=http://example.com/Server/$basearch/os', content)
|
||||
self.assertIn('exclude=systemd-container', content)
|
||||
self.assertIn('gpgcheck=0', content)
|
||||
else:
|
||||
# this is the Everything repo (source_repo_from)
|
||||
with open(os.path.join(self.topdir, 'work/ostree-1/config_repo', fp)) as f:
|
||||
self.assertIn('baseurl=http://example.com/Everything/$basearch/os', f.read())
|
||||
# test new repos files created
|
||||
self.assertEqual(len(repo_files), 3 + 1 + len(cfg['extra_source_repos']))
|
||||
|
||||
@mock.patch('pungi.wrappers.scm.get_dir_from_scm')
|
||||
@mock.patch('pungi.wrappers.kojiwrapper.KojiWrapper')
|
||||
def test_run_with_keep_original_source_repos(self, KojiWrapper, get_dir_from_scm):
|
||||
get_dir_from_scm.side_effect = self._dummy_config_repo
|
||||
|
||||
koji = KojiWrapper.return_value
|
||||
koji.run_runroot_cmd.side_effect = self._mock_runroot(0)
|
||||
|
||||
cfg = {
|
||||
'source_repo_from': 'Everything',
|
||||
'keep_original_sources': True,
|
||||
'config_url': 'https://git.fedorahosted.org/git/fedora-atomic.git',
|
||||
'config_branch': 'f24',
|
||||
'treefile': 'fedora-atomic-docker-host.json',
|
||||
'ostree_repo': self.repo
|
||||
}
|
||||
|
||||
t = ostree.OSTreeThread(self.pool)
|
||||
|
||||
t.process((self.compose, self.compose.variants['Everything'], 'x86_64', cfg), 1)
|
||||
|
||||
treeconf_content = json.load(open(os.path.join(self.topdir,
|
||||
'work/ostree-1/config_repo',
|
||||
cfg['treefile'])))
|
||||
|
||||
# added 1 repo (Everything), have 4 (3 + 1) repos now
|
||||
self.assertEqual(len(treeconf_content['repos']), 4)
|
||||
|
||||
if __name__ == '__main__':
|
||||
unittest.main()
|
||||
|
||||
Loading…
Reference in New Issue
Block a user