almalinux/pungi
14
1
Fork 0
Code Issues Pull Requests Releases Activity

Merge #463 [ostree] Allow extra repos to get packages for composing OSTree repository

Browse Source
This commit is contained in:
Dennis Gilmore 2016-11-15 21:06:05 +00:00
commit 69d949aaf9
4 changed files with 210 additions and 34 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

28
doc/configuration.rst
View File

@ -1087,6 +1087,19 @@ a new commit.
These keys are optional:
* ``extra_source_repos`` -- (*[dict]*) Extra source repos to get packages
while composing the OSTree repository. Each dict represents a yum repo.
The allowed keys are:
* ``name`` (required)
* ``baseurl`` (required) -- URL of external repo or variant UID, in the case
of variant UID, url to variant repo will be built automatically.
* ``gpgcheck`` (optional)
* ``exclude`` (optional)
* ``keep_original_sources`` -- (*bool*) Keep the existing source repos in
the tree config file. If not enabled, all the original source repos will
be removed from the tree config file.
* ``config_branch`` -- (*str*) Git branch of the repo to use. Defaults to
``master``.
* ``failable`` -- (*[str]*) List of architectures for which this
@ -1107,7 +1120,20 @@ Example config
"x86_64": {
"treefile": "fedora-atomic-docker-host.json",
"config_url": "https://git.fedorahosted.org/git/fedora-atomic.git",
"source_repo_from": "Everything",
"source_repo_from": "Server",
"extra_source_repos": [
{
"name": "repo_a",
"baseurl": "http://example.com/repo/x86_64/os",
"exclude": "systemd-container",
"gpgcheck": False
},
{
"name": "Everything",
"baseurl": "Everything",
}
],
"keep_original_sources": True,
"ostree_repo": "/mnt/koji/compose/atomic/Rawhide/",
"update_summary": True,
"version": "24"

18
pungi/checks.py
View File

@ -318,11 +318,27 @@ def _make_schema():
]
},
"source_repo_dict": {
"type": "object",
"properties": {
"name": {"type": "string"},
"baseurl": {"type": "string"},
"exclude": {"type": "string"},
"gpgcheck": {"type": "boolean"},
},
"additionalProperties": False,
},
"list_of_strings": {
"type": "array",
"items": {"type": "string"},
},
"list_of_source_repo_dicts": {
"type": "array",
"items": {"$ref": "#/definitions/source_repo_dict"},
},
"strings": {
"anyOf": [
{"type": "string"},
@ -675,6 +691,8 @@ def _make_schema():
"treefile": {"type": "string"},
"config_url": {"type": "string"},
"source_repo_from": {"type": "string"},
"extra_source_repos": {"$ref": "#/definitions/list_of_source_repo_dicts"},
"keep_original_sources": {"type": "boolean"},
"ostree_repo": {"type": "string"},
"failable": {"$ref": "#/definitions/list_of_strings"},
"update_summary": {"type": "boolean"},

79
pungi/phases/ostree.py
View File

@ -1,9 +1,9 @@
# -*- coding: utf-8 -*-
import datetime
import json
import os
from kobo.threads import ThreadPool, WorkerThread
import re
from .base import ConfigGuardedPhase
from .. import util
@ -52,7 +52,29 @@ class OSTreeThread(WorkerThread):
create_dir=False))
self._clone_repo(repodir, config['config_url'], config.get('config_branch', 'master'))
self._tweak_mirrorlist(repodir, source_repo)
treeconf = os.path.join(repodir, config['treefile'])
source_repos = [{'name': '%s-%s' % (compose.compose_id, config['source_repo_from']),
'baseurl': source_repo}]
extra_source_repos = config.get('extra_source_repos', None)
if extra_source_repos:
for extra in extra_source_repos:
baseurl = extra['baseurl']
if "://" not in baseurl:
# it's variant UID, translate to url
variant = compose.variants[baseurl]
url = translate_path(compose,
compose.paths.compose.repository('$basearch',
variant,
create_dir=False))
extra['baseurl'] = url
source_repos = source_repos + extra_source_repos
keep_original_sources = config.get('keep_original_sources', False)
self._tweak_treeconf(treeconf, source_repos=source_repos,
keep_original_sources=keep_original_sources)
# Ensure target directory exists, otherwise Koji task will fail to
# mount it.
@ -136,23 +158,42 @@ class OSTreeThread(WorkerThread):
scm.get_dir_from_scm({'scm': 'git', 'repo': url, 'branch': branch, 'dir': '.'},
repodir, logger=self.pool._logger)
def _tweak_mirrorlist(self, repodir, source_repo):
for file in os.listdir(repodir):
if file.endswith('.repo'):
tweak_file(os.path.join(repodir, file), source_repo)
def _tweak_treeconf(self, treeconf, source_repos, keep_original_sources=False):
"""
Update tree config file by adding new repos and remove existing repos
from the tree config file if 'keep_original_sources' is not enabled.
"""
# add this timestamp to repo name to get unique repo filename and repo name
# should be safe enough
time = datetime.datetime.now().strftime("%Y%m%d%H%M%S")
treeconf_dir = os.path.dirname(treeconf)
with open(treeconf, 'r') as f:
treeconf_content = json.load(f)
def tweak_file(path, source_repo):
"""
Ensure a given .repo file points to `source_repo`.
# backup the old tree config
os.rename(treeconf, '%s.%s.bak' % (treeconf, time))
This function replaces all lines starting with `mirrorlist`, `metalink` or
`baseurl` with `baseurl` set to requested repository.
"""
with open(path, 'r') as f:
contents = f.read()
replacement = 'baseurl=%s' % source_repo
exp = re.compile(r'^(mirrorlist|metalink|baseurl)=.*$', re.MULTILINE)
contents = exp.sub(replacement, contents)
with open(path, 'w') as f:
f.write(contents)
repos = []
for repo in source_repos:
name = "%s-%s" % (repo['name'], time)
with open("%s/%s.repo" % (treeconf_dir, name), 'w') as f:
f.write("[%s]\n" % name)
f.write("name=%s\n" % name)
f.write("baseurl=%s\n" % repo['baseurl'])
exclude = repo.get('exclude', None)
if exclude:
f.write("exclude=%s\n" % exclude)
gpgcheck = '1' if repo.get('gpgcheck', False) else '0'
f.write("gpgcheck=%s\n" % gpgcheck)
repos.append(name)
original_repos = treeconf_content.get('repos', [])
if keep_original_sources:
treeconf_content['repos'] = original_repos + repos
else:
treeconf_content['repos'] = repos
# update tree config to add new repos
with open(treeconf, 'w') as f:
json.dump(treeconf_content, f, indent=4)

119
tests/test_ostree_phase.py
View File

@ -69,13 +69,14 @@ class OSTreeThreadTest(helpers.PungiTestCase):
def _dummy_config_repo(self, scm_dict, target, logger=None):
os.makedirs(target)
helpers.touch(os.path.join(target, 'fedora-atomic-docker-host.json'),
json.dumps({'ref': 'fedora-atomic/25/x86_64'}))
json.dumps({'ref': 'fedora-atomic/25/x86_64',
'repos': ['fedora-rawhide', 'fedora-24', 'fedora-23']}))
helpers.touch(os.path.join(target, 'fedora-rawhide.repo'),
'mirrorlist=mirror-mirror-on-the-wall')
'[fedora-rawhide]\nmirrorlist=mirror-mirror-on-the-wall')
helpers.touch(os.path.join(target, 'fedora-24.repo'),
'metalink=who-is-the-fairest-of-them-all')
'[fedora-24]\nmetalink=who-is-the-fairest-of-them-all')
helpers.touch(os.path.join(target, 'fedora-23.repo'),
'baseurl=why-not-zoidberg?')
'[fedora-23]\nbaseurl=why-not-zoidberg?')
def _mock_runroot(self, retcode, writefiles=None):
"""Pretend to run a task in runroot, creating a log file with given line
@ -120,10 +121,17 @@ class OSTreeThreadTest(helpers.PungiTestCase):
[mock.call(koji.get_runroot_cmd.return_value,
log_file=self.topdir + '/logs/x86_64/Everything/ostree-1/runroot.log')])
for fp in ['fedora-rawhide.repo', 'fedora-24.repo', 'fedora-24.repo']:
with open(os.path.join(self.topdir, 'work/ostree-1/config_repo', fp)) as f:
self.assertIn('baseurl=http://example.com/Everything/$basearch/os',
f.read())
repo_files = []
for fp in os.listdir(os.path.join(self.topdir, 'work/ostree-1/config_repo')):
if fp.endswith('.repo'):
repo_files.append(fp)
if fp not in ['fedora-rawhide.repo', 'fedora-24.repo', 'fedora-23.repo']:
with open(os.path.join(self.topdir, 'work/ostree-1/config_repo', fp)) as f:
self.assertIn('baseurl=http://example.com/Everything/$basearch/os', f.read())
# test a new repo file created
self.assertEqual(len(repo_files), 4)
self.assertTrue(os.path.isdir(self.repo))
@mock.patch('pungi.wrappers.scm.get_dir_from_scm')
@ -257,12 +265,6 @@ class OSTreeThreadTest(helpers.PungiTestCase):
[mock.call(koji.get_runroot_cmd.return_value,
log_file=self.topdir + '/logs/x86_64/Everything/ostree-1/runroot.log')])
for fp in ['fedora-rawhide.repo', 'fedora-24.repo', 'fedora-24.repo']:
with open(os.path.join(self.topdir, 'work/ostree-1/config_repo', fp)) as f:
self.assertIn('baseurl=http://example.com/Everything/$basearch/os',
f.read())
self.assertTrue(os.path.isdir(self.repo))
@mock.patch('pungi.wrappers.scm.get_dir_from_scm')
@mock.patch('pungi.wrappers.kojiwrapper.KojiWrapper')
def test_run_with_versioning_metadata(self, KojiWrapper, get_dir_from_scm):
@ -295,5 +297,94 @@ class OSTreeThreadTest(helpers.PungiTestCase):
[mock.call(koji.get_runroot_cmd.return_value,
log_file=self.topdir + '/logs/x86_64/Everything/ostree-1/runroot.log')])
@mock.patch('pungi.wrappers.scm.get_dir_from_scm')
@mock.patch('pungi.wrappers.kojiwrapper.KojiWrapper')
def test_run_with_extra_source_repos(self, KojiWrapper, get_dir_from_scm):
get_dir_from_scm.side_effect = self._dummy_config_repo
koji = KojiWrapper.return_value
koji.run_runroot_cmd.side_effect = self._mock_runroot(0)
cfg = {
'source_repo_from': 'Everything',
'extra_source_repos': [
{
'name': 'repo_a',
'baseurl': 'http://url/to/repo/a',
'exclude': 'systemd-container'
},
{
'name': 'Server',
'baseurl': 'Server',
'exclude': 'systemd-container'
}
],
'config_url': 'https://git.fedorahosted.org/git/fedora-atomic.git',
'config_branch': 'f24',
'treefile': 'fedora-atomic-docker-host.json',
'ostree_repo': self.repo
}
t = ostree.OSTreeThread(self.pool)
t.process((self.compose, self.compose.variants['Everything'], 'x86_64', cfg), 1)
repo_files = []
for fp in os.listdir(os.path.join(self.topdir, 'work/ostree-1/config_repo')):
if fp.endswith('.repo'):
repo_files.append(fp)
if fp not in ['fedora-rawhide.repo', 'fedora-24.repo', 'fedora-23.repo']:
if fp.startswith('repo_a'):
with open(os.path.join(self.topdir, 'work/ostree-1/config_repo', fp)) as f:
# ignore timestamp in repo name while checking
content = f.read()
self.assertIn('[repo_a', content)
self.assertIn('name=repo_a', content)
self.assertIn('baseurl=http://url/to/repo/a', content)
self.assertIn('exclude=systemd-container', content)
self.assertIn('gpgcheck=0', content)
elif fp.startswith('Server'):
with open(os.path.join(self.topdir, 'work/ostree-1/config_repo', fp)) as f:
content = f.read()
self.assertIn('[Server', content)
self.assertIn('baseurl=http://example.com/Server/$basearch/os', content)
self.assertIn('exclude=systemd-container', content)
self.assertIn('gpgcheck=0', content)
else:
# this is the Everything repo (source_repo_from)
with open(os.path.join(self.topdir, 'work/ostree-1/config_repo', fp)) as f:
self.assertIn('baseurl=http://example.com/Everything/$basearch/os', f.read())
# test new repos files created
self.assertEqual(len(repo_files), 3 + 1 + len(cfg['extra_source_repos']))
@mock.patch('pungi.wrappers.scm.get_dir_from_scm')
@mock.patch('pungi.wrappers.kojiwrapper.KojiWrapper')
def test_run_with_keep_original_source_repos(self, KojiWrapper, get_dir_from_scm):
get_dir_from_scm.side_effect = self._dummy_config_repo
koji = KojiWrapper.return_value
koji.run_runroot_cmd.side_effect = self._mock_runroot(0)
cfg = {
'source_repo_from': 'Everything',
'keep_original_sources': True,
'config_url': 'https://git.fedorahosted.org/git/fedora-atomic.git',
'config_branch': 'f24',
'treefile': 'fedora-atomic-docker-host.json',
'ostree_repo': self.repo
}
t = ostree.OSTreeThread(self.pool)
t.process((self.compose, self.compose.variants['Everything'], 'x86_64', cfg), 1)
treeconf_content = json.load(open(os.path.join(self.topdir,
'work/ostree-1/config_repo',
cfg['treefile'])))
# added 1 repo (Everything), have 4 (3 + 1) repos now
self.assertEqual(len(treeconf_content['repos']), 4)
if __name__ == '__main__':
unittest.main()