almalinux
/
pungi
15
1
Fork 0
Code Issues Pull Requests Releases Activity

[ostree] Add 'tree' sub-command to pungi-make-ostree script 

Update pungi-make-ostree to supourt sub-command 'tree', which is just
as the original feature of pungi-make-ostree to compose OSTree tree.
With the change we can add other sub commands later to build other
OSTree artifacts, like the installer image.

Inaddtional to the change, now the the 'tree' command can accept an
optional '--extra-config' parameter to update the original tree
configuration with extra configurations specified in a json file
before composing the OSTree tree.

Example:

pungi-make-ostree tree --repo=/ostree --treefile=/path/to/treefile \
--log-dir=/path/to/log --extra-config=/path/to/extra-config.json

The extra-config file can contains the same configuration as OSTree
phase, the difference is it doesn't understand variant UID as source
repo since it's not ran in the chain of phases. A valid configuration
can be like:

{
    "source_repo_from": "http://example.com/repo/x86_64/Server",
    "extra_source_repos": [
        {
            "name": "optional",
            "baseurl": "http://example.com/repo/x86_64/optional",
            "exclude": "systemd-container",
            "gpgcheck": False
        },
        {
            "name": "extra",
            "baseurl": "http://example.com/repo/x86_64/extra",
        }
    ],
    "keep_original_sources": True
}

The OSTree phase is updated to move out the task of updating treefile,
instead of that, it writes the extra configurations to a json file,
then 'pungi-make-ostree tree' will take it by option '--extra-config'.

Signed-off-by: Qixiang Wan <qwan@redhat.com>
This commit is contained in:
Qixiang Wan 2016-12-02 20:11:09 +08:00
parent d3cad4795c
commit e043604822
8 changed files with 465 additions and 239 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

81
pungi/ostree.py
View File

@ -1,81 +0,0 @@
# -*- coding: utf-8 -*-
"""
This module contains functions required by pungi-make-ostree.
It is expected to be runnable in Koji runroot.
"""
import argparse
import os
from kobo import shortcuts
import errno
def ensure_dir(path):
try:
os.makedirs(path)
except OSError as err:
if err.errno != errno.EEXIST:
raise
return path
def make_log_file(log_dir, filename):
"""Return path to log file with given name, if log_dir is set."""
if not log_dir:
return None
ensure_dir(log_dir)
return os.path.join(log_dir, '%s.log' % filename)
def init_ostree_repo(repo, log_dir=None):
"""If the ostree repo does not exist, initialize it."""
log_file = make_log_file(log_dir, 'init-ostree-repo')
if not os.path.isdir(repo) or not os.listdir(repo):
ensure_dir(repo)
shortcuts.run(['ostree', 'init', '--repo=%s' % repo, '--mode=archive-z2'],
show_cmd=True, stdout=True, logfile=log_file)
def make_ostree_repo(repo, config, version=None, log_dir=None):
log_file = make_log_file(log_dir, 'create-ostree-repo')
cmd = ['rpm-ostree', 'compose', 'tree', '--repo=%s' % repo,
'--write-commitid-to=%s' % make_log_file(log_dir, 'commitid')]
if version:
# Add versioning metadata
cmd.append('--add-metadata-string=version=%s' % version)
cmd.append(config)
shortcuts.run(cmd, show_cmd=True, stdout=True, logfile=log_file)
def update_ostree_summary(repo, log_dir=None):
log_file = make_log_file(log_dir, 'ostree-summary')
shortcuts.run(['ostree', 'summary', '-u', '--repo=%s' % repo],
show_cmd=True, stdout=True, logfile=log_file)
def run(opts):
init_ostree_repo(opts.ostree_repo, log_dir=opts.log_dir)
make_ostree_repo(opts.ostree_repo, opts.treefile, version=opts.version, log_dir=opts.log_dir)
if opts.update_summary:
update_ostree_summary(opts.ostree_repo, log_dir=opts.log_dir)
def main(args=None):
parser = argparse.ArgumentParser()
parser.add_argument('--log-dir',
help='where to log output')
parser.add_argument('ostree_repo', metavar='OSTREE_REPO',
help='where to put the ostree repo')
parser.add_argument('--treefile', required=True,
help='treefile for rpm-ostree')
parser.add_argument('--version',
help='version string to be added as versioning metadata')
parser.add_argument('--update-summary', action='store_true',
help='update summary metadata')
opts = parser.parse_args(args)
run(opts)

45
pungi/ostree/__init__.py Normal file
View File

@ -0,0 +1,45 @@
# -*- coding: utf-8 -*-
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation; version 2 of the License.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Library General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program; if not, see <https://gnu.org/licenses/>.
import argparse
from .tree import Tree
def main(args=None):
parser = argparse.ArgumentParser()
subparser = parser.add_subparsers(help="Sub commands")
treep = subparser.add_parser("tree", help="Compose OSTree repository")
treep.set_defaults(_class=Tree, func='run')
treep.add_argument('--repo', metavar='PATH', required=True,
help='where to put the OSTree repo (required)')
treep.add_argument('--treefile', metavar="FILE", required=True,
help='treefile for rpm-ostree (required)')
treep.add_argument('--log-dir', metavar="DIR",
help='where to log output')
treep.add_argument('--extra-config', metavar="FILE",
help='JSON file contains extra configurations')
treep.add_argument('--version', metavar="VERSION",
help='version string to be added as versioning metadata')
treep.add_argument('--update-summary', action='store_true',
help='update summary metadata')
args = parser.parse_args(args)
_class = args._class()
_class.set_args(args)
func = getattr(_class, args.func)
func()

19
pungi/ostree/base.py Normal file
View File

@ -0,0 +1,19 @@
# -*- coding: utf-8 -*-
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation; version 2 of the License.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Library General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program; if not, see <https://gnu.org/licenses/>.
class OSTree(object):
def set_args(self, args):
self.args = args

101
pungi/ostree/tree.py Normal file
View File

@ -0,0 +1,101 @@
# -*- coding: utf-8 -*-
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation; version 2 of the License.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Library General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program; if not, see <https://gnu.org/licenses/>.
import os
import json
from kobo import shortcuts
from pungi.util import makedirs
from .base import OSTree
from .utils import (make_log_file, tweak_treeconf,
get_ref_from_treefile, get_commitid_from_commitid_file)
class Tree(OSTree):
def _init_repo(self):
"""If the ostree repo does not exist, initialize it."""
log_file = make_log_file(self.logdir, 'init-ostree-repo')
if not os.path.isdir(self.repo) or not os.listdir(self.repo):
makedirs(self.repo)
shortcuts.run(['ostree', 'init', '--repo=%s' % self.repo, '--mode=archive-z2'],
show_cmd=True, stdout=True, logfile=log_file)
def _make_tree(self):
"""Compose OSTree tree"""
log_file = make_log_file(self.logdir, 'create-ostree-repo')
cmd = ['rpm-ostree', 'compose', 'tree', '--repo=%s' % self.repo,
'--write-commitid-to=%s' % self.commitid_file]
if self.version:
# Add versioning metadata
cmd.append('--add-metadata-string=version=%s' % self.version)
cmd.append(self.treefile)
shortcuts.run(cmd, show_cmd=True, stdout=True, logfile=log_file)
def _update_summary(self):
"""Update summary metadata"""
log_file = make_log_file(self.logdir, 'ostree-summary')
shortcuts.run(['ostree', 'summary', '-u', '--repo=%s' % self.repo],
show_cmd=True, stdout=True, logfile=log_file)
def _update_ref(self):
"""
Update the ref.
'--write-commitid-to' is specified when compose the tree, so we need
to update the ref by ourselves. ref is retrieved from treefile and
commitid is retrieved from the committid file.
"""
tag_ref = True
if self.extra_config:
tag_ref = self.extra_config.get('tag_ref', True)
if not tag_ref:
return
ref = get_ref_from_treefile(self.treefile)
commitid = get_commitid_from_commitid_file(self.commitid_file)
if ref and commitid:
# Let's write the tag out ourselves
heads_dir = os.path.join(self.repo, 'refs', 'heads')
if not os.path.exists(heads_dir):
raise RuntimeError('Refs/heads did not exist in ostree repo')
ref_path = os.path.join(heads_dir, ref)
makedirs(os.path.dirname(ref_path))
with open(ref_path, 'w') as f:
f.write(commitid + '\n')
def run(self):
self.repo = self.args.repo
self.treefile = self.args.treefile
self.version = self.args.version
self.logdir = self.args.log_dir
self.update_summary = self.args.update_summary
self.extra_config = self.args.extra_config
if self.extra_config:
self.extra_config = json.load(open(self.extra_config, 'r'))
source_repo_from = self.extra_config.get('source_repo_from', None)
extra_source_repos = self.extra_config.get('extra_source_repos', None)
keep_original_sources = self.extra_config.get('keep_original_sources', False)
repos = extra_source_repos + [{'name': 'source_repo_from', 'baseurl': source_repo_from}]
tweak_treeconf(self.treefile, source_repos=repos, keep_original_sources=keep_original_sources)
self.commitid_file = make_log_file(self.logdir, 'commitid')
self._init_repo()
self._make_tree()
self._update_ref()
if self.update_summary:
self._update_summary()

93
pungi/ostree/utils.py Normal file
View File

@ -0,0 +1,93 @@
# -*- coding: utf-8 -*-
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation; version 2 of the License.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Library General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program; if not, see <https://gnu.org/licenses/>.
import datetime
import json
import os
from pungi.util import makedirs
def make_log_file(log_dir, filename):
"""Return path to log file with given name, if log_dir is set."""
if not log_dir:
return None
makedirs(log_dir)
return os.path.join(log_dir, '%s.log' % filename)
def get_ref_from_treefile(treefile):
"""Return ref name by parsing the tree config file"""
ref = None
if os.path.isfile(treefile):
with open(treefile, 'r') as f:
try:
parsed = json.loads(f.read())
ref = parsed['ref']
except Exception:
pass
return ref
def get_commitid_from_commitid_file(commitid_file):
"""Return commit id which is read from the commitid file"""
commitid = None
if os.path.isfile(commitid_file):
with open(commitid_file, 'r') as f:
commitid = f.read().replace('\n', '')
return commitid
def tweak_treeconf(treeconf, source_repos=None, keep_original_sources=False):
"""
Update tree config file by adding new repos, and remove existing repos
from the tree config file if 'keep_original_sources' is not enabled.
"""
# add this timestamp to repo name to get unique repo filename and repo name
# should be safe enough
time = datetime.datetime.now().strftime("%Y%m%d%H%M%S")
treeconf_dir = os.path.dirname(treeconf)
with open(treeconf, 'r') as f:
treeconf_content = json.load(f)
# backup the old tree config
os.rename(treeconf, '%s.%s.bak' % (treeconf, time))
repos = []
if source_repos:
for repo in source_repos:
name = "%s-%s" % (repo['name'], time)
with open("%s/%s.repo" % (treeconf_dir, name), 'w') as f:
f.write("[%s]\n" % name)
f.write("name=%s\n" % name)
f.write("baseurl=%s\n" % repo['baseurl'])
exclude = repo.get('exclude', None)
if exclude:
f.write("exclude=%s\n" % exclude)
gpgcheck = '1' if repo.get('gpgcheck', False) else '0'
f.write("gpgcheck=%s\n" % gpgcheck)
repos.append(name)
original_repos = treeconf_content.get('repos', [])
if keep_original_sources:
treeconf_content['repos'] = original_repos + repos
else:
treeconf_content['repos'] = repos
# update tree config to add new repos
with open(treeconf, 'w') as f:
json.dump(treeconf_content, f, indent=4)

117
pungi/phases/ostree.py
View File

@ -1,12 +1,13 @@
# -*- coding: utf-8 -*-
import datetime
import copy
import json
import os
from kobo.threads import ThreadPool, WorkerThread
from .base import ConfigGuardedPhase
from .. import util
from ..ostree.utils import get_ref_from_treefile, get_commitid_from_commitid_file
from ..paths import translate_path
from ..wrappers import kojiwrapper, scm
@ -53,7 +54,6 @@ class OSTreeThread(WorkerThread):
self._clone_repo(repodir, config['config_url'], config.get('config_branch', 'master'))
treeconf = os.path.join(repodir, config['treefile'])
source_repos = [{'name': '%s-%s' % (compose.compose_id, config['source_repo_from']),
'baseurl': source_repo}]
@ -72,30 +72,41 @@ class OSTreeThread(WorkerThread):
source_repos = source_repos + extra_source_repos
keep_original_sources = config.get('keep_original_sources', False)
self._tweak_treeconf(treeconf, source_repos=source_repos,
keep_original_sources=keep_original_sources)
# copy the original config and update before save to a json file
new_config = copy.copy(config)
# repos in configuration can have repo url set to variant UID,
# update it to have the actual url that we just translated.
new_config.update({'source_repo_from': source_repo})
if extra_source_repos:
new_config.update({'extra_source_repos': extra_source_repos})
# remove unnecessary (for 'pungi-make-ostree tree' script ) elements
# from config, it doesn't hurt to have them, however remove them can
# reduce confusion
for k in ['ostree_repo', 'treefile', 'config_url', 'config_branch',
'failable', 'version', 'update_summary']:
new_config.pop(k, None)
extra_config_file = None
if new_config:
# write a json file to save the configuration, so 'pungi-make-ostree tree'
# can take use of it
extra_config_file = os.path.join(workdir, 'extra_config.json')
with open(extra_config_file, 'w') as f:
json.dump(new_config, f, indent=4)
# Ensure target directory exists, otherwise Koji task will fail to
# mount it.
util.makedirs(config['ostree_repo'])
self._run_ostree_cmd(compose, variant, arch, config, repodir)
ref, commitid = self._get_commit_info(config, repodir)
if config.get('tag_ref', True) and ref and commitid:
# Let's write the tag out ourselves
heads_dir = os.path.join(config['ostree_repo'], 'refs', 'heads')
if not os.path.exists(heads_dir):
raise RuntimeError('Refs/heads did not exist in ostree repo')
ref_path = os.path.join(heads_dir, ref)
if not os.path.exists(os.path.dirname(ref_path)):
os.makedirs(os.path.dirname(ref_path))
with open(ref_path, 'w') as f:
f.write(commitid + '\n')
self._run_ostree_cmd(compose, variant, arch, config, repodir,
extra_config_file=extra_config_file)
if compose.notifier:
ref = get_ref_from_treefile(os.path.join(repodir, config['treefile']))
# 'pungi-make-ostree tree' writes commitid to commitid.log in logdir
commitid = get_commitid_from_commitid_file(os.path.join(self.logdir, 'commitid.log'))
compose.notifier.send('ostree',
variant=variant.uid,
arch=arch,
@ -104,26 +115,12 @@ class OSTreeThread(WorkerThread):
self.pool.log_info('[DONE ] %s' % msg)
def _get_commit_info(self, config, config_repo):
ref = None
commitid = None
with open(os.path.join(config_repo, config['treefile']), 'r') as f:
try:
parsed = json.loads(f.read())
ref = parsed['ref']
except ValueError:
return None, None
if os.path.exists(os.path.join(self.logdir, 'commitid')):
with open(os.path.join(self.logdir, 'commitid'), 'r') as f:
commitid = f.read().replace('\n', '')
else:
return None, None
return ref, commitid
def _run_ostree_cmd(self, compose, variant, arch, config, config_repo):
def _run_ostree_cmd(self, compose, variant, arch, config, config_repo, extra_config_file=None):
cmd = [
'pungi-make-ostree',
'--log-dir=%s' % os.path.join(self.logdir),
'tree',
'--repo=%s' % config['ostree_repo'],
'--log-dir=%s' % self.logdir,
'--treefile=%s' % os.path.join(config_repo, config['treefile']),
]
@ -131,12 +128,12 @@ class OSTreeThread(WorkerThread):
if version:
cmd.append('--version=%s' % version)
if extra_config_file:
cmd.append('--extra-config=%s' % extra_config_file)
if config.get('update_summary', False):
cmd.append('--update-summary')
# positional argument: ostree_repo
cmd.append(config['ostree_repo'])
runroot_channel = compose.conf.get("runroot_channel")
runroot_tag = compose.conf["runroot_tag"]
@ -157,43 +154,3 @@ class OSTreeThread(WorkerThread):
def _clone_repo(self, repodir, url, branch):
scm.get_dir_from_scm({'scm': 'git', 'repo': url, 'branch': branch, 'dir': '.'},
repodir, logger=self.pool._logger)
def _tweak_treeconf(self, treeconf, source_repos, keep_original_sources=False):
"""
Update tree config file by adding new repos and remove existing repos
from the tree config file if 'keep_original_sources' is not enabled.
"""
# add this timestamp to repo name to get unique repo filename and repo name
# should be safe enough
time = datetime.datetime.now().strftime("%Y%m%d%H%M%S")
treeconf_dir = os.path.dirname(treeconf)
with open(treeconf, 'r') as f:
treeconf_content = json.load(f)
# backup the old tree config
os.rename(treeconf, '%s.%s.bak' % (treeconf, time))
repos = []
for repo in source_repos:
name = "%s-%s" % (repo['name'], time)
with open("%s/%s.repo" % (treeconf_dir, name), 'w') as f:
f.write("[%s]\n" % name)
f.write("name=%s\n" % name)
f.write("baseurl=%s\n" % repo['baseurl'])
exclude = repo.get('exclude', None)
if exclude:
f.write("exclude=%s\n" % exclude)
gpgcheck = '1' if repo.get('gpgcheck', False) else '0'
f.write("gpgcheck=%s\n" % gpgcheck)
repos.append(name)
original_repos = treeconf_content.get('repos', [])
if keep_original_sources:
treeconf_content['repos'] = original_repos + repos
else:
treeconf_content['repos'] = repos
# update tree config to add new repos
with open(treeconf, 'w') as f:
json.dump(treeconf_content, f, indent=4)

94
tests/test_ostree_phase.py
View File

@ -110,10 +110,12 @@ class OSTreeThreadTest(helpers.PungiTestCase):
self.assertEqual(koji.get_runroot_cmd.call_args_list,
[mock.call('rrt', 'x86_64',
['pungi-make-ostree',
'tree',
'--repo=%s' % self.repo,
'--log-dir=%s/logs/x86_64/Everything/ostree-1' % self.topdir,
'--treefile=%s/fedora-atomic-docker-host.json' % (
self.topdir + '/work/ostree-1/config_repo'),
self.repo],
'--extra-config=%s/extra_config.json' % (self.topdir + '/work/ostree-1')],
channel=None, mounts=[self.topdir, self.repo],
packages=['pungi', 'ostree', 'rpm-ostree'],
task_id=True, use_shell=True, new_chroot=True)])
@ -121,17 +123,7 @@ class OSTreeThreadTest(helpers.PungiTestCase):
[mock.call(koji.get_runroot_cmd.return_value,
log_file=self.topdir + '/logs/x86_64/Everything/ostree-1/runroot.log')])
repo_files = []
for fp in os.listdir(os.path.join(self.topdir, 'work/ostree-1/config_repo')):
if fp.endswith('.repo'):
repo_files.append(fp)
if fp not in ['fedora-rawhide.repo', 'fedora-24.repo', 'fedora-23.repo']:
with open(os.path.join(self.topdir, 'work/ostree-1/config_repo', fp)) as f:
self.assertIn('baseurl=http://example.com/Everything/$basearch/os', f.read())
# test a new repo file created
self.assertEqual(len(repo_files), 4)
self.assertTrue(os.path.isfile(os.path.join(self.topdir, 'work/ostree-1/extra_config.json')))
self.assertTrue(os.path.isdir(self.repo))
@mock.patch('pungi.wrappers.scm.get_dir_from_scm')
@ -181,7 +173,7 @@ class OSTreeThreadTest(helpers.PungiTestCase):
koji = KojiWrapper.return_value
koji.run_runroot_cmd.side_effect = self._mock_runroot(
0,
{'commitid': 'fca3465861a',
{'commitid.log': 'fca3465861a',
'create-ostree-repo.log':
['Doing work', 'fedora-atomic/25/x86_64 -> fca3465861a']})
t = ostree.OSTreeThread(self.pool)
@ -214,7 +206,7 @@ class OSTreeThreadTest(helpers.PungiTestCase):
[mock.call('ostree',
variant='Everything',
arch='x86_64',
ref=None,
ref='fedora-atomic/25/x86_64',
commitid=None)])
@mock.patch('pungi.wrappers.scm.get_dir_from_scm')
@ -254,10 +246,13 @@ class OSTreeThreadTest(helpers.PungiTestCase):
self.assertEqual(koji.get_runroot_cmd.call_args_list,
[mock.call('rrt', 'x86_64',
['pungi-make-ostree',
'tree',
'--repo=%s' % self.repo,
'--log-dir=%s/logs/x86_64/Everything/ostree-1' % self.topdir,
'--treefile=%s/fedora-atomic-docker-host.json' % (
self.topdir + '/work/ostree-1/config_repo'),
'--update-summary', self.repo],
'--extra-config=%s/work/ostree-1/extra_config.json' % self.topdir,
'--update-summary'],
channel=None, mounts=[self.topdir, self.repo],
packages=['pungi', 'ostree', 'rpm-ostree'],
task_id=True, use_shell=True, new_chroot=True)])
@ -286,10 +281,13 @@ class OSTreeThreadTest(helpers.PungiTestCase):
self.assertEqual(koji.get_runroot_cmd.call_args_list,
[mock.call('rrt', 'x86_64',
['pungi-make-ostree',
'tree',
'--repo=%s' % self.repo,
'--log-dir=%s/logs/x86_64/Everything/ostree-1' % self.topdir,
'--treefile=%s/fedora-atomic-docker-host.json' % (
self.topdir + '/work/ostree-1/config_repo'),
'--version=24', self.repo],
'--version=24',
'--extra-config=%s/work/ostree-1/extra_config.json' % self.topdir],
channel=None, mounts=[self.topdir, self.repo],
packages=['pungi', 'ostree', 'rpm-ostree'],
task_id=True, use_shell=True, new_chroot=True)])
@ -299,7 +297,7 @@ class OSTreeThreadTest(helpers.PungiTestCase):
@mock.patch('pungi.wrappers.scm.get_dir_from_scm')
@mock.patch('pungi.wrappers.kojiwrapper.KojiWrapper')
def test_run_with_extra_source_repos(self, KojiWrapper, get_dir_from_scm):
def test_write_extra_config_file(self, KojiWrapper, get_dir_from_scm):
get_dir_from_scm.side_effect = self._dummy_config_repo
koji = KojiWrapper.return_value
@ -319,55 +317,6 @@ class OSTreeThreadTest(helpers.PungiTestCase):
'exclude': 'systemd-container'
}
],
'config_url': 'https://git.fedorahosted.org/git/fedora-atomic.git',
'config_branch': 'f24',
'treefile': 'fedora-atomic-docker-host.json',
'ostree_repo': self.repo
}
t = ostree.OSTreeThread(self.pool)
t.process((self.compose, self.compose.variants['Everything'], 'x86_64', cfg), 1)
repo_files = []
for fp in os.listdir(os.path.join(self.topdir, 'work/ostree-1/config_repo')):
if fp.endswith('.repo'):
repo_files.append(fp)
if fp not in ['fedora-rawhide.repo', 'fedora-24.repo', 'fedora-23.repo']:
if fp.startswith('repo_a'):
with open(os.path.join(self.topdir, 'work/ostree-1/config_repo', fp)) as f:
# ignore timestamp in repo name while checking
content = f.read()
self.assertIn('[repo_a', content)
self.assertIn('name=repo_a', content)
self.assertIn('baseurl=http://url/to/repo/a', content)
self.assertIn('exclude=systemd-container', content)
self.assertIn('gpgcheck=0', content)
elif fp.startswith('Server'):
with open(os.path.join(self.topdir, 'work/ostree-1/config_repo', fp)) as f:
content = f.read()
self.assertIn('[Server', content)
self.assertIn('baseurl=http://example.com/Server/$basearch/os', content)
self.assertIn('exclude=systemd-container', content)
self.assertIn('gpgcheck=0', content)
else:
# this is the Everything repo (source_repo_from)
with open(os.path.join(self.topdir, 'work/ostree-1/config_repo', fp)) as f:
self.assertIn('baseurl=http://example.com/Everything/$basearch/os', f.read())
# test new repos files created
self.assertEqual(len(repo_files), 3 + 1 + len(cfg['extra_source_repos']))
@mock.patch('pungi.wrappers.scm.get_dir_from_scm')
@mock.patch('pungi.wrappers.kojiwrapper.KojiWrapper')
def test_run_with_keep_original_source_repos(self, KojiWrapper, get_dir_from_scm):
get_dir_from_scm.side_effect = self._dummy_config_repo
koji = KojiWrapper.return_value
koji.run_runroot_cmd.side_effect = self._mock_runroot(0)
cfg = {
'source_repo_from': 'Everything',
'keep_original_sources': True,
'config_url': 'https://git.fedorahosted.org/git/fedora-atomic.git',
'config_branch': 'f24',
@ -379,12 +328,13 @@ class OSTreeThreadTest(helpers.PungiTestCase):
t.process((self.compose, self.compose.variants['Everything'], 'x86_64', cfg), 1)
treeconf_content = json.load(open(os.path.join(self.topdir,
'work/ostree-1/config_repo',
cfg['treefile'])))
# added 1 repo (Everything), have 4 (3 + 1) repos now
self.assertEqual(len(treeconf_content['repos']), 4)
extra_config_file = os.path.join(self.topdir, 'work/ostree-1/extra_config.json')
self.assertTrue(os.path.isfile(extra_config_file))
extra_config = json.load(open(extra_config_file, 'r'))
self.assertTrue(extra_config.get('keep_original_sources', False))
self.assertEqual(extra_config.get('source_repo_from', None), 'http://example.com/Everything/$basearch/os')
self.assertEqual(len(extra_config.get('extra_source_repos', [])), len(cfg['extra_source_repos']))
self.assertEqual(extra_config.get('extra_source_repos').pop()['baseurl'], 'http://example.com/Server/$basearch/os')
if __name__ == '__main__':
unittest.main()

154
tests/test_ostree_script.py
View File

@ -6,7 +6,9 @@ import unittest
import mock
import os
import json
import sys
import datetime
sys.path.insert(0, os.path.join(os.path.dirname(__file__), '..'))
sys.path.insert(0, os.path.join(os.path.dirname(__file__), 'bin'))
@ -15,16 +17,28 @@ from tests import helpers
from pungi import ostree
class OstreeScriptTest(helpers.PungiTestCase):
class OstreeTreeScriptTest(helpers.PungiTestCase):
def _make_dummy_config_dir(self, path):
helpers.touch(os.path.join(path, 'fedora-atomic-docker-host.json'),
json.dumps({'ref': 'fedora-atomic/25/x86_64',
'repos': ['fedora-rawhide', 'fedora-24', 'fedora-23']}))
helpers.touch(os.path.join(path, 'fedora-rawhide.repo'),
'[fedora-rawhide]\nmirrorlist=mirror-mirror-on-the-wall')
helpers.touch(os.path.join(path, 'fedora-24.repo'),
'[fedora-24]\nmetalink=who-is-the-fairest-of-them-all')
helpers.touch(os.path.join(path, 'fedora-23.repo'),
'[fedora-23]\nbaseurl=why-not-zoidberg?')
@mock.patch('kobo.shortcuts.run')
def test_full_run(self, run):
repo = os.path.join(self.topdir, 'atomic')
ostree.main([
'tree',
'--repo=%s' % repo,
'--log-dir=%s' % os.path.join(self.topdir, 'logs', 'Atomic'),
'--treefile=%s/fedora-atomic-docker-host.json' % self.topdir,
repo,
])
self.maxDiff = None
@ -44,9 +58,10 @@ class OstreeScriptTest(helpers.PungiTestCase):
os.mkdir(repo)
ostree.main([
'tree',
'--repo=%s' % repo,
'--log-dir=%s' % os.path.join(self.topdir, 'logs', 'Atomic'),
'--treefile=%s/fedora-atomic-docker-host.json' % self.topdir,
repo,
])
self.maxDiff = None
@ -66,9 +81,10 @@ class OstreeScriptTest(helpers.PungiTestCase):
helpers.touch(os.path.join(repo, 'initialized'))
ostree.main([
'tree',
'--repo=%s' % repo,
'--log-dir=%s' % os.path.join(self.topdir, 'logs', 'Atomic'),
'--treefile=%s/fedora-atomic-docker-host.json' % self.topdir,
repo,
])
self.maxDiff = None
@ -84,10 +100,11 @@ class OstreeScriptTest(helpers.PungiTestCase):
repo = os.path.join(self.topdir, 'atomic')
ostree.main([
'tree',
'--repo=%s' % repo,
'--log-dir=%s' % os.path.join(self.topdir, 'logs', 'Atomic'),
'--treefile=%s/fedora-atomic-docker-host.json' % self.topdir,
'--update-summary',
repo,
])
self.maxDiff = None
@ -107,10 +124,11 @@ class OstreeScriptTest(helpers.PungiTestCase):
repo = os.path.join(self.topdir, 'atomic')
ostree.main([
'tree',
'--repo=%s' % repo,
'--log-dir=%s' % os.path.join(self.topdir, 'logs', 'Atomic'),
'--treefile=%s/fedora-atomic-docker-host.json' % self.topdir,
'--version=24',
repo,
])
self.maxDiff = None
@ -124,5 +142,129 @@ class OstreeScriptTest(helpers.PungiTestCase):
self.topdir + '/fedora-atomic-docker-host.json'],
logfile=self.topdir + '/logs/Atomic/create-ostree-repo.log', show_cmd=True, stdout=True)])
@mock.patch('pungi.ostree.utils.datetime')
@mock.patch('kobo.shortcuts.run')
def test_extra_config_with_extra_repos(self, run, time):
time.datetime.now.return_value = datetime.datetime(2016, 1, 1, 1, 1)
timestamp = time.datetime.now().strftime("%Y%m%d%H%M%S")
configdir = os.path.join(self.topdir, 'config')
self._make_dummy_config_dir(configdir)
treefile = os.path.join(configdir, 'fedora-atomic-docker-host.json')
repo = os.path.join(self.topdir, 'atomic')
extra_config_file = os.path.join(self.topdir, 'extra_config.json')
extra_config = {
"source_repo_from": "http://www.example.com/Server.repo",
"extra_source_repos": [
{
"name": "optional",
"baseurl": "http://example.com/repo/x86_64/optional",
"exclude": "systemd-container",
"gpgcheck": False
},
{
"name": "extra",
"baseurl": "http://example.com/repo/x86_64/extra",
}
]
}
helpers.touch(extra_config_file, json.dumps(extra_config))
ostree.main([
'tree',
'--repo=%s' % repo,
'--log-dir=%s' % os.path.join(self.topdir, 'logs', 'Atomic'),
'--treefile=%s' % treefile,
'--extra-config=%s' % extra_config_file,
])
source_repo_from_name = "source_repo_from-%s" % timestamp
source_repo_from_repo = os.path.join(configdir, "%s.repo" % source_repo_from_name)
self.assertTrue(os.path.isfile(source_repo_from_repo))
with open(source_repo_from_repo, 'r') as f:
content = f.read()
self.assertIn("[%s]" % source_repo_from_name, content)
self.assertIn("name=%s" % source_repo_from_name, content)
self.assertIn("baseurl=http://www.example.com/Server.repo", content)
self.assertIn("gpgcheck=0", content)
optional_repo_name = "optional-%s" % timestamp
optional_repo = os.path.join(configdir, "%s.repo" % optional_repo_name)
self.assertTrue(os.path.isfile(optional_repo))
with open(optional_repo, 'r') as f:
content = f.read()
self.assertIn("[%s]" % optional_repo_name, content)
self.assertIn("name=%s" % optional_repo_name, content)
self.assertIn("baseurl=http://example.com/repo/x86_64/optional", content)
self.assertIn("gpgcheck=0", content)
extra_repo_name = "extra-%s" % timestamp
extra_repo = os.path.join(configdir, "%s.repo" % extra_repo_name)
self.assertTrue(os.path.isfile(extra_repo))
with open(extra_repo, 'r') as f:
content = f.read()
self.assertIn("[%s]" % extra_repo_name, content)
self.assertIn("name=%s" % extra_repo_name, content)
self.assertIn("baseurl=http://example.com/repo/x86_64/extra", content)
self.assertIn("gpgcheck=0", content)
treeconf = json.load(open(treefile, 'r'))
repos = treeconf['repos']
self.assertEqual(len(repos), 3)
for name in [source_repo_from_name, optional_repo_name, extra_repo_name]:
self.assertIn(name, repos)
@mock.patch('pungi.ostree.utils.datetime')
@mock.patch('kobo.shortcuts.run')
def test_extra_config_with_keep_original_sources(self, run, time):
time.datetime.now.return_value = datetime.datetime(2016, 1, 1, 1, 1)
timestamp = time.datetime.now().strftime("%Y%m%d%H%M%S")
configdir = os.path.join(self.topdir, 'config')
self._make_dummy_config_dir(configdir)
treefile = os.path.join(configdir, 'fedora-atomic-docker-host.json')
repo = os.path.join(self.topdir, 'atomic')
extra_config_file = os.path.join(self.topdir, 'extra_config.json')
extra_config = {
"source_repo_from": "http://www.example.com/Server.repo",
"extra_source_repos": [
{
"name": "optional",
"baseurl": "http://example.com/repo/x86_64/optional",
"exclude": "systemd-container",
"gpgcheck": False
},
{
"name": "extra",
"baseurl": "http://example.com/repo/x86_64/extra",
}
],
"keep_original_sources": True
}
helpers.touch(extra_config_file, json.dumps(extra_config))
ostree.main([
'tree',
'--repo=%s' % repo,
'--log-dir=%s' % os.path.join(self.topdir, 'logs', 'Atomic'),
'--treefile=%s' % treefile,
'--extra-config=%s' % extra_config_file,
])
source_repo_from_name = "source_repo_from-%s" % timestamp
optional_repo_name = "optional-%s" % timestamp
extra_repo_name = "extra-%s" % timestamp
treeconf = json.load(open(treefile, 'r'))
repos = treeconf['repos']
self.assertEqual(len(repos), 6)
for name in ['fedora-rawhide', 'fedora-24', 'fedora-23',
source_repo_from_name, optional_repo_name, extra_repo_name]:
self.assertIn(name, repos)
if __name__ == '__main__':
unittest.main()