From e043604822c46102ffdb85fa3018279c7eb5c4d9 Mon Sep 17 00:00:00 2001 From: Qixiang Wan Date: Fri, 2 Dec 2016 20:11:09 +0800 Subject: [PATCH] [ostree] Add 'tree' sub-command to pungi-make-ostree script Update pungi-make-ostree to supourt sub-command 'tree', which is just as the original feature of pungi-make-ostree to compose OSTree tree. With the change we can add other sub commands later to build other OSTree artifacts, like the installer image. Inaddtional to the change, now the the 'tree' command can accept an optional '--extra-config' parameter to update the original tree configuration with extra configurations specified in a json file before composing the OSTree tree. Example: pungi-make-ostree tree --repo=/ostree --treefile=/path/to/treefile \ --log-dir=/path/to/log --extra-config=/path/to/extra-config.json The extra-config file can contains the same configuration as OSTree phase, the difference is it doesn't understand variant UID as source repo since it's not ran in the chain of phases. A valid configuration can be like: { "source_repo_from": "http://example.com/repo/x86_64/Server", "extra_source_repos": [ { "name": "optional", "baseurl": "http://example.com/repo/x86_64/optional", "exclude": "systemd-container", "gpgcheck": False }, { "name": "extra", "baseurl": "http://example.com/repo/x86_64/extra", } ], "keep_original_sources": True } The OSTree phase is updated to move out the task of updating treefile, instead of that, it writes the extra configurations to a json file, then 'pungi-make-ostree tree' will take it by option '--extra-config'. Signed-off-by: Qixiang Wan --- pungi/ostree.py | 81 ------------------- pungi/ostree/__init__.py | 45 +++++++++++ pungi/ostree/base.py | 19 +++++ pungi/ostree/tree.py | 101 +++++++++++++++++++++++ pungi/ostree/utils.py | 93 ++++++++++++++++++++++ pungi/phases/ostree.py | 117 +++++++++------------------ tests/test_ostree_phase.py | 94 ++++++---------------- tests/test_ostree_script.py | 154 ++++++++++++++++++++++++++++++++++-- 8 files changed, 465 insertions(+), 239 deletions(-) delete mode 100644 pungi/ostree.py create mode 100644 pungi/ostree/__init__.py create mode 100644 pungi/ostree/base.py create mode 100644 pungi/ostree/tree.py create mode 100644 pungi/ostree/utils.py diff --git a/pungi/ostree.py b/pungi/ostree.py deleted file mode 100644 index 16e7fc2d..00000000 --- a/pungi/ostree.py +++ /dev/null @@ -1,81 +0,0 @@ -# -*- coding: utf-8 -*- - -""" -This module contains functions required by pungi-make-ostree. -It is expected to be runnable in Koji runroot. -""" - -import argparse -import os -from kobo import shortcuts -import errno - - -def ensure_dir(path): - try: - os.makedirs(path) - except OSError as err: - if err.errno != errno.EEXIST: - raise - return path - - -def make_log_file(log_dir, filename): - """Return path to log file with given name, if log_dir is set.""" - if not log_dir: - return None - ensure_dir(log_dir) - return os.path.join(log_dir, '%s.log' % filename) - - -def init_ostree_repo(repo, log_dir=None): - """If the ostree repo does not exist, initialize it.""" - log_file = make_log_file(log_dir, 'init-ostree-repo') - if not os.path.isdir(repo) or not os.listdir(repo): - ensure_dir(repo) - shortcuts.run(['ostree', 'init', '--repo=%s' % repo, '--mode=archive-z2'], - show_cmd=True, stdout=True, logfile=log_file) - - -def make_ostree_repo(repo, config, version=None, log_dir=None): - log_file = make_log_file(log_dir, 'create-ostree-repo') - cmd = ['rpm-ostree', 'compose', 'tree', '--repo=%s' % repo, - '--write-commitid-to=%s' % make_log_file(log_dir, 'commitid')] - if version: - # Add versioning metadata - cmd.append('--add-metadata-string=version=%s' % version) - cmd.append(config) - - shortcuts.run(cmd, show_cmd=True, stdout=True, logfile=log_file) - - -def update_ostree_summary(repo, log_dir=None): - log_file = make_log_file(log_dir, 'ostree-summary') - shortcuts.run(['ostree', 'summary', '-u', '--repo=%s' % repo], - show_cmd=True, stdout=True, logfile=log_file) - - -def run(opts): - init_ostree_repo(opts.ostree_repo, log_dir=opts.log_dir) - make_ostree_repo(opts.ostree_repo, opts.treefile, version=opts.version, log_dir=opts.log_dir) - if opts.update_summary: - update_ostree_summary(opts.ostree_repo, log_dir=opts.log_dir) - - -def main(args=None): - parser = argparse.ArgumentParser() - parser.add_argument('--log-dir', - help='where to log output') - - parser.add_argument('ostree_repo', metavar='OSTREE_REPO', - help='where to put the ostree repo') - parser.add_argument('--treefile', required=True, - help='treefile for rpm-ostree') - parser.add_argument('--version', - help='version string to be added as versioning metadata') - parser.add_argument('--update-summary', action='store_true', - help='update summary metadata') - - opts = parser.parse_args(args) - - run(opts) diff --git a/pungi/ostree/__init__.py b/pungi/ostree/__init__.py new file mode 100644 index 00000000..fe8c53ea --- /dev/null +++ b/pungi/ostree/__init__.py @@ -0,0 +1,45 @@ +# -*- coding: utf-8 -*- + + +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; version 2 of the License. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Library General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, see . + + +import argparse + +from .tree import Tree + + +def main(args=None): + parser = argparse.ArgumentParser() + subparser = parser.add_subparsers(help="Sub commands") + + treep = subparser.add_parser("tree", help="Compose OSTree repository") + treep.set_defaults(_class=Tree, func='run') + treep.add_argument('--repo', metavar='PATH', required=True, + help='where to put the OSTree repo (required)') + treep.add_argument('--treefile', metavar="FILE", required=True, + help='treefile for rpm-ostree (required)') + treep.add_argument('--log-dir', metavar="DIR", + help='where to log output') + treep.add_argument('--extra-config', metavar="FILE", + help='JSON file contains extra configurations') + treep.add_argument('--version', metavar="VERSION", + help='version string to be added as versioning metadata') + treep.add_argument('--update-summary', action='store_true', + help='update summary metadata') + + args = parser.parse_args(args) + _class = args._class() + _class.set_args(args) + func = getattr(_class, args.func) + func() diff --git a/pungi/ostree/base.py b/pungi/ostree/base.py new file mode 100644 index 00000000..4932f597 --- /dev/null +++ b/pungi/ostree/base.py @@ -0,0 +1,19 @@ +# -*- coding: utf-8 -*- + + +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; version 2 of the License. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Library General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, see . + + +class OSTree(object): + def set_args(self, args): + self.args = args diff --git a/pungi/ostree/tree.py b/pungi/ostree/tree.py new file mode 100644 index 00000000..1b616b2c --- /dev/null +++ b/pungi/ostree/tree.py @@ -0,0 +1,101 @@ +# -*- coding: utf-8 -*- + + +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; version 2 of the License. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Library General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, see . + + +import os +import json +from kobo import shortcuts + +from pungi.util import makedirs +from .base import OSTree +from .utils import (make_log_file, tweak_treeconf, + get_ref_from_treefile, get_commitid_from_commitid_file) + + +class Tree(OSTree): + def _init_repo(self): + """If the ostree repo does not exist, initialize it.""" + log_file = make_log_file(self.logdir, 'init-ostree-repo') + if not os.path.isdir(self.repo) or not os.listdir(self.repo): + makedirs(self.repo) + shortcuts.run(['ostree', 'init', '--repo=%s' % self.repo, '--mode=archive-z2'], + show_cmd=True, stdout=True, logfile=log_file) + + def _make_tree(self): + """Compose OSTree tree""" + log_file = make_log_file(self.logdir, 'create-ostree-repo') + cmd = ['rpm-ostree', 'compose', 'tree', '--repo=%s' % self.repo, + '--write-commitid-to=%s' % self.commitid_file] + if self.version: + # Add versioning metadata + cmd.append('--add-metadata-string=version=%s' % self.version) + cmd.append(self.treefile) + + shortcuts.run(cmd, show_cmd=True, stdout=True, logfile=log_file) + + def _update_summary(self): + """Update summary metadata""" + log_file = make_log_file(self.logdir, 'ostree-summary') + shortcuts.run(['ostree', 'summary', '-u', '--repo=%s' % self.repo], + show_cmd=True, stdout=True, logfile=log_file) + + def _update_ref(self): + """ + Update the ref. + + '--write-commitid-to' is specified when compose the tree, so we need + to update the ref by ourselves. ref is retrieved from treefile and + commitid is retrieved from the committid file. + """ + tag_ref = True + if self.extra_config: + tag_ref = self.extra_config.get('tag_ref', True) + if not tag_ref: + return + ref = get_ref_from_treefile(self.treefile) + commitid = get_commitid_from_commitid_file(self.commitid_file) + if ref and commitid: + # Let's write the tag out ourselves + heads_dir = os.path.join(self.repo, 'refs', 'heads') + if not os.path.exists(heads_dir): + raise RuntimeError('Refs/heads did not exist in ostree repo') + + ref_path = os.path.join(heads_dir, ref) + makedirs(os.path.dirname(ref_path)) + with open(ref_path, 'w') as f: + f.write(commitid + '\n') + + def run(self): + self.repo = self.args.repo + self.treefile = self.args.treefile + self.version = self.args.version + self.logdir = self.args.log_dir + self.update_summary = self.args.update_summary + self.extra_config = self.args.extra_config + if self.extra_config: + self.extra_config = json.load(open(self.extra_config, 'r')) + source_repo_from = self.extra_config.get('source_repo_from', None) + extra_source_repos = self.extra_config.get('extra_source_repos', None) + keep_original_sources = self.extra_config.get('keep_original_sources', False) + repos = extra_source_repos + [{'name': 'source_repo_from', 'baseurl': source_repo_from}] + tweak_treeconf(self.treefile, source_repos=repos, keep_original_sources=keep_original_sources) + + self.commitid_file = make_log_file(self.logdir, 'commitid') + + self._init_repo() + self._make_tree() + self._update_ref() + if self.update_summary: + self._update_summary() diff --git a/pungi/ostree/utils.py b/pungi/ostree/utils.py new file mode 100644 index 00000000..fb5e3300 --- /dev/null +++ b/pungi/ostree/utils.py @@ -0,0 +1,93 @@ +# -*- coding: utf-8 -*- + + +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; version 2 of the License. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Library General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, see . + + +import datetime +import json +import os + +from pungi.util import makedirs + + +def make_log_file(log_dir, filename): + """Return path to log file with given name, if log_dir is set.""" + if not log_dir: + return None + makedirs(log_dir) + return os.path.join(log_dir, '%s.log' % filename) + + +def get_ref_from_treefile(treefile): + """Return ref name by parsing the tree config file""" + ref = None + if os.path.isfile(treefile): + with open(treefile, 'r') as f: + try: + parsed = json.loads(f.read()) + ref = parsed['ref'] + except Exception: + pass + return ref + + +def get_commitid_from_commitid_file(commitid_file): + """Return commit id which is read from the commitid file""" + commitid = None + if os.path.isfile(commitid_file): + with open(commitid_file, 'r') as f: + commitid = f.read().replace('\n', '') + return commitid + + +def tweak_treeconf(treeconf, source_repos=None, keep_original_sources=False): + """ + Update tree config file by adding new repos, and remove existing repos + from the tree config file if 'keep_original_sources' is not enabled. + """ + # add this timestamp to repo name to get unique repo filename and repo name + # should be safe enough + time = datetime.datetime.now().strftime("%Y%m%d%H%M%S") + + treeconf_dir = os.path.dirname(treeconf) + with open(treeconf, 'r') as f: + treeconf_content = json.load(f) + + # backup the old tree config + os.rename(treeconf, '%s.%s.bak' % (treeconf, time)) + + repos = [] + if source_repos: + for repo in source_repos: + name = "%s-%s" % (repo['name'], time) + with open("%s/%s.repo" % (treeconf_dir, name), 'w') as f: + f.write("[%s]\n" % name) + f.write("name=%s\n" % name) + f.write("baseurl=%s\n" % repo['baseurl']) + exclude = repo.get('exclude', None) + if exclude: + f.write("exclude=%s\n" % exclude) + gpgcheck = '1' if repo.get('gpgcheck', False) else '0' + f.write("gpgcheck=%s\n" % gpgcheck) + repos.append(name) + + original_repos = treeconf_content.get('repos', []) + if keep_original_sources: + treeconf_content['repos'] = original_repos + repos + else: + treeconf_content['repos'] = repos + + # update tree config to add new repos + with open(treeconf, 'w') as f: + json.dump(treeconf_content, f, indent=4) diff --git a/pungi/phases/ostree.py b/pungi/phases/ostree.py index bd45cd2f..ebe03d0d 100644 --- a/pungi/phases/ostree.py +++ b/pungi/phases/ostree.py @@ -1,12 +1,13 @@ # -*- coding: utf-8 -*- -import datetime +import copy import json import os from kobo.threads import ThreadPool, WorkerThread from .base import ConfigGuardedPhase from .. import util +from ..ostree.utils import get_ref_from_treefile, get_commitid_from_commitid_file from ..paths import translate_path from ..wrappers import kojiwrapper, scm @@ -53,7 +54,6 @@ class OSTreeThread(WorkerThread): self._clone_repo(repodir, config['config_url'], config.get('config_branch', 'master')) - treeconf = os.path.join(repodir, config['treefile']) source_repos = [{'name': '%s-%s' % (compose.compose_id, config['source_repo_from']), 'baseurl': source_repo}] @@ -72,30 +72,41 @@ class OSTreeThread(WorkerThread): source_repos = source_repos + extra_source_repos - keep_original_sources = config.get('keep_original_sources', False) - self._tweak_treeconf(treeconf, source_repos=source_repos, - keep_original_sources=keep_original_sources) + # copy the original config and update before save to a json file + new_config = copy.copy(config) + + # repos in configuration can have repo url set to variant UID, + # update it to have the actual url that we just translated. + new_config.update({'source_repo_from': source_repo}) + if extra_source_repos: + new_config.update({'extra_source_repos': extra_source_repos}) + + # remove unnecessary (for 'pungi-make-ostree tree' script ) elements + # from config, it doesn't hurt to have them, however remove them can + # reduce confusion + for k in ['ostree_repo', 'treefile', 'config_url', 'config_branch', + 'failable', 'version', 'update_summary']: + new_config.pop(k, None) + + extra_config_file = None + if new_config: + # write a json file to save the configuration, so 'pungi-make-ostree tree' + # can take use of it + extra_config_file = os.path.join(workdir, 'extra_config.json') + with open(extra_config_file, 'w') as f: + json.dump(new_config, f, indent=4) # Ensure target directory exists, otherwise Koji task will fail to # mount it. util.makedirs(config['ostree_repo']) - self._run_ostree_cmd(compose, variant, arch, config, repodir) - ref, commitid = self._get_commit_info(config, repodir) - if config.get('tag_ref', True) and ref and commitid: - # Let's write the tag out ourselves - heads_dir = os.path.join(config['ostree_repo'], 'refs', 'heads') - if not os.path.exists(heads_dir): - raise RuntimeError('Refs/heads did not exist in ostree repo') - - ref_path = os.path.join(heads_dir, ref) - if not os.path.exists(os.path.dirname(ref_path)): - os.makedirs(os.path.dirname(ref_path)) - - with open(ref_path, 'w') as f: - f.write(commitid + '\n') + self._run_ostree_cmd(compose, variant, arch, config, repodir, + extra_config_file=extra_config_file) if compose.notifier: + ref = get_ref_from_treefile(os.path.join(repodir, config['treefile'])) + # 'pungi-make-ostree tree' writes commitid to commitid.log in logdir + commitid = get_commitid_from_commitid_file(os.path.join(self.logdir, 'commitid.log')) compose.notifier.send('ostree', variant=variant.uid, arch=arch, @@ -104,26 +115,12 @@ class OSTreeThread(WorkerThread): self.pool.log_info('[DONE ] %s' % msg) - def _get_commit_info(self, config, config_repo): - ref = None - commitid = None - with open(os.path.join(config_repo, config['treefile']), 'r') as f: - try: - parsed = json.loads(f.read()) - ref = parsed['ref'] - except ValueError: - return None, None - if os.path.exists(os.path.join(self.logdir, 'commitid')): - with open(os.path.join(self.logdir, 'commitid'), 'r') as f: - commitid = f.read().replace('\n', '') - else: - return None, None - return ref, commitid - - def _run_ostree_cmd(self, compose, variant, arch, config, config_repo): + def _run_ostree_cmd(self, compose, variant, arch, config, config_repo, extra_config_file=None): cmd = [ 'pungi-make-ostree', - '--log-dir=%s' % os.path.join(self.logdir), + 'tree', + '--repo=%s' % config['ostree_repo'], + '--log-dir=%s' % self.logdir, '--treefile=%s' % os.path.join(config_repo, config['treefile']), ] @@ -131,12 +128,12 @@ class OSTreeThread(WorkerThread): if version: cmd.append('--version=%s' % version) + if extra_config_file: + cmd.append('--extra-config=%s' % extra_config_file) + if config.get('update_summary', False): cmd.append('--update-summary') - # positional argument: ostree_repo - cmd.append(config['ostree_repo']) - runroot_channel = compose.conf.get("runroot_channel") runroot_tag = compose.conf["runroot_tag"] @@ -157,43 +154,3 @@ class OSTreeThread(WorkerThread): def _clone_repo(self, repodir, url, branch): scm.get_dir_from_scm({'scm': 'git', 'repo': url, 'branch': branch, 'dir': '.'}, repodir, logger=self.pool._logger) - - def _tweak_treeconf(self, treeconf, source_repos, keep_original_sources=False): - """ - Update tree config file by adding new repos and remove existing repos - from the tree config file if 'keep_original_sources' is not enabled. - """ - # add this timestamp to repo name to get unique repo filename and repo name - # should be safe enough - time = datetime.datetime.now().strftime("%Y%m%d%H%M%S") - - treeconf_dir = os.path.dirname(treeconf) - with open(treeconf, 'r') as f: - treeconf_content = json.load(f) - - # backup the old tree config - os.rename(treeconf, '%s.%s.bak' % (treeconf, time)) - - repos = [] - for repo in source_repos: - name = "%s-%s" % (repo['name'], time) - with open("%s/%s.repo" % (treeconf_dir, name), 'w') as f: - f.write("[%s]\n" % name) - f.write("name=%s\n" % name) - f.write("baseurl=%s\n" % repo['baseurl']) - exclude = repo.get('exclude', None) - if exclude: - f.write("exclude=%s\n" % exclude) - gpgcheck = '1' if repo.get('gpgcheck', False) else '0' - f.write("gpgcheck=%s\n" % gpgcheck) - repos.append(name) - - original_repos = treeconf_content.get('repos', []) - if keep_original_sources: - treeconf_content['repos'] = original_repos + repos - else: - treeconf_content['repos'] = repos - - # update tree config to add new repos - with open(treeconf, 'w') as f: - json.dump(treeconf_content, f, indent=4) diff --git a/tests/test_ostree_phase.py b/tests/test_ostree_phase.py index bf01f6c1..0861d9e5 100644 --- a/tests/test_ostree_phase.py +++ b/tests/test_ostree_phase.py @@ -110,10 +110,12 @@ class OSTreeThreadTest(helpers.PungiTestCase): self.assertEqual(koji.get_runroot_cmd.call_args_list, [mock.call('rrt', 'x86_64', ['pungi-make-ostree', + 'tree', + '--repo=%s' % self.repo, '--log-dir=%s/logs/x86_64/Everything/ostree-1' % self.topdir, '--treefile=%s/fedora-atomic-docker-host.json' % ( self.topdir + '/work/ostree-1/config_repo'), - self.repo], + '--extra-config=%s/extra_config.json' % (self.topdir + '/work/ostree-1')], channel=None, mounts=[self.topdir, self.repo], packages=['pungi', 'ostree', 'rpm-ostree'], task_id=True, use_shell=True, new_chroot=True)]) @@ -121,17 +123,7 @@ class OSTreeThreadTest(helpers.PungiTestCase): [mock.call(koji.get_runroot_cmd.return_value, log_file=self.topdir + '/logs/x86_64/Everything/ostree-1/runroot.log')]) - repo_files = [] - for fp in os.listdir(os.path.join(self.topdir, 'work/ostree-1/config_repo')): - if fp.endswith('.repo'): - repo_files.append(fp) - - if fp not in ['fedora-rawhide.repo', 'fedora-24.repo', 'fedora-23.repo']: - with open(os.path.join(self.topdir, 'work/ostree-1/config_repo', fp)) as f: - self.assertIn('baseurl=http://example.com/Everything/$basearch/os', f.read()) - # test a new repo file created - self.assertEqual(len(repo_files), 4) - + self.assertTrue(os.path.isfile(os.path.join(self.topdir, 'work/ostree-1/extra_config.json'))) self.assertTrue(os.path.isdir(self.repo)) @mock.patch('pungi.wrappers.scm.get_dir_from_scm') @@ -181,7 +173,7 @@ class OSTreeThreadTest(helpers.PungiTestCase): koji = KojiWrapper.return_value koji.run_runroot_cmd.side_effect = self._mock_runroot( 0, - {'commitid': 'fca3465861a', + {'commitid.log': 'fca3465861a', 'create-ostree-repo.log': ['Doing work', 'fedora-atomic/25/x86_64 -> fca3465861a']}) t = ostree.OSTreeThread(self.pool) @@ -214,7 +206,7 @@ class OSTreeThreadTest(helpers.PungiTestCase): [mock.call('ostree', variant='Everything', arch='x86_64', - ref=None, + ref='fedora-atomic/25/x86_64', commitid=None)]) @mock.patch('pungi.wrappers.scm.get_dir_from_scm') @@ -254,10 +246,13 @@ class OSTreeThreadTest(helpers.PungiTestCase): self.assertEqual(koji.get_runroot_cmd.call_args_list, [mock.call('rrt', 'x86_64', ['pungi-make-ostree', + 'tree', + '--repo=%s' % self.repo, '--log-dir=%s/logs/x86_64/Everything/ostree-1' % self.topdir, '--treefile=%s/fedora-atomic-docker-host.json' % ( self.topdir + '/work/ostree-1/config_repo'), - '--update-summary', self.repo], + '--extra-config=%s/work/ostree-1/extra_config.json' % self.topdir, + '--update-summary'], channel=None, mounts=[self.topdir, self.repo], packages=['pungi', 'ostree', 'rpm-ostree'], task_id=True, use_shell=True, new_chroot=True)]) @@ -286,10 +281,13 @@ class OSTreeThreadTest(helpers.PungiTestCase): self.assertEqual(koji.get_runroot_cmd.call_args_list, [mock.call('rrt', 'x86_64', ['pungi-make-ostree', + 'tree', + '--repo=%s' % self.repo, '--log-dir=%s/logs/x86_64/Everything/ostree-1' % self.topdir, '--treefile=%s/fedora-atomic-docker-host.json' % ( self.topdir + '/work/ostree-1/config_repo'), - '--version=24', self.repo], + '--version=24', + '--extra-config=%s/work/ostree-1/extra_config.json' % self.topdir], channel=None, mounts=[self.topdir, self.repo], packages=['pungi', 'ostree', 'rpm-ostree'], task_id=True, use_shell=True, new_chroot=True)]) @@ -299,7 +297,7 @@ class OSTreeThreadTest(helpers.PungiTestCase): @mock.patch('pungi.wrappers.scm.get_dir_from_scm') @mock.patch('pungi.wrappers.kojiwrapper.KojiWrapper') - def test_run_with_extra_source_repos(self, KojiWrapper, get_dir_from_scm): + def test_write_extra_config_file(self, KojiWrapper, get_dir_from_scm): get_dir_from_scm.side_effect = self._dummy_config_repo koji = KojiWrapper.return_value @@ -319,55 +317,6 @@ class OSTreeThreadTest(helpers.PungiTestCase): 'exclude': 'systemd-container' } ], - 'config_url': 'https://git.fedorahosted.org/git/fedora-atomic.git', - 'config_branch': 'f24', - 'treefile': 'fedora-atomic-docker-host.json', - 'ostree_repo': self.repo - } - - t = ostree.OSTreeThread(self.pool) - - t.process((self.compose, self.compose.variants['Everything'], 'x86_64', cfg), 1) - - repo_files = [] - for fp in os.listdir(os.path.join(self.topdir, 'work/ostree-1/config_repo')): - if fp.endswith('.repo'): - repo_files.append(fp) - - if fp not in ['fedora-rawhide.repo', 'fedora-24.repo', 'fedora-23.repo']: - if fp.startswith('repo_a'): - with open(os.path.join(self.topdir, 'work/ostree-1/config_repo', fp)) as f: - # ignore timestamp in repo name while checking - content = f.read() - self.assertIn('[repo_a', content) - self.assertIn('name=repo_a', content) - self.assertIn('baseurl=http://url/to/repo/a', content) - self.assertIn('exclude=systemd-container', content) - self.assertIn('gpgcheck=0', content) - elif fp.startswith('Server'): - with open(os.path.join(self.topdir, 'work/ostree-1/config_repo', fp)) as f: - content = f.read() - self.assertIn('[Server', content) - self.assertIn('baseurl=http://example.com/Server/$basearch/os', content) - self.assertIn('exclude=systemd-container', content) - self.assertIn('gpgcheck=0', content) - else: - # this is the Everything repo (source_repo_from) - with open(os.path.join(self.topdir, 'work/ostree-1/config_repo', fp)) as f: - self.assertIn('baseurl=http://example.com/Everything/$basearch/os', f.read()) - # test new repos files created - self.assertEqual(len(repo_files), 3 + 1 + len(cfg['extra_source_repos'])) - - @mock.patch('pungi.wrappers.scm.get_dir_from_scm') - @mock.patch('pungi.wrappers.kojiwrapper.KojiWrapper') - def test_run_with_keep_original_source_repos(self, KojiWrapper, get_dir_from_scm): - get_dir_from_scm.side_effect = self._dummy_config_repo - - koji = KojiWrapper.return_value - koji.run_runroot_cmd.side_effect = self._mock_runroot(0) - - cfg = { - 'source_repo_from': 'Everything', 'keep_original_sources': True, 'config_url': 'https://git.fedorahosted.org/git/fedora-atomic.git', 'config_branch': 'f24', @@ -379,12 +328,13 @@ class OSTreeThreadTest(helpers.PungiTestCase): t.process((self.compose, self.compose.variants['Everything'], 'x86_64', cfg), 1) - treeconf_content = json.load(open(os.path.join(self.topdir, - 'work/ostree-1/config_repo', - cfg['treefile']))) - - # added 1 repo (Everything), have 4 (3 + 1) repos now - self.assertEqual(len(treeconf_content['repos']), 4) + extra_config_file = os.path.join(self.topdir, 'work/ostree-1/extra_config.json') + self.assertTrue(os.path.isfile(extra_config_file)) + extra_config = json.load(open(extra_config_file, 'r')) + self.assertTrue(extra_config.get('keep_original_sources', False)) + self.assertEqual(extra_config.get('source_repo_from', None), 'http://example.com/Everything/$basearch/os') + self.assertEqual(len(extra_config.get('extra_source_repos', [])), len(cfg['extra_source_repos'])) + self.assertEqual(extra_config.get('extra_source_repos').pop()['baseurl'], 'http://example.com/Server/$basearch/os') if __name__ == '__main__': unittest.main() diff --git a/tests/test_ostree_script.py b/tests/test_ostree_script.py index 89b18268..735a8fb7 100644 --- a/tests/test_ostree_script.py +++ b/tests/test_ostree_script.py @@ -6,7 +6,9 @@ import unittest import mock import os +import json import sys +import datetime sys.path.insert(0, os.path.join(os.path.dirname(__file__), '..')) sys.path.insert(0, os.path.join(os.path.dirname(__file__), 'bin')) @@ -15,16 +17,28 @@ from tests import helpers from pungi import ostree -class OstreeScriptTest(helpers.PungiTestCase): +class OstreeTreeScriptTest(helpers.PungiTestCase): + + def _make_dummy_config_dir(self, path): + helpers.touch(os.path.join(path, 'fedora-atomic-docker-host.json'), + json.dumps({'ref': 'fedora-atomic/25/x86_64', + 'repos': ['fedora-rawhide', 'fedora-24', 'fedora-23']})) + helpers.touch(os.path.join(path, 'fedora-rawhide.repo'), + '[fedora-rawhide]\nmirrorlist=mirror-mirror-on-the-wall') + helpers.touch(os.path.join(path, 'fedora-24.repo'), + '[fedora-24]\nmetalink=who-is-the-fairest-of-them-all') + helpers.touch(os.path.join(path, 'fedora-23.repo'), + '[fedora-23]\nbaseurl=why-not-zoidberg?') @mock.patch('kobo.shortcuts.run') def test_full_run(self, run): repo = os.path.join(self.topdir, 'atomic') ostree.main([ + 'tree', + '--repo=%s' % repo, '--log-dir=%s' % os.path.join(self.topdir, 'logs', 'Atomic'), '--treefile=%s/fedora-atomic-docker-host.json' % self.topdir, - repo, ]) self.maxDiff = None @@ -44,9 +58,10 @@ class OstreeScriptTest(helpers.PungiTestCase): os.mkdir(repo) ostree.main([ + 'tree', + '--repo=%s' % repo, '--log-dir=%s' % os.path.join(self.topdir, 'logs', 'Atomic'), '--treefile=%s/fedora-atomic-docker-host.json' % self.topdir, - repo, ]) self.maxDiff = None @@ -66,9 +81,10 @@ class OstreeScriptTest(helpers.PungiTestCase): helpers.touch(os.path.join(repo, 'initialized')) ostree.main([ + 'tree', + '--repo=%s' % repo, '--log-dir=%s' % os.path.join(self.topdir, 'logs', 'Atomic'), '--treefile=%s/fedora-atomic-docker-host.json' % self.topdir, - repo, ]) self.maxDiff = None @@ -84,10 +100,11 @@ class OstreeScriptTest(helpers.PungiTestCase): repo = os.path.join(self.topdir, 'atomic') ostree.main([ + 'tree', + '--repo=%s' % repo, '--log-dir=%s' % os.path.join(self.topdir, 'logs', 'Atomic'), '--treefile=%s/fedora-atomic-docker-host.json' % self.topdir, '--update-summary', - repo, ]) self.maxDiff = None @@ -107,10 +124,11 @@ class OstreeScriptTest(helpers.PungiTestCase): repo = os.path.join(self.topdir, 'atomic') ostree.main([ + 'tree', + '--repo=%s' % repo, '--log-dir=%s' % os.path.join(self.topdir, 'logs', 'Atomic'), '--treefile=%s/fedora-atomic-docker-host.json' % self.topdir, '--version=24', - repo, ]) self.maxDiff = None @@ -124,5 +142,129 @@ class OstreeScriptTest(helpers.PungiTestCase): self.topdir + '/fedora-atomic-docker-host.json'], logfile=self.topdir + '/logs/Atomic/create-ostree-repo.log', show_cmd=True, stdout=True)]) + @mock.patch('pungi.ostree.utils.datetime') + @mock.patch('kobo.shortcuts.run') + def test_extra_config_with_extra_repos(self, run, time): + time.datetime.now.return_value = datetime.datetime(2016, 1, 1, 1, 1) + timestamp = time.datetime.now().strftime("%Y%m%d%H%M%S") + + configdir = os.path.join(self.topdir, 'config') + self._make_dummy_config_dir(configdir) + treefile = os.path.join(configdir, 'fedora-atomic-docker-host.json') + + repo = os.path.join(self.topdir, 'atomic') + + extra_config_file = os.path.join(self.topdir, 'extra_config.json') + extra_config = { + "source_repo_from": "http://www.example.com/Server.repo", + "extra_source_repos": [ + { + "name": "optional", + "baseurl": "http://example.com/repo/x86_64/optional", + "exclude": "systemd-container", + "gpgcheck": False + }, + { + "name": "extra", + "baseurl": "http://example.com/repo/x86_64/extra", + } + ] + } + helpers.touch(extra_config_file, json.dumps(extra_config)) + + ostree.main([ + 'tree', + '--repo=%s' % repo, + '--log-dir=%s' % os.path.join(self.topdir, 'logs', 'Atomic'), + '--treefile=%s' % treefile, + '--extra-config=%s' % extra_config_file, + ]) + + source_repo_from_name = "source_repo_from-%s" % timestamp + source_repo_from_repo = os.path.join(configdir, "%s.repo" % source_repo_from_name) + self.assertTrue(os.path.isfile(source_repo_from_repo)) + with open(source_repo_from_repo, 'r') as f: + content = f.read() + self.assertIn("[%s]" % source_repo_from_name, content) + self.assertIn("name=%s" % source_repo_from_name, content) + self.assertIn("baseurl=http://www.example.com/Server.repo", content) + self.assertIn("gpgcheck=0", content) + + optional_repo_name = "optional-%s" % timestamp + optional_repo = os.path.join(configdir, "%s.repo" % optional_repo_name) + self.assertTrue(os.path.isfile(optional_repo)) + with open(optional_repo, 'r') as f: + content = f.read() + self.assertIn("[%s]" % optional_repo_name, content) + self.assertIn("name=%s" % optional_repo_name, content) + self.assertIn("baseurl=http://example.com/repo/x86_64/optional", content) + self.assertIn("gpgcheck=0", content) + + extra_repo_name = "extra-%s" % timestamp + extra_repo = os.path.join(configdir, "%s.repo" % extra_repo_name) + self.assertTrue(os.path.isfile(extra_repo)) + with open(extra_repo, 'r') as f: + content = f.read() + self.assertIn("[%s]" % extra_repo_name, content) + self.assertIn("name=%s" % extra_repo_name, content) + self.assertIn("baseurl=http://example.com/repo/x86_64/extra", content) + self.assertIn("gpgcheck=0", content) + + treeconf = json.load(open(treefile, 'r')) + repos = treeconf['repos'] + self.assertEqual(len(repos), 3) + for name in [source_repo_from_name, optional_repo_name, extra_repo_name]: + self.assertIn(name, repos) + + @mock.patch('pungi.ostree.utils.datetime') + @mock.patch('kobo.shortcuts.run') + def test_extra_config_with_keep_original_sources(self, run, time): + time.datetime.now.return_value = datetime.datetime(2016, 1, 1, 1, 1) + timestamp = time.datetime.now().strftime("%Y%m%d%H%M%S") + + configdir = os.path.join(self.topdir, 'config') + self._make_dummy_config_dir(configdir) + treefile = os.path.join(configdir, 'fedora-atomic-docker-host.json') + + repo = os.path.join(self.topdir, 'atomic') + + extra_config_file = os.path.join(self.topdir, 'extra_config.json') + extra_config = { + "source_repo_from": "http://www.example.com/Server.repo", + "extra_source_repos": [ + { + "name": "optional", + "baseurl": "http://example.com/repo/x86_64/optional", + "exclude": "systemd-container", + "gpgcheck": False + }, + { + "name": "extra", + "baseurl": "http://example.com/repo/x86_64/extra", + } + ], + "keep_original_sources": True + } + helpers.touch(extra_config_file, json.dumps(extra_config)) + + ostree.main([ + 'tree', + '--repo=%s' % repo, + '--log-dir=%s' % os.path.join(self.topdir, 'logs', 'Atomic'), + '--treefile=%s' % treefile, + '--extra-config=%s' % extra_config_file, + ]) + + source_repo_from_name = "source_repo_from-%s" % timestamp + optional_repo_name = "optional-%s" % timestamp + extra_repo_name = "extra-%s" % timestamp + + treeconf = json.load(open(treefile, 'r')) + repos = treeconf['repos'] + self.assertEqual(len(repos), 6) + for name in ['fedora-rawhide', 'fedora-24', 'fedora-23', + source_repo_from_name, optional_repo_name, extra_repo_name]: + self.assertIn(name, repos) + if __name__ == '__main__': unittest.main()