From e705cdd165a0542e2a021b08f13201f4c3b09a7c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lubom=C3=ADr=20Sedl=C3=A1=C5=99?= Date: Thu, 7 Apr 2016 14:33:37 +0200 Subject: [PATCH] [ostree] Move cloning repo back to compose box MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit and pass absolute path to the config into runroot. This could avoid the problem of not being able to clone the repo in runroot. Signed-off-by: Lubomír Sedlář --- pungi/ostree.py | 46 +------------------------------------ pungi/phases/ostree.py | 38 ++++++++++++++++++++++-------- tests/test_ostree_phase.py | 30 +++++++++++++++++------- tests/test_ostree_script.py | 21 +++-------------- 4 files changed, 55 insertions(+), 80 deletions(-) diff --git a/pungi/ostree.py b/pungi/ostree.py index c0c8a048..205b7299 100644 --- a/pungi/ostree.py +++ b/pungi/ostree.py @@ -8,11 +8,8 @@ It is expected to be runnable in Koji runroot. import argparse import os from kobo import shortcuts -import re import errno -from .wrappers import scm - def ensure_dir(path): try: @@ -46,61 +43,20 @@ def make_ostree_repo(repo, config, log_dir=None): show_cmd=True, logfile=log_file) -def clone_repo(repodir, url, branch): - scm.get_dir_from_scm( - {'scm': 'git', 'repo': url, 'branch': branch, 'dir': '.'}, repodir) - - -def tweak_mirrorlist(repodir, source_repo): - for file in os.listdir(repodir): - if file.endswith('.repo'): - tweak_file(os.path.join(repodir, file), source_repo) - - -def tweak_file(path, source_repo): - """Replace mirrorlist line in repo file with baseurl pointing to source_repo.""" - with open(path, 'r') as f: - contents = f.read() - replacement = 'baseurl={}'.format(source_repo) - contents = re.sub(r'^mirrorlist=.*$', replacement, contents, flags=re.MULTILINE) - with open(path, 'w') as f: - f.write(contents) - - -def prepare_config(workdir, config_url, config_branch, source_repo): - repodir = os.path.join(workdir, 'config_repo') - clone_repo(repodir, config_url, config_branch) - tweak_mirrorlist(repodir, source_repo) - return repodir - - def run(opts): - workdir = ensure_dir(opts.work_dir) - repodir = prepare_config(workdir, opts.config_url, opts.config_branch, - opts.source_repo) - ensure_dir(repodir) init_ostree_repo(opts.ostree_repo, log_dir=opts.log_dir) - treefile = os.path.join(repodir, opts.treefile) - make_ostree_repo(opts.ostree_repo, treefile, log_dir=opts.log_dir) + make_ostree_repo(opts.ostree_repo, opts.treefile, log_dir=opts.log_dir) def main(args=None): parser = argparse.ArgumentParser() parser.add_argument('--log-dir', help='where to log output') - parser.add_argument('--work-dir', required=True, - help='where to put temporary files') parser.add_argument('ostree_repo', metavar='OSTREE_REPO', help='where to put the ostree repo') parser.add_argument('--treefile', required=True, help='treefile for rpm-ostree') - parser.add_argument('--config-url', required=True, - help='git repository with the treefile') - parser.add_argument('--config-branch', default='master', - help='git branch to be used') - parser.add_argument('--source-repo', required=True, - help='yum repo used as source for') opts = parser.parse_args(args) diff --git a/pungi/phases/ostree.py b/pungi/phases/ostree.py index 8c46ac0c..3d738de8 100644 --- a/pungi/phases/ostree.py +++ b/pungi/phases/ostree.py @@ -2,11 +2,12 @@ import os from kobo.threads import ThreadPool, WorkerThread +import re from .base import ConfigGuardedPhase from .. import util from ..paths import translate_path -from ..wrappers import kojiwrapper +from ..wrappers import kojiwrapper, scm class OSTreePhase(ConfigGuardedPhase): @@ -44,25 +45,25 @@ class OSTreeThread(WorkerThread): def worker(self, compose, variant, arch, config): msg = 'OSTree phase for variant %s, arch %s' % (variant.uid, arch) self.pool.log_info('[BEGIN] %s' % msg) + workdir = compose.paths.work.topdir('ostree') self.logdir = compose.paths.log.topdir('{}/ostree'.format(arch)) + repodir = os.path.join(workdir, 'config_repo') source_variant = compose.variants[config['source_repo_from']] source_repo = translate_path(compose, compose.paths.compose.repository(arch, source_variant)) - self._run_ostree_cmd(compose, variant, arch, config, source_repo) + self._clone_repo(repodir, config['config_url'], config.get('config_branch', 'master')) + self._tweak_mirrorlist(repodir, source_repo) + + self._run_ostree_cmd(compose, variant, arch, config, repodir) self.pool.log_info('[DONE ] %s' % msg) - def _run_ostree_cmd(self, compose, variant, arch, config, source_repo): - workdir = os.path.join(compose.paths.work.topdir(arch), 'ostree') + def _run_ostree_cmd(self, compose, variant, arch, config, config_repo): cmd = [ 'pungi-make-ostree', '--log-dir={}'.format(self.logdir), - '--work-dir={}'.format(workdir), - '--treefile={}'.format(config['treefile']), - '--config-url={}'.format(config['config_url']), - '--config-branch={}'.format(config.get('config_branch', 'master')), - '--source-repo={}'.format(source_repo), + '--treefile={}'.format(os.path.join(config_repo, config['treefile'])), config['ostree_repo'] ] @@ -81,3 +82,22 @@ class OSTreeThread(WorkerThread): if output["retcode"] != 0: raise RuntimeError("Runroot task failed: %s. See %s for more details." % (output["task_id"], log_file)) + + def _clone_repo(self, repodir, url, branch): + scm.get_dir_from_scm({'scm': 'git', 'repo': url, 'branch': branch, 'dir': '.'}, + repodir, logger=self.pool._logger) + + def _tweak_mirrorlist(self, repodir, source_repo): + for file in os.listdir(repodir): + if file.endswith('.repo'): + tweak_file(os.path.join(repodir, file), source_repo) + + +def tweak_file(path, source_repo): + """Replace mirrorlist line in repo file with baseurl pointing to source_repo.""" + with open(path, 'r') as f: + contents = f.read() + replacement = 'baseurl={}'.format(source_repo) + contents = re.sub(r'^mirrorlist=.*$', replacement, contents, flags=re.MULTILINE) + with open(path, 'w') as f: + f.write(contents) diff --git a/tests/test_ostree_phase.py b/tests/test_ostree_phase.py index 2a460f5f..2da1f413 100755 --- a/tests/test_ostree_phase.py +++ b/tests/test_ostree_phase.py @@ -74,8 +74,15 @@ class OSTreePhaseTest(helpers.PungiTestCase): class OSTreeThreadTest(helpers.PungiTestCase): + def _dummy_config_repo(self, scm_dict, target, logger=None): + helpers.touch(os.path.join(target, 'fedora-atomic-docker-host.json')) + helpers.touch(os.path.join(target, 'fedora-rawhide.repo')) + + @mock.patch('pungi.wrappers.scm.get_dir_from_scm') @mock.patch('pungi.wrappers.kojiwrapper.KojiWrapper') - def test_run(self, KojiWrapper): + def test_run(self, KojiWrapper, get_dir_from_scm): + get_dir_from_scm.side_effect = self._dummy_config_repo + compose = helpers.DummyCompose(self.topdir, { 'koji_profile': 'koji', 'runroot_tag': 'rrt', @@ -99,15 +106,16 @@ class OSTreeThreadTest(helpers.PungiTestCase): t.process((compose, compose.variants['Everything'], 'x86_64', cfg), 1) + self.assertEqual(get_dir_from_scm.call_args_list, + [mock.call({'scm': 'git', 'repo': 'https://git.fedorahosted.org/git/fedora-atomic.git', + 'branch': 'f24', 'dir': '.'}, + self.topdir + '/work/ostree/config_repo', logger=pool._logger)]) self.assertEqual(koji.get_runroot_cmd.call_args_list, [mock.call('rrt', 'x86_64', ['pungi-make-ostree', '--log-dir={}/logs/x86_64/ostree'.format(self.topdir), - '--work-dir={}/work/x86_64/ostree'.format(self.topdir), - '--treefile=fedora-atomic-docker-host.json', - '--config-url=https://git.fedorahosted.org/git/fedora-atomic.git', - '--config-branch=f24', - '--source-repo={}/compose/Everything/x86_64/os'.format(self.topdir), + '--treefile={}/fedora-atomic-docker-host.json'.format( + self.topdir + '/work/ostree/config_repo'), '/other/place/for/atomic'], channel=None, mounts=[self.topdir, '/other/place/for/atomic'], packages=['pungi', 'ostree', 'rpm-ostree'], @@ -116,8 +124,11 @@ class OSTreeThreadTest(helpers.PungiTestCase): [mock.call(koji.get_runroot_cmd.return_value, log_file=self.topdir + '/logs/x86_64/ostree/runroot.log')]) + @mock.patch('pungi.wrappers.scm.get_dir_from_scm') @mock.patch('pungi.wrappers.kojiwrapper.KojiWrapper') - def test_run_fail(self, KojiWrapper): + def test_run_fail(self, KojiWrapper, get_dir_from_scm): + get_dir_from_scm.side_effect = self._dummy_config_repo + compose = helpers.DummyCompose(self.topdir, { 'koji_profile': 'koji', 'runroot_tag': 'rrt', @@ -150,8 +161,11 @@ class OSTreeThreadTest(helpers.PungiTestCase): self.topdir + '/logs/x86_64/ostree/runroot.log')) ]) + @mock.patch('pungi.wrappers.scm.get_dir_from_scm') @mock.patch('pungi.wrappers.kojiwrapper.KojiWrapper') - def test_run_handle_exception(self, KojiWrapper): + def test_run_handle_exception(self, KojiWrapper, get_dir_from_scm): + get_dir_from_scm.side_effect = self._dummy_config_repo + compose = helpers.DummyCompose(self.topdir, { 'koji_profile': 'koji', 'runroot_tag': 'rrt', diff --git a/tests/test_ostree_script.py b/tests/test_ostree_script.py index 7b596807..15034e85 100755 --- a/tests/test_ostree_script.py +++ b/tests/test_ostree_script.py @@ -17,38 +17,23 @@ from pungi import ostree class OstreeScriptTest(helpers.PungiTestCase): - def _dummy_config_repo(self, scm_dict, target, logger=None): - helpers.touch(os.path.join(target, 'fedora-atomic-docker-host.json')) - helpers.touch(os.path.join(target, 'fedora-rawhide.repo')) - @mock.patch('kobo.shortcuts.run') - @mock.patch('pungi.wrappers.scm.get_dir_from_scm') - def test_full_run(self, get_dir_from_scm, run): - get_dir_from_scm.side_effect = self._dummy_config_repo - + def test_full_run(self, run): repo = os.path.join(self.topdir, 'atomic') ostree.main([ '--log-dir={}'.format(os.path.join(self.topdir, 'logs', 'Atomic')), - '--work-dir={}'.format(self.topdir), - '--treefile={}'.format('fedora-atomic-docker-host.json'), - '--config-url=https://git.fedorahosted.org/git/fedora-atomic.git', - '--config-branch=f24', - '--source-repo=https://kojipkgs.fedoraproject.org/repo', + '--treefile={}/fedora-atomic-docker-host.json'.format(self.topdir), repo, ]) self.maxDiff = None - self.assertEqual(get_dir_from_scm.call_args_list, - [mock.call({'scm': 'git', 'repo': 'https://git.fedorahosted.org/git/fedora-atomic.git', - 'branch': 'f24', 'dir': '.'}, - self.topdir + '/config_repo')]) self.assertItemsEqual( run.call_args_list, [mock.call(['ostree', 'init', '--repo={}'.format(repo), '--mode=archive-z2'], logfile=self.topdir + '/logs/Atomic/init-ostree-repo.log', show_cmd=True), mock.call(['rpm-ostree', 'compose', 'tree', '--repo={}'.format(repo), - self.topdir + '/config_repo/fedora-atomic-docker-host.json'], + self.topdir + '/fedora-atomic-docker-host.json'], logfile=self.topdir + '/logs/Atomic/create-ostree-repo.log', show_cmd=True)])