From cae202c17b0a34419313d9fe5095311c8419c729 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lubom=C3=ADr=20Sedl=C3=A1=C5=99?= Date: Tue, 8 Aug 2017 13:14:48 +0200 Subject: [PATCH] scm-wrapper: Allow running command after git clone MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit When a file should be obtained from a git repository, allow running an arbitrary command (like `make`) after clone but before copying the files out. This only works for the Git backend. The downside is that a clone is needed and we can no longer use `git archive` to speed things up. Fixes: https://pagure.io/pungi/issue/5 Signed-off-by: Lubomír Sedlář --- pungi/checks.py | 1 + pungi/wrappers/scm.py | 26 +++++++-- tests/test_scm.py | 127 +++++++++++++++++++++++++++++++++++++++++- 3 files changed, 147 insertions(+), 7 deletions(-) diff --git a/pungi/checks.py b/pungi/checks.py index 5dbca6d7..f6f87f67 100644 --- a/pungi/checks.py +++ b/pungi/checks.py @@ -422,6 +422,7 @@ def _make_schema(): "branch": {"$ref": "#/definitions/optional_string"}, "file": {"type": "string"}, "dir": {"type": "string"}, + "command": {"type": "string"}, }, "additionalProperties": False, }, diff --git a/pungi/wrappers/scm.py b/pungi/wrappers/scm.py index 4fceab5d..04dc8290 100644 --- a/pungi/wrappers/scm.py +++ b/pungi/wrappers/scm.py @@ -27,8 +27,9 @@ from pungi.util import (explode_rpm_package, makedirs, copy_all, temp_dir, class ScmBase(kobo.log.LoggingBase): - def __init__(self, logger=None): + def __init__(self, logger=None, command=None): kobo.log.LoggingBase.__init__(self, logger=logger) + self.command = command @retry(interval=60, timeout=300, wait_on=RuntimeError) def retry_run(self, cmd, **kwargs): @@ -39,6 +40,15 @@ class ScmBase(kobo.log.LoggingBase): return run(cmd, **kwargs) + def run_process_command(self, cwd): + if self.command: + self.log_debug('Running "%s"' % self.command) + retcode, output = run(self.command, workdir=cwd, can_fail=True) + if retcode != 0: + self.log_error('Output was: "%s"' % output) + raise RuntimeError('%r failed with exit code %s' + % (self.command, retcode)) + class FileWrapper(ScmBase): def export_dir(self, scm_root, scm_dir, target_dir, scm_branch=None): @@ -104,10 +114,11 @@ class GitWrapper(ScmBase): % (pipes.quote(scm_root), pipes.quote(scm_branch), pipes.quote(scm_dir))) # git archive is not supported by http/https # or by smart http https://git-scm.com/book/en/v2/Git-on-the-Server-Smart-HTTP - if scm_root.startswith("http"): + if scm_root.startswith("http") or self.command: cmd = ("/usr/bin/git clone --depth 1 --branch=%s %s %s" % (pipes.quote(scm_branch), pipes.quote(scm_root), pipes.quote(tmp_dir))) self.retry_run(cmd, workdir=tmp_dir, show_cmd=True) + self.run_process_command(tmp_dir) copy_all(os.path.join(tmp_dir, scm_dir), target_dir) @@ -127,10 +138,11 @@ class GitWrapper(ScmBase): % (pipes.quote(scm_root), pipes.quote(scm_branch), pipes.quote(scm_file))) # git archive is not supported by http/https # or by smart http https://git-scm.com/book/en/v2/Git-on-the-Server-Smart-HTTP - if scm_root.startswith("http"): + if scm_root.startswith("http") or self.command: cmd = ("/usr/bin/git clone --depth 1 --branch=%s %s %s" % (pipes.quote(scm_branch), pipes.quote(scm_root), pipes.quote(tmp_dir))) self.retry_run(cmd, workdir=tmp_dir, show_cmd=True) + self.run_process_command(tmp_dir) makedirs(target_dir) shutil.copy2(os.path.join(tmp_dir, scm_file), target_path) @@ -219,13 +231,15 @@ def get_file_from_scm(scm_dict, target_path, logger=None): scm_repo = None scm_file = os.path.abspath(scm_dict) scm_branch = None + command = None else: scm_type = scm_dict["scm"] scm_repo = scm_dict["repo"] scm_file = scm_dict["file"] scm_branch = scm_dict.get("branch", None) + command = scm_dict.get('command') - scm = _get_wrapper(scm_type, logger=logger) + scm = _get_wrapper(scm_type, logger=logger, command=command) files_copied = [] for i in force_list(scm_file): @@ -270,13 +284,15 @@ def get_dir_from_scm(scm_dict, target_path, logger=None): scm_repo = None scm_dir = os.path.abspath(scm_dict) scm_branch = None + command = None else: scm_type = scm_dict["scm"] scm_repo = scm_dict.get("repo", None) scm_dir = scm_dict["dir"] scm_branch = scm_dict.get("branch", None) + command = scm_dict.get("command") - scm = _get_wrapper(scm_type, logger=logger) + scm = _get_wrapper(scm_type, logger=logger, command=command) with temp_dir(prefix="scm_checkout_") as tmp_dir: scm.export_dir(scm_repo, scm_dir, scm_branch=scm_branch, target_dir=tmp_dir) diff --git a/tests/test_scm.py b/tests/test_scm.py index 991a6ad1..3ad54ede 100644 --- a/tests/test_scm.py +++ b/tests/test_scm.py @@ -109,8 +109,9 @@ class GitSCMTestCase(SCMBaseTest): commands = [] def process(cmd, workdir=None, **kwargs): - fname = cmd.split('|')[0].strip().split(' ')[-1] - touch(os.path.join(workdir, fname)) + if cmd.startswith('/usr/bin/git'): + fname = cmd.split('|')[0].strip().split(' ')[-1] + touch(os.path.join(workdir, fname)) commands.append(cmd) run.side_effect = process @@ -124,6 +125,76 @@ class GitSCMTestCase(SCMBaseTest): commands, ['/usr/bin/git archive --remote=git://example.com/git/repo.git master some_file.txt | tar xf -']) + @mock.patch('pungi.wrappers.scm.run') + def test_get_file_generated_by_command_via_git(self, run): + commands = [] + + def process(cmd, workdir=None, **kwargs): + if cmd.startswith('/usr/bin/git'): + checkout = cmd.split(' ')[-1] + touch(os.path.join(checkout, 'some_file.txt')) + commands.append(cmd) + return 0, '' + + run.side_effect = process + + retval = scm.get_file_from_scm({'scm': 'git', + 'repo': 'git://example.com/git/repo.git', + 'file': 'some_file.txt', + 'command': 'make'}, + self.destdir) + self.assertStructure(retval, ['some_file.txt']) + self.assertRegexpMatches( + commands[0], + r'/usr/bin/git clone --depth 1 --branch=master git://example.com/git/repo.git /tmp/.+') + self.assertEqual(commands[1:], ['make']) + + @mock.patch('pungi.wrappers.scm.run') + def test_get_file_and_fail_to_generate(self, run): + commands = [] + + def process(cmd, workdir=None, **kwargs): + if cmd.startswith('/usr/bin/git'): + checkout = cmd.split(' ')[-1] + touch(os.path.join(checkout, 'some_file.txt')) + commands.append(cmd) + return len(commands), 'output' + + run.side_effect = process + + with self.assertRaises(RuntimeError) as ctx: + scm.get_file_from_scm({'scm': 'git', + 'repo': 'git://example.com/git/repo.git', + 'file': 'some_file.txt', + 'command': 'make'}, + self.destdir) + + self.assertEqual(str(ctx.exception), "'make' failed with exit code 2") + + @mock.patch('pungi.wrappers.scm.run') + def test_get_file_generated_by_command_via_https(self, run): + commands = [] + + def process(cmd, workdir=None, **kwargs): + if cmd.startswith('/usr/bin/git'): + checkout = cmd.split(' ')[-1] + touch(os.path.join(checkout, 'some_file.txt')) + commands.append(cmd) + return 0, '' + + run.side_effect = process + + retval = scm.get_file_from_scm({'scm': 'git', + 'repo': 'https://example.com/git/repo.git', + 'file': 'some_file.txt', + 'command': 'make'}, + self.destdir) + self.assertStructure(retval, ['some_file.txt']) + self.assertRegexpMatches( + commands[0], + r'/usr/bin/git clone --depth 1 --branch=master https://example.com/git/repo.git /tmp/.+') + self.assertEqual(commands[1:], ['make']) + @mock.patch('pungi.wrappers.scm.run') def test_get_file_via_https(self, run): commands = [] @@ -168,6 +239,32 @@ class GitSCMTestCase(SCMBaseTest): commands, ['/usr/bin/git archive --remote=git://example.com/git/repo.git master subdir | tar xf -']) + @mock.patch('pungi.wrappers.scm.run') + def test_get_dir_via_git_and_generate(self, run): + commands = [] + + def process(cmd, workdir=None, **kwargs): + if cmd.startswith('/usr/bin/git'): + checkout = cmd.split(' ')[-1] + touch(os.path.join(checkout, 'subdir', 'first')) + touch(os.path.join(checkout, 'subdir', 'second')) + commands.append(cmd) + return 0, '' + + run.side_effect = process + + retval = scm.get_dir_from_scm({'scm': 'git', + 'repo': 'git://example.com/git/repo.git', + 'dir': 'subdir', + 'command': 'make'}, + self.destdir) + self.assertStructure(retval, ['first', 'second']) + + self.assertRegexpMatches( + commands[0], + r'/usr/bin/git clone --depth 1 --branch=master git://example.com/git/repo.git /tmp/.+') + self.assertEqual(commands[1:], ['make']) + @mock.patch('pungi.wrappers.scm.run') def test_get_dir_via_https(self, run): commands = [] @@ -190,6 +287,32 @@ class GitSCMTestCase(SCMBaseTest): commands[0], r'/usr/bin/git clone --depth 1 --branch=master https://example.com/git/repo.git /tmp/.+') + @mock.patch('pungi.wrappers.scm.run') + def test_get_dir_via_https_and_generate(self, run): + commands = [] + + def process(cmd, workdir=None, **kwargs): + if cmd.startswith('/usr/bin/git'): + checkout = cmd.split(' ')[-1] + touch(os.path.join(checkout, 'subdir', 'first')) + touch(os.path.join(checkout, 'subdir', 'second')) + commands.append(cmd) + return 0, '' + + run.side_effect = process + + retval = scm.get_dir_from_scm({'scm': 'git', + 'repo': 'https://example.com/git/repo.git', + 'dir': 'subdir', + 'command': 'make'}, + self.destdir) + self.assertStructure(retval, ['first', 'second']) + + self.assertRegexpMatches( + commands[0], + r'/usr/bin/git clone --depth 1 --branch=master https://example.com/git/repo.git /tmp/.+') + self.assertEqual(commands[1:], ['make']) + class RpmSCMTestCase(SCMBaseTest): def setUp(self):