scm-wrapper: Allow running command after git clone
When a file should be obtained from a git repository, allow running an arbitrary command (like `make`) after clone but before copying the files out. This only works for the Git backend. The downside is that a clone is needed and we can no longer use `git archive` to speed things up. Fixes: https://pagure.io/pungi/issue/5 Signed-off-by: Lubomír Sedlář <lsedlar@redhat.com>
This commit is contained in:
Lubomír Sedlář
parent
f9640ae0b4
commit
cae202c17b
|
|
@ -422,6 +422,7 @@ def _make_schema():
|
|||
"branch": {"$ref": "#/definitions/optional_string"},
|
||||
"file": {"type": "string"},
|
||||
"dir": {"type": "string"},
|
||||
"command": {"type": "string"},
|
||||
},
|
||||
"additionalProperties": False,
|
||||
},
|
||||
|
|
|
|||
|
|
@ -27,8 +27,9 @@ from pungi.util import (explode_rpm_package, makedirs, copy_all, temp_dir,
|
|||
|
||||
|
||||
class ScmBase(kobo.log.LoggingBase):
|
||||
def __init__(self, logger=None):
|
||||
def __init__(self, logger=None, command=None):
|
||||
kobo.log.LoggingBase.__init__(self, logger=logger)
|
||||
self.command = command
|
||||
|
||||
@retry(interval=60, timeout=300, wait_on=RuntimeError)
|
||||
def retry_run(self, cmd, **kwargs):
|
||||
|
|
@ -39,6 +40,15 @@ class ScmBase(kobo.log.LoggingBase):
|
|||
|
||||
return run(cmd, **kwargs)
|
||||
|
||||
def run_process_command(self, cwd):
|
||||
if self.command:
|
||||
self.log_debug('Running "%s"' % self.command)
|
||||
retcode, output = run(self.command, workdir=cwd, can_fail=True)
|
||||
if retcode != 0:
|
||||
self.log_error('Output was: "%s"' % output)
|
||||
raise RuntimeError('%r failed with exit code %s'
|
||||
% (self.command, retcode))
|
||||
|
||||
|
||||
class FileWrapper(ScmBase):
|
||||
def export_dir(self, scm_root, scm_dir, target_dir, scm_branch=None):
|
||||
|
|
@ -104,10 +114,11 @@ class GitWrapper(ScmBase):
|
|||
% (pipes.quote(scm_root), pipes.quote(scm_branch), pipes.quote(scm_dir)))
|
||||
# git archive is not supported by http/https
|
||||
# or by smart http https://git-scm.com/book/en/v2/Git-on-the-Server-Smart-HTTP
|
||||
if scm_root.startswith("http"):
|
||||
if scm_root.startswith("http") or self.command:
|
||||
cmd = ("/usr/bin/git clone --depth 1 --branch=%s %s %s"
|
||||
% (pipes.quote(scm_branch), pipes.quote(scm_root), pipes.quote(tmp_dir)))
|
||||
self.retry_run(cmd, workdir=tmp_dir, show_cmd=True)
|
||||
self.run_process_command(tmp_dir)
|
||||
|
||||
copy_all(os.path.join(tmp_dir, scm_dir), target_dir)
|
||||
|
||||
|
|
@ -127,10 +138,11 @@ class GitWrapper(ScmBase):
|
|||
% (pipes.quote(scm_root), pipes.quote(scm_branch), pipes.quote(scm_file)))
|
||||
# git archive is not supported by http/https
|
||||
# or by smart http https://git-scm.com/book/en/v2/Git-on-the-Server-Smart-HTTP
|
||||
if scm_root.startswith("http"):
|
||||
if scm_root.startswith("http") or self.command:
|
||||
cmd = ("/usr/bin/git clone --depth 1 --branch=%s %s %s"
|
||||
% (pipes.quote(scm_branch), pipes.quote(scm_root), pipes.quote(tmp_dir)))
|
||||
self.retry_run(cmd, workdir=tmp_dir, show_cmd=True)
|
||||
self.run_process_command(tmp_dir)
|
||||
|
||||
makedirs(target_dir)
|
||||
shutil.copy2(os.path.join(tmp_dir, scm_file), target_path)
|
||||
|
|
@ -219,13 +231,15 @@ def get_file_from_scm(scm_dict, target_path, logger=None):
|
|||
scm_repo = None
|
||||
scm_file = os.path.abspath(scm_dict)
|
||||
scm_branch = None
|
||||
command = None
|
||||
else:
|
||||
scm_type = scm_dict["scm"]
|
||||
scm_repo = scm_dict["repo"]
|
||||
scm_file = scm_dict["file"]
|
||||
scm_branch = scm_dict.get("branch", None)
|
||||
command = scm_dict.get('command')
|
||||
|
||||
scm = _get_wrapper(scm_type, logger=logger)
|
||||
scm = _get_wrapper(scm_type, logger=logger, command=command)
|
||||
|
||||
files_copied = []
|
||||
for i in force_list(scm_file):
|
||||
|
|
@ -270,13 +284,15 @@ def get_dir_from_scm(scm_dict, target_path, logger=None):
|
|||
scm_repo = None
|
||||
scm_dir = os.path.abspath(scm_dict)
|
||||
scm_branch = None
|
||||
command = None
|
||||
else:
|
||||
scm_type = scm_dict["scm"]
|
||||
scm_repo = scm_dict.get("repo", None)
|
||||
scm_dir = scm_dict["dir"]
|
||||
scm_branch = scm_dict.get("branch", None)
|
||||
command = scm_dict.get("command")
|
||||
|
||||
scm = _get_wrapper(scm_type, logger=logger)
|
||||
scm = _get_wrapper(scm_type, logger=logger, command=command)
|
||||
|
||||
with temp_dir(prefix="scm_checkout_") as tmp_dir:
|
||||
scm.export_dir(scm_repo, scm_dir, scm_branch=scm_branch, target_dir=tmp_dir)
|
||||
|
|
|
|||
|
|
@ -109,8 +109,9 @@ class GitSCMTestCase(SCMBaseTest):
|
|||
commands = []
|
||||
|
||||
def process(cmd, workdir=None, **kwargs):
|
||||
fname = cmd.split('|')[0].strip().split(' ')[-1]
|
||||
touch(os.path.join(workdir, fname))
|
||||
if cmd.startswith('/usr/bin/git'):
|
||||
fname = cmd.split('|')[0].strip().split(' ')[-1]
|
||||
touch(os.path.join(workdir, fname))
|
||||
commands.append(cmd)
|
||||
|
||||
run.side_effect = process
|
||||
|
|
@ -124,6 +125,76 @@ class GitSCMTestCase(SCMBaseTest):
|
|||
commands,
|
||||
['/usr/bin/git archive --remote=git://example.com/git/repo.git master some_file.txt | tar xf -'])
|
||||
|
||||
@mock.patch('pungi.wrappers.scm.run')
|
||||
def test_get_file_generated_by_command_via_git(self, run):
|
||||
commands = []
|
||||
|
||||
def process(cmd, workdir=None, **kwargs):
|
||||
if cmd.startswith('/usr/bin/git'):
|
||||
checkout = cmd.split(' ')[-1]
|
||||
touch(os.path.join(checkout, 'some_file.txt'))
|
||||
commands.append(cmd)
|
||||
return 0, ''
|
||||
|
||||
run.side_effect = process
|
||||
|
||||
retval = scm.get_file_from_scm({'scm': 'git',
|
||||
'repo': 'git://example.com/git/repo.git',
|
||||
'file': 'some_file.txt',
|
||||
'command': 'make'},
|
||||
self.destdir)
|
||||
self.assertStructure(retval, ['some_file.txt'])
|
||||
self.assertRegexpMatches(
|
||||
commands[0],
|
||||
r'/usr/bin/git clone --depth 1 --branch=master git://example.com/git/repo.git /tmp/.+')
|
||||
self.assertEqual(commands[1:], ['make'])
|
||||
|
||||
@mock.patch('pungi.wrappers.scm.run')
|
||||
def test_get_file_and_fail_to_generate(self, run):
|
||||
commands = []
|
||||
|
||||
def process(cmd, workdir=None, **kwargs):
|
||||
if cmd.startswith('/usr/bin/git'):
|
||||
checkout = cmd.split(' ')[-1]
|
||||
touch(os.path.join(checkout, 'some_file.txt'))
|
||||
commands.append(cmd)
|
||||
return len(commands), 'output'
|
||||
|
||||
run.side_effect = process
|
||||
|
||||
with self.assertRaises(RuntimeError) as ctx:
|
||||
scm.get_file_from_scm({'scm': 'git',
|
||||
'repo': 'git://example.com/git/repo.git',
|
||||
'file': 'some_file.txt',
|
||||
'command': 'make'},
|
||||
self.destdir)
|
||||
|
||||
self.assertEqual(str(ctx.exception), "'make' failed with exit code 2")
|
||||
|
||||
@mock.patch('pungi.wrappers.scm.run')
|
||||
def test_get_file_generated_by_command_via_https(self, run):
|
||||
commands = []
|
||||
|
||||
def process(cmd, workdir=None, **kwargs):
|
||||
if cmd.startswith('/usr/bin/git'):
|
||||
checkout = cmd.split(' ')[-1]
|
||||
touch(os.path.join(checkout, 'some_file.txt'))
|
||||
commands.append(cmd)
|
||||
return 0, ''
|
||||
|
||||
run.side_effect = process
|
||||
|
||||
retval = scm.get_file_from_scm({'scm': 'git',
|
||||
'repo': 'https://example.com/git/repo.git',
|
||||
'file': 'some_file.txt',
|
||||
'command': 'make'},
|
||||
self.destdir)
|
||||
self.assertStructure(retval, ['some_file.txt'])
|
||||
self.assertRegexpMatches(
|
||||
commands[0],
|
||||
r'/usr/bin/git clone --depth 1 --branch=master https://example.com/git/repo.git /tmp/.+')
|
||||
self.assertEqual(commands[1:], ['make'])
|
||||
|
||||
@mock.patch('pungi.wrappers.scm.run')
|
||||
def test_get_file_via_https(self, run):
|
||||
commands = []
|
||||
|
|
@ -168,6 +239,32 @@ class GitSCMTestCase(SCMBaseTest):
|
|||
commands,
|
||||
['/usr/bin/git archive --remote=git://example.com/git/repo.git master subdir | tar xf -'])
|
||||
|
||||
@mock.patch('pungi.wrappers.scm.run')
|
||||
def test_get_dir_via_git_and_generate(self, run):
|
||||
commands = []
|
||||
|
||||
def process(cmd, workdir=None, **kwargs):
|
||||
if cmd.startswith('/usr/bin/git'):
|
||||
checkout = cmd.split(' ')[-1]
|
||||
touch(os.path.join(checkout, 'subdir', 'first'))
|
||||
touch(os.path.join(checkout, 'subdir', 'second'))
|
||||
commands.append(cmd)
|
||||
return 0, ''
|
||||
|
||||
run.side_effect = process
|
||||
|
||||
retval = scm.get_dir_from_scm({'scm': 'git',
|
||||
'repo': 'git://example.com/git/repo.git',
|
||||
'dir': 'subdir',
|
||||
'command': 'make'},
|
||||
self.destdir)
|
||||
self.assertStructure(retval, ['first', 'second'])
|
||||
|
||||
self.assertRegexpMatches(
|
||||
commands[0],
|
||||
r'/usr/bin/git clone --depth 1 --branch=master git://example.com/git/repo.git /tmp/.+')
|
||||
self.assertEqual(commands[1:], ['make'])
|
||||
|
||||
@mock.patch('pungi.wrappers.scm.run')
|
||||
def test_get_dir_via_https(self, run):
|
||||
commands = []
|
||||
|
|
@ -190,6 +287,32 @@ class GitSCMTestCase(SCMBaseTest):
|
|||
commands[0],
|
||||
r'/usr/bin/git clone --depth 1 --branch=master https://example.com/git/repo.git /tmp/.+')
|
||||
|
||||
@mock.patch('pungi.wrappers.scm.run')
|
||||
def test_get_dir_via_https_and_generate(self, run):
|
||||
commands = []
|
||||
|
||||
def process(cmd, workdir=None, **kwargs):
|
||||
if cmd.startswith('/usr/bin/git'):
|
||||
checkout = cmd.split(' ')[-1]
|
||||
touch(os.path.join(checkout, 'subdir', 'first'))
|
||||
touch(os.path.join(checkout, 'subdir', 'second'))
|
||||
commands.append(cmd)
|
||||
return 0, ''
|
||||
|
||||
run.side_effect = process
|
||||
|
||||
retval = scm.get_dir_from_scm({'scm': 'git',
|
||||
'repo': 'https://example.com/git/repo.git',
|
||||
'dir': 'subdir',
|
||||
'command': 'make'},
|
||||
self.destdir)
|
||||
self.assertStructure(retval, ['first', 'second'])
|
||||
|
||||
self.assertRegexpMatches(
|
||||
commands[0],
|
||||
r'/usr/bin/git clone --depth 1 --branch=master https://example.com/git/repo.git /tmp/.+')
|
||||
self.assertEqual(commands[1:], ['make'])
|
||||
|
||||
|
||||
class RpmSCMTestCase(SCMBaseTest):
|
||||
def setUp(self):
|
||||
|
|
|
|||
Loading…
Reference in New Issue