almalinux/pungi
15
1
Fork 0
Code Issues Pull Requests Releases Activity

scm-wrapper: Allow running command after git clone

Browse Source 
When a file should be obtained from a git repository, allow running an
arbitrary command (like `make`) after clone but before copying the files
out. This only works for the Git backend.

The downside is that a clone is needed and we can no longer use `git
archive` to speed things up.

Fixes: https://pagure.io/pungi/issue/5
Signed-off-by: Lubomír Sedlář <lsedlar@redhat.com>
This commit is contained in:
Lubomír Sedlář 2017-08-08 13:14:48 +02:00
parent f9640ae0b4
commit cae202c17b
3 changed files with 147 additions and 7 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
pungi/checks.py
View File

@ -422,6 +422,7 @@ def _make_schema():
"branch": {"$ref": "#/definitions/optional_string"},
"file": {"type": "string"},
"dir": {"type": "string"},
"command": {"type": "string"},
},
"additionalProperties": False,
},

26
pungi/wrappers/scm.py
View File

@ -27,8 +27,9 @@ from pungi.util import (explode_rpm_package, makedirs, copy_all, temp_dir,
class ScmBase(kobo.log.LoggingBase):
def __init__(self, logger=None):
def __init__(self, logger=None, command=None):
kobo.log.LoggingBase.__init__(self, logger=logger)
self.command = command
@retry(interval=60, timeout=300, wait_on=RuntimeError)
def retry_run(self, cmd, **kwargs):
@ -39,6 +40,15 @@ class ScmBase(kobo.log.LoggingBase):
return run(cmd, **kwargs)
def run_process_command(self, cwd):
if self.command:
self.log_debug('Running "%s"' % self.command)
retcode, output = run(self.command, workdir=cwd, can_fail=True)
if retcode != 0:
self.log_error('Output was: "%s"' % output)
raise RuntimeError('%r failed with exit code %s'
% (self.command, retcode))
class FileWrapper(ScmBase):
def export_dir(self, scm_root, scm_dir, target_dir, scm_branch=None):
@ -104,10 +114,11 @@ class GitWrapper(ScmBase):
% (pipes.quote(scm_root), pipes.quote(scm_branch), pipes.quote(scm_dir)))
# git archive is not supported by http/https
# or by smart http https://git-scm.com/book/en/v2/Git-on-the-Server-Smart-HTTP
if scm_root.startswith("http"):
if scm_root.startswith("http") or self.command:
cmd = ("/usr/bin/git clone --depth 1 --branch=%s %s %s"
% (pipes.quote(scm_branch), pipes.quote(scm_root), pipes.quote(tmp_dir)))
self.retry_run(cmd, workdir=tmp_dir, show_cmd=True)
self.run_process_command(tmp_dir)
copy_all(os.path.join(tmp_dir, scm_dir), target_dir)
@ -127,10 +138,11 @@ class GitWrapper(ScmBase):
% (pipes.quote(scm_root), pipes.quote(scm_branch), pipes.quote(scm_file)))
# git archive is not supported by http/https
# or by smart http https://git-scm.com/book/en/v2/Git-on-the-Server-Smart-HTTP
if scm_root.startswith("http"):
if scm_root.startswith("http") or self.command:
cmd = ("/usr/bin/git clone --depth 1 --branch=%s %s %s"
% (pipes.quote(scm_branch), pipes.quote(scm_root), pipes.quote(tmp_dir)))
self.retry_run(cmd, workdir=tmp_dir, show_cmd=True)
self.run_process_command(tmp_dir)
makedirs(target_dir)
shutil.copy2(os.path.join(tmp_dir, scm_file), target_path)
@ -219,13 +231,15 @@ def get_file_from_scm(scm_dict, target_path, logger=None):
scm_repo = None
scm_file = os.path.abspath(scm_dict)
scm_branch = None
command = None
else:
scm_type = scm_dict["scm"]
scm_repo = scm_dict["repo"]
scm_file = scm_dict["file"]
scm_branch = scm_dict.get("branch", None)
command = scm_dict.get('command')
scm = _get_wrapper(scm_type, logger=logger)
scm = _get_wrapper(scm_type, logger=logger, command=command)
files_copied = []
for i in force_list(scm_file):
@ -270,13 +284,15 @@ def get_dir_from_scm(scm_dict, target_path, logger=None):
scm_repo = None
scm_dir = os.path.abspath(scm_dict)
scm_branch = None
command = None
else:
scm_type = scm_dict["scm"]
scm_repo = scm_dict.get("repo", None)
scm_dir = scm_dict["dir"]
scm_branch = scm_dict.get("branch", None)
command = scm_dict.get("command")
scm = _get_wrapper(scm_type, logger=logger)
scm = _get_wrapper(scm_type, logger=logger, command=command)
with temp_dir(prefix="scm_checkout_") as tmp_dir:
scm.export_dir(scm_repo, scm_dir, scm_branch=scm_branch, target_dir=tmp_dir)

127
tests/test_scm.py
View File

@ -109,8 +109,9 @@ class GitSCMTestCase(SCMBaseTest):
commands = []
def process(cmd, workdir=None, **kwargs):
fname = cmd.split('|')[0].strip().split(' ')[-1]
touch(os.path.join(workdir, fname))
if cmd.startswith('/usr/bin/git'):
fname = cmd.split('|')[0].strip().split(' ')[-1]
touch(os.path.join(workdir, fname))
commands.append(cmd)
run.side_effect = process
@ -124,6 +125,76 @@ class GitSCMTestCase(SCMBaseTest):
commands,
['/usr/bin/git archive --remote=git://example.com/git/repo.git master some_file.txt | tar xf -'])
@mock.patch('pungi.wrappers.scm.run')
def test_get_file_generated_by_command_via_git(self, run):
commands = []
def process(cmd, workdir=None, **kwargs):
if cmd.startswith('/usr/bin/git'):
checkout = cmd.split(' ')[-1]
touch(os.path.join(checkout, 'some_file.txt'))
commands.append(cmd)
return 0, ''
run.side_effect = process
retval = scm.get_file_from_scm({'scm': 'git',
'repo': 'git://example.com/git/repo.git',
'file': 'some_file.txt',
'command': 'make'},
self.destdir)
self.assertStructure(retval, ['some_file.txt'])
self.assertRegexpMatches(
commands[0],
r'/usr/bin/git clone --depth 1 --branch=master git://example.com/git/repo.git /tmp/.+')
self.assertEqual(commands[1:], ['make'])
@mock.patch('pungi.wrappers.scm.run')
def test_get_file_and_fail_to_generate(self, run):
commands = []
def process(cmd, workdir=None, **kwargs):
if cmd.startswith('/usr/bin/git'):
checkout = cmd.split(' ')[-1]
touch(os.path.join(checkout, 'some_file.txt'))
commands.append(cmd)
return len(commands), 'output'
run.side_effect = process
with self.assertRaises(RuntimeError) as ctx:
scm.get_file_from_scm({'scm': 'git',
'repo': 'git://example.com/git/repo.git',
'file': 'some_file.txt',
'command': 'make'},
self.destdir)
self.assertEqual(str(ctx.exception), "'make' failed with exit code 2")
@mock.patch('pungi.wrappers.scm.run')
def test_get_file_generated_by_command_via_https(self, run):
commands = []
def process(cmd, workdir=None, **kwargs):
if cmd.startswith('/usr/bin/git'):
checkout = cmd.split(' ')[-1]
touch(os.path.join(checkout, 'some_file.txt'))
commands.append(cmd)
return 0, ''
run.side_effect = process
retval = scm.get_file_from_scm({'scm': 'git',
'repo': 'https://example.com/git/repo.git',
'file': 'some_file.txt',
'command': 'make'},
self.destdir)
self.assertStructure(retval, ['some_file.txt'])
self.assertRegexpMatches(
commands[0],
r'/usr/bin/git clone --depth 1 --branch=master https://example.com/git/repo.git /tmp/.+')
self.assertEqual(commands[1:], ['make'])
@mock.patch('pungi.wrappers.scm.run')
def test_get_file_via_https(self, run):
commands = []
@ -168,6 +239,32 @@ class GitSCMTestCase(SCMBaseTest):
commands,
['/usr/bin/git archive --remote=git://example.com/git/repo.git master subdir | tar xf -'])
@mock.patch('pungi.wrappers.scm.run')
def test_get_dir_via_git_and_generate(self, run):
commands = []
def process(cmd, workdir=None, **kwargs):
if cmd.startswith('/usr/bin/git'):
checkout = cmd.split(' ')[-1]
touch(os.path.join(checkout, 'subdir', 'first'))
touch(os.path.join(checkout, 'subdir', 'second'))
commands.append(cmd)
return 0, ''
run.side_effect = process
retval = scm.get_dir_from_scm({'scm': 'git',
'repo': 'git://example.com/git/repo.git',
'dir': 'subdir',
'command': 'make'},
self.destdir)
self.assertStructure(retval, ['first', 'second'])
self.assertRegexpMatches(
commands[0],
r'/usr/bin/git clone --depth 1 --branch=master git://example.com/git/repo.git /tmp/.+')
self.assertEqual(commands[1:], ['make'])
@mock.patch('pungi.wrappers.scm.run')
def test_get_dir_via_https(self, run):
commands = []
@ -190,6 +287,32 @@ class GitSCMTestCase(SCMBaseTest):
commands[0],
r'/usr/bin/git clone --depth 1 --branch=master https://example.com/git/repo.git /tmp/.+')
@mock.patch('pungi.wrappers.scm.run')
def test_get_dir_via_https_and_generate(self, run):
commands = []
def process(cmd, workdir=None, **kwargs):
if cmd.startswith('/usr/bin/git'):
checkout = cmd.split(' ')[-1]
touch(os.path.join(checkout, 'subdir', 'first'))
touch(os.path.join(checkout, 'subdir', 'second'))
commands.append(cmd)
return 0, ''
run.side_effect = process
retval = scm.get_dir_from_scm({'scm': 'git',
'repo': 'https://example.com/git/repo.git',
'dir': 'subdir',
'command': 'make'},
self.destdir)
self.assertStructure(retval, ['first', 'second'])
self.assertRegexpMatches(
commands[0],
r'/usr/bin/git clone --depth 1 --branch=master https://example.com/git/repo.git /tmp/.+')
self.assertEqual(commands[1:], ['make'])
class RpmSCMTestCase(SCMBaseTest):
def setUp(self):