scm-wrapper: Allow running command after git clone

When a file should be obtained from a git repository, allow running an
arbitrary command (like `make`) after clone but before copying the files
out. This only works for the Git backend.

The downside is that a clone is needed and we can no longer use `git
archive` to speed things up.

Fixes: https://pagure.io/pungi/issue/5
Signed-off-by: Lubomír Sedlář <lsedlar@redhat.com>

pungi/checks.py

@@ -422,6 +422,7 @@ def _make_schema():
                     "branch": {"$ref": "#/definitions/optional_string"},
                     "file": {"type": "string"},
                     "dir": {"type": "string"},
+                    "command": {"type": "string"},
                 },
                 "additionalProperties": False,
             },

pungi/wrappers/scm.py

@@ -27,8 +27,9 @@ from pungi.util import (explode_rpm_package, makedirs, copy_all, temp_dir,
 
 
 class ScmBase(kobo.log.LoggingBase):
-    def __init__(self, logger=None):
+    def __init__(self, logger=None, command=None):
         kobo.log.LoggingBase.__init__(self, logger=logger)
+        self.command = command
 
     @retry(interval=60, timeout=300, wait_on=RuntimeError)
     def retry_run(self, cmd, **kwargs):
@@ -39,6 +40,15 @@ class ScmBase(kobo.log.LoggingBase):
 
         return run(cmd, **kwargs)
 
+    def run_process_command(self, cwd):
+        if self.command:
+            self.log_debug('Running "%s"' % self.command)
+            retcode, output = run(self.command, workdir=cwd, can_fail=True)
+            if retcode != 0:
+                self.log_error('Output was: "%s"' % output)
+                raise RuntimeError('%r failed with exit code %s'
+                                   % (self.command, retcode))
+
 
 class FileWrapper(ScmBase):
     def export_dir(self, scm_root, scm_dir, target_dir, scm_branch=None):
@@ -104,10 +114,11 @@ class GitWrapper(ScmBase):
                    % (pipes.quote(scm_root), pipes.quote(scm_branch), pipes.quote(scm_dir)))
             # git archive is not supported by http/https
             # or by smart http https://git-scm.com/book/en/v2/Git-on-the-Server-Smart-HTTP
-            if scm_root.startswith("http"):
+            if scm_root.startswith("http") or self.command:
                 cmd = ("/usr/bin/git clone --depth 1 --branch=%s %s %s"
                        % (pipes.quote(scm_branch), pipes.quote(scm_root), pipes.quote(tmp_dir)))
             self.retry_run(cmd, workdir=tmp_dir, show_cmd=True)
+            self.run_process_command(tmp_dir)
 
             copy_all(os.path.join(tmp_dir, scm_dir), target_dir)
 
@@ -127,10 +138,11 @@ class GitWrapper(ScmBase):
                    % (pipes.quote(scm_root), pipes.quote(scm_branch), pipes.quote(scm_file)))
             # git archive is not supported by http/https
             # or by smart http https://git-scm.com/book/en/v2/Git-on-the-Server-Smart-HTTP
-            if scm_root.startswith("http"):
+            if scm_root.startswith("http") or self.command:
                 cmd = ("/usr/bin/git clone --depth 1 --branch=%s %s %s"
                        % (pipes.quote(scm_branch), pipes.quote(scm_root), pipes.quote(tmp_dir)))
             self.retry_run(cmd, workdir=tmp_dir, show_cmd=True)
+            self.run_process_command(tmp_dir)
 
             makedirs(target_dir)
             shutil.copy2(os.path.join(tmp_dir, scm_file), target_path)
@@ -219,13 +231,15 @@ def get_file_from_scm(scm_dict, target_path, logger=None):
         scm_repo = None
         scm_file = os.path.abspath(scm_dict)
         scm_branch = None
+        command = None
     else:
         scm_type = scm_dict["scm"]
         scm_repo = scm_dict["repo"]
         scm_file = scm_dict["file"]
         scm_branch = scm_dict.get("branch", None)
+        command = scm_dict.get('command')
 
-    scm = _get_wrapper(scm_type, logger=logger)
+    scm = _get_wrapper(scm_type, logger=logger, command=command)
 
     files_copied = []
     for i in force_list(scm_file):
@@ -270,13 +284,15 @@ def get_dir_from_scm(scm_dict, target_path, logger=None):
         scm_repo = None
         scm_dir = os.path.abspath(scm_dict)
         scm_branch = None
+        command = None
     else:
         scm_type = scm_dict["scm"]
         scm_repo = scm_dict.get("repo", None)
         scm_dir = scm_dict["dir"]
         scm_branch = scm_dict.get("branch", None)
+        command = scm_dict.get("command")
 
-    scm = _get_wrapper(scm_type, logger=logger)
+    scm = _get_wrapper(scm_type, logger=logger, command=command)
 
     with temp_dir(prefix="scm_checkout_") as tmp_dir:
         scm.export_dir(scm_repo, scm_dir, scm_branch=scm_branch, target_dir=tmp_dir)

tests/test_scm.py

@@ -109,6 +109,7 @@ class GitSCMTestCase(SCMBaseTest):
         commands = []
 
         def process(cmd, workdir=None, **kwargs):
+            if cmd.startswith('/usr/bin/git'):
                 fname = cmd.split('|')[0].strip().split(' ')[-1]
                 touch(os.path.join(workdir, fname))
             commands.append(cmd)
@@ -124,6 +125,76 @@ class GitSCMTestCase(SCMBaseTest):
             commands,
             ['/usr/bin/git archive --remote=git://example.com/git/repo.git master some_file.txt | tar xf -'])
 
+    @mock.patch('pungi.wrappers.scm.run')
+    def test_get_file_generated_by_command_via_git(self, run):
+        commands = []
+
+        def process(cmd, workdir=None, **kwargs):
+            if cmd.startswith('/usr/bin/git'):
+                checkout = cmd.split(' ')[-1]
+                touch(os.path.join(checkout, 'some_file.txt'))
+            commands.append(cmd)
+            return 0, ''
+
+        run.side_effect = process
+
+        retval = scm.get_file_from_scm({'scm': 'git',
+                                        'repo': 'git://example.com/git/repo.git',
+                                        'file': 'some_file.txt',
+                                        'command': 'make'},
+                                       self.destdir)
+        self.assertStructure(retval, ['some_file.txt'])
+        self.assertRegexpMatches(
+            commands[0],
+            r'/usr/bin/git clone --depth 1 --branch=master git://example.com/git/repo.git /tmp/.+')
+        self.assertEqual(commands[1:], ['make'])
+
+    @mock.patch('pungi.wrappers.scm.run')
+    def test_get_file_and_fail_to_generate(self, run):
+        commands = []
+
+        def process(cmd, workdir=None, **kwargs):
+            if cmd.startswith('/usr/bin/git'):
+                checkout = cmd.split(' ')[-1]
+                touch(os.path.join(checkout, 'some_file.txt'))
+            commands.append(cmd)
+            return len(commands), 'output'
+
+        run.side_effect = process
+
+        with self.assertRaises(RuntimeError) as ctx:
+            scm.get_file_from_scm({'scm': 'git',
+                                   'repo': 'git://example.com/git/repo.git',
+                                   'file': 'some_file.txt',
+                                   'command': 'make'},
+                                  self.destdir)
+
+        self.assertEqual(str(ctx.exception), "'make' failed with exit code 2")
+
+    @mock.patch('pungi.wrappers.scm.run')
+    def test_get_file_generated_by_command_via_https(self, run):
+        commands = []
+
+        def process(cmd, workdir=None, **kwargs):
+            if cmd.startswith('/usr/bin/git'):
+                checkout = cmd.split(' ')[-1]
+                touch(os.path.join(checkout, 'some_file.txt'))
+            commands.append(cmd)
+            return 0, ''
+
+        run.side_effect = process
+
+        retval = scm.get_file_from_scm({'scm': 'git',
+                                        'repo': 'https://example.com/git/repo.git',
+                                        'file': 'some_file.txt',
+                                        'command': 'make'},
+                                       self.destdir)
+        self.assertStructure(retval, ['some_file.txt'])
+        self.assertRegexpMatches(
+            commands[0],
+            r'/usr/bin/git clone --depth 1 --branch=master https://example.com/git/repo.git /tmp/.+')
+        self.assertEqual(commands[1:], ['make'])
+
     @mock.patch('pungi.wrappers.scm.run')
     def test_get_file_via_https(self, run):
         commands = []
@@ -168,6 +239,32 @@ class GitSCMTestCase(SCMBaseTest):
             commands,
             ['/usr/bin/git archive --remote=git://example.com/git/repo.git master subdir | tar xf -'])
 
+    @mock.patch('pungi.wrappers.scm.run')
+    def test_get_dir_via_git_and_generate(self, run):
+        commands = []
+
+        def process(cmd, workdir=None, **kwargs):
+            if cmd.startswith('/usr/bin/git'):
+                checkout = cmd.split(' ')[-1]
+                touch(os.path.join(checkout, 'subdir', 'first'))
+                touch(os.path.join(checkout, 'subdir', 'second'))
+            commands.append(cmd)
+            return 0, ''
+
+        run.side_effect = process
+
+        retval = scm.get_dir_from_scm({'scm': 'git',
+                                       'repo': 'git://example.com/git/repo.git',
+                                       'dir': 'subdir',
+                                       'command': 'make'},
+                                      self.destdir)
+        self.assertStructure(retval, ['first', 'second'])
+
+        self.assertRegexpMatches(
+            commands[0],
+            r'/usr/bin/git clone --depth 1 --branch=master git://example.com/git/repo.git /tmp/.+')
+        self.assertEqual(commands[1:], ['make'])
+
     @mock.patch('pungi.wrappers.scm.run')
     def test_get_dir_via_https(self, run):
         commands = []
@@ -190,6 +287,32 @@ class GitSCMTestCase(SCMBaseTest):
             commands[0],
             r'/usr/bin/git clone --depth 1 --branch=master https://example.com/git/repo.git /tmp/.+')
 
+    @mock.patch('pungi.wrappers.scm.run')
+    def test_get_dir_via_https_and_generate(self, run):
+        commands = []
+
+        def process(cmd, workdir=None, **kwargs):
+            if cmd.startswith('/usr/bin/git'):
+                checkout = cmd.split(' ')[-1]
+                touch(os.path.join(checkout, 'subdir', 'first'))
+                touch(os.path.join(checkout, 'subdir', 'second'))
+            commands.append(cmd)
+            return 0, ''
+
+        run.side_effect = process
+
+        retval = scm.get_dir_from_scm({'scm': 'git',
+                                       'repo': 'https://example.com/git/repo.git',
+                                       'dir': 'subdir',
+                                       'command': 'make'},
+                                      self.destdir)
+        self.assertStructure(retval, ['first', 'second'])
+
+        self.assertRegexpMatches(
+            commands[0],
+            r'/usr/bin/git clone --depth 1 --branch=master https://example.com/git/repo.git /tmp/.+')
+        self.assertEqual(commands[1:], ['make'])
+
 
 class RpmSCMTestCase(SCMBaseTest):
     def setUp(self):