diff --git a/doc/scm_support.rst b/doc/scm_support.rst index 401ef489..d86d6f67 100644 --- a/doc/scm_support.rst +++ b/doc/scm_support.rst @@ -41,6 +41,14 @@ which can contain following keys. * ``command`` -- defines a shell command to run after Git clone to generate the needed file (for example to run ``make``). Only supported in Git backend. +* ``options`` -- a dictionary of additional configuration options. These are + specific to different backends. + + Currently supported values for Git: + + * ``credential_helper`` -- path to a credential helper used to supply + username/password for remotes that require authentication. + Koji examples ------------- diff --git a/pungi/checks.py b/pungi/checks.py index 0e34f62d..855746e6 100644 --- a/pungi/checks.py +++ b/pungi/checks.py @@ -387,6 +387,7 @@ def _extend_with_default_and_alias(validator_class, offline=False): instance[property]["branch"] = resolver( instance[property]["repo"], instance[property].get("branch") or "HEAD", + instance[property].get("options"), ) for error in _hook_errors(properties, instance, schema): @@ -520,6 +521,13 @@ def make_schema(): "file": {"type": "string"}, "dir": {"type": "string"}, "command": {"type": "string"}, + "options": { + "type": "object", + "properties": { + "credential_helper": {"type": "string"}, + }, + "additionalProperties": False, + }, }, "additionalProperties": False, }, diff --git a/pungi/util.py b/pungi/util.py index b744f572..e5bf0743 100644 --- a/pungi/util.py +++ b/pungi/util.py @@ -279,7 +279,7 @@ class GitUrlResolveError(RuntimeError): pass -def resolve_git_ref(repourl, ref): +def resolve_git_ref(repourl, ref, credential_helper=None): """Resolve a reference in a Git repo to a commit. Raises RuntimeError if there was an error. Most likely cause is failure to @@ -289,7 +289,7 @@ def resolve_git_ref(repourl, ref): # This looks like a commit ID already. return ref try: - _, output = git_ls_remote(repourl, ref) + _, output = git_ls_remote(repourl, ref, credential_helper) except RuntimeError as e: raise GitUrlResolveError( "ref does not exist in remote repo %s with the error %s %s" @@ -316,7 +316,7 @@ def resolve_git_ref(repourl, ref): return lines[0].split()[0] -def resolve_git_url(url): +def resolve_git_url(url, credential_helper=None): """Given a url to a Git repo specifying HEAD or origin/ as a ref, replace that specifier with actual SHA1 of the commit. @@ -335,7 +335,7 @@ def resolve_git_url(url): scheme = r.scheme.replace("git+", "") baseurl = urllib.parse.urlunsplit((scheme, r.netloc, r.path, "", "")) - fragment = resolve_git_ref(baseurl, ref) + fragment = resolve_git_ref(baseurl, ref, credential_helper) result = urllib.parse.urlunsplit((r.scheme, r.netloc, r.path, r.query, fragment)) if "?#" in url: @@ -354,13 +354,18 @@ class GitUrlResolver(object): self.offline = offline self.cache = {} - def __call__(self, url, branch=None): + def __call__(self, url, branch=None, options=None): + credential_helper = options.get("credential_helper") if options else None if self.offline: return branch or url key = (url, branch) if key not in self.cache: try: - res = resolve_git_ref(url, branch) if branch else resolve_git_url(url) + res = ( + resolve_git_ref(url, branch, credential_helper) + if branch + else resolve_git_url(url, credential_helper) + ) self.cache[key] = res except GitUrlResolveError as exc: self.cache[key] = exc @@ -991,8 +996,12 @@ def retry(timeout=120, interval=30, wait_on=Exception): @retry(wait_on=RuntimeError) -def git_ls_remote(baseurl, ref): - return run(["git", "ls-remote", baseurl, ref], universal_newlines=True) +def git_ls_remote(baseurl, ref, credential_helper=None): + cmd = ["git"] + if credential_helper: + cmd.extend(["-c", "credential.useHttpPath=true"]) + cmd.extend(["-c", "credential.helper=%s" % credential_helper]) + return run(cmd + ["ls-remote", baseurl, ref], universal_newlines=True) def get_tz_offset(): diff --git a/pungi/wrappers/scm.py b/pungi/wrappers/scm.py index 5c4b37fb..cefb2ae4 100644 --- a/pungi/wrappers/scm.py +++ b/pungi/wrappers/scm.py @@ -31,10 +31,11 @@ from .kojiwrapper import KojiWrapper class ScmBase(kobo.log.LoggingBase): - def __init__(self, logger=None, command=None, compose=None): + def __init__(self, logger=None, command=None, compose=None, options=None): kobo.log.LoggingBase.__init__(self, logger=logger) self.command = command self.compose = compose + self.options = options or {} @retry(interval=60, timeout=300, wait_on=RuntimeError) def retry_run(self, cmd, **kwargs): @@ -156,22 +157,31 @@ class GitWrapper(ScmBase): if "://" not in repo: repo = "file://%s" % repo + git_cmd = ["git"] + if "credential_helper" in self.options: + git_cmd.extend(["-c", "credential.useHttpPath=true"]) + git_cmd.extend( + ["-c", "credential.helper=%s" % self.options["credential_helper"]] + ) + run(["git", "init"], workdir=destdir) try: - run(["git", "fetch", "--depth=1", repo, branch], workdir=destdir) + run(git_cmd + ["fetch", "--depth=1", repo, branch], workdir=destdir) run(["git", "checkout", "FETCH_HEAD"], workdir=destdir) except RuntimeError as e: # Fetch failed, to do a full clone we add a remote to our empty # repo, get its content and check out the reference we want. self.log_debug( "Trying to do a full clone because shallow clone failed: %s %s" - % (e, e.output) + % (e, getattr(e, "output", "")) ) try: # Re-run git init in case of previous failure breaking .git dir run(["git", "init"], workdir=destdir) run(["git", "remote", "add", "origin", repo], workdir=destdir) - self.retry_run(["git", "remote", "update", "origin"], workdir=destdir) + self.retry_run( + git_cmd + ["remote", "update", "origin"], workdir=destdir + ) run(["git", "checkout", branch], workdir=destdir) except RuntimeError: if self.compose: @@ -361,15 +371,19 @@ def get_file_from_scm(scm_dict, target_path, compose=None): scm_file = os.path.abspath(scm_dict) scm_branch = None command = None + options = {} else: scm_type = scm_dict["scm"] scm_repo = scm_dict["repo"] scm_file = scm_dict["file"] scm_branch = scm_dict.get("branch", None) command = scm_dict.get("command") + options = scm_dict.get("options", {}) logger = compose._logger if compose else None - scm = _get_wrapper(scm_type, logger=logger, command=command, compose=compose) + scm = _get_wrapper( + scm_type, logger=logger, command=command, compose=compose, options=options + ) files_copied = [] for i in force_list(scm_file): @@ -450,15 +464,19 @@ def get_dir_from_scm(scm_dict, target_path, compose=None): scm_dir = os.path.abspath(scm_dict) scm_branch = None command = None + options = {} else: scm_type = scm_dict["scm"] scm_repo = scm_dict.get("repo", None) scm_dir = scm_dict["dir"] scm_branch = scm_dict.get("branch", None) command = scm_dict.get("command") + options = scm_dict.get("options", {}) logger = compose._logger if compose else None - scm = _get_wrapper(scm_type, logger=logger, command=command, compose=compose) + scm = _get_wrapper( + scm_type, logger=logger, command=command, compose=compose, options=options + ) with temp_dir(prefix="scm_checkout_") as tmp_dir: scm.export_dir(scm_repo, scm_dir, scm_branch=scm_branch, target_dir=tmp_dir) diff --git a/tests/helpers.py b/tests/helpers.py index 948b901f..200d933c 100644 --- a/tests/helpers.py +++ b/tests/helpers.py @@ -21,6 +21,15 @@ from pungi import paths, checks from pungi.module_util import Modulemd +GIT_WITH_CREDS = [ + "git", + "-c", + "credential.useHttpPath=true", + "-c", + "credential.helper=!ch", +] + + class BaseTestCase(unittest.TestCase): def assertFilesEqual(self, fn1, fn2): with open(fn1, "rb") as f1: diff --git a/tests/test_config.py b/tests/test_config.py index 8911a927..1dda6639 100644 --- a/tests/test_config.py +++ b/tests/test_config.py @@ -440,7 +440,7 @@ class LiveMediaConfigTestCase(ConfigTestCase): live_media_version="Rawhide", ) - resolve_git_url.side_effect = lambda x: x.replace("HEAD", "CAFE") + resolve_git_url.side_effect = lambda x, _helper: x.replace("HEAD", "CAFE") self.assertValidation(cfg) self.assertEqual(cfg["live_media_ksurl"], "git://example.com/repo.git#CAFE") diff --git a/tests/test_scm.py b/tests/test_scm.py index bf0c4347..3886c27d 100644 --- a/tests/test_scm.py +++ b/tests/test_scm.py @@ -13,8 +13,10 @@ import random import os import six +from parameterized import parameterized + from pungi.wrappers import scm -from tests.helpers import touch +from tests.helpers import touch, GIT_WITH_CREDS from kobo.shortcuts import run @@ -109,37 +111,45 @@ class FileSCMTestCase(SCMBaseTest): self.assertIn("No directories matched", str(ctx.exception)) +CREDENTIALS_CONFIG = {"credential_helper": "!ch"} + + class GitSCMTestCase(SCMBaseTest): - def assertCalls(self, mock_run, url, branch, command=None): + def assertCalls(self, mock_run, url, branch, command=None, with_creds=False): + git = GIT_WITH_CREDS if with_creds else ["git"] command = [command] if command else [] self.assertEqual( [call[0][0] for call in mock_run.call_args_list], [ ["git", "init"], - ["git", "fetch", "--depth=1", url, branch], + git + ["fetch", "--depth=1", url, branch], ["git", "checkout", "FETCH_HEAD"], ] + command, ) - @mock.patch("pungi.wrappers.scm.run") - def test_get_file(self, run): + @parameterized.expand([("without_creds", {}), ("with_creds", CREDENTIALS_CONFIG)]) + def test_get_file(self, _name, config): def process(cmd, workdir=None, **kwargs): touch(os.path.join(workdir, "some_file.txt")) touch(os.path.join(workdir, "other_file.txt")) - run.side_effect = process + with mock.patch("pungi.wrappers.scm.run") as run: + run.side_effect = process + retval = scm.get_file_from_scm( + { + "scm": "git", + "repo": "git://example.com/git/repo.git", + "file": "some_file.txt", + "options": config, + }, + self.destdir, + ) - retval = scm.get_file_from_scm( - { - "scm": "git", - "repo": "git://example.com/git/repo.git", - "file": "some_file.txt", - }, - self.destdir, - ) self.assertStructure(retval, ["some_file.txt"]) - self.assertCalls(run, "git://example.com/git/repo.git", "master") + self.assertCalls( + run, "git://example.com/git/repo.git", "master", with_creds=bool(config) + ) @mock.patch("pungi.wrappers.scm.run") def test_get_file_function(self, run): @@ -163,9 +173,10 @@ class GitSCMTestCase(SCMBaseTest): self.assertEqual(retval, destination) self.assertCalls(run, "git://example.com/git/repo.git", "master") - @mock.patch("pungi.wrappers.scm.run") - def test_get_file_fetch_fails(self, run): + @parameterized.expand([("without_creds", {}), ("with_creds", CREDENTIALS_CONFIG)]) + def test_get_file_fetch_fails(self, _name, config): url = "git://example.com/git/repo.git" + git = GIT_WITH_CREDS if config else ["git"] def process(cmd, workdir=None, **kwargs): if "fetch" in cmd: @@ -175,18 +186,20 @@ class GitSCMTestCase(SCMBaseTest): touch(os.path.join(workdir, "some_file.txt")) touch(os.path.join(workdir, "other_file.txt")) - run.side_effect = process + with mock.patch("pungi.wrappers.scm.run") as run: + run.side_effect = process + retval = scm.get_file_from_scm( + {"scm": "git", "repo": url, "file": "some_file.txt", "options": config}, + self.destdir, + ) - retval = scm.get_file_from_scm( - {"scm": "git", "repo": url, "file": "some_file.txt"}, self.destdir - ) self.assertStructure(retval, ["some_file.txt"]) self.assertEqual( [call[0][0] for call in run.call_args_list], [ ["git", "init"], - [ - "git", + git + + [ "fetch", "--depth=1", "git://example.com/git/repo.git", @@ -194,7 +207,7 @@ class GitSCMTestCase(SCMBaseTest): ], ["git", "init"], ["git", "remote", "add", "origin", url], - ["git", "remote", "update", "origin"], + git + ["remote", "update", "origin"], ["git", "checkout", "master"], ], ) @@ -243,20 +256,28 @@ class GitSCMTestCase(SCMBaseTest): self.assertEqual(str(ctx.exception), "'make' failed with exit code 1") - @mock.patch("pungi.wrappers.scm.run") - def test_get_dir(self, run): + @parameterized.expand([("without_creds", {}), ("with_creds", CREDENTIALS_CONFIG)]) + def test_get_dir(self, _name, config): def process(cmd, workdir=None, **kwargs): touch(os.path.join(workdir, "subdir", "first")) touch(os.path.join(workdir, "subdir", "second")) - run.side_effect = process + with mock.patch("pungi.wrappers.scm.run") as run: + run.side_effect = process + retval = scm.get_dir_from_scm( + { + "scm": "git", + "repo": "git://example.com/git/repo.git", + "dir": "subdir", + "options": config, + }, + self.destdir, + ) - retval = scm.get_dir_from_scm( - {"scm": "git", "repo": "git://example.com/git/repo.git", "dir": "subdir"}, - self.destdir, - ) self.assertStructure(retval, ["first", "second"]) - self.assertCalls(run, "git://example.com/git/repo.git", "master") + self.assertCalls( + run, "git://example.com/git/repo.git", "master", with_creds=bool(config) + ) @mock.patch("pungi.wrappers.scm.run") def test_get_dir_and_generate(self, run): diff --git a/tests/test_util.py b/tests/test_util.py index 033f251e..52948d42 100644 --- a/tests/test_util.py +++ b/tests/test_util.py @@ -16,7 +16,7 @@ import six from pungi import compose from pungi import util -from tests.helpers import touch, PungiTestCase, mk_boom +from tests.helpers import touch, PungiTestCase, mk_boom, GIT_WITH_CREDS class TestGitRefResolver(unittest.TestCase): @@ -32,6 +32,20 @@ class TestGitRefResolver(unittest.TestCase): universal_newlines=True, ) + @mock.patch("pungi.util.run") + def test_successful_resolve_with_credentials(self, run): + run.return_value = (0, "CAFEBABE\tHEAD\n") + + url = util.resolve_git_url( + "https://git.example.com/repo.git?somedir#HEAD", "!ch" + ) + + self.assertEqual(url, "https://git.example.com/repo.git?somedir#CAFEBABE") + run.assert_called_once_with( + GIT_WITH_CREDS + ["ls-remote", "https://git.example.com/repo.git", "HEAD"], + universal_newlines=True, + ) + @mock.patch("pungi.util.run") def test_successful_resolve_branch(self, run): run.return_value = (0, "CAFEBABE\trefs/heads/f24\n") @@ -211,11 +225,12 @@ class TestGitRefResolver(unittest.TestCase): self.assertEqual(resolver(url2), "2") self.assertEqual(resolver(url3, ref2), "beef") self.assertEqual( - mock_resolve_url.call_args_list, [mock.call(url1), mock.call(url2)] + mock_resolve_url.call_args_list, + [mock.call(url1, None), mock.call(url2, None)], ) self.assertEqual( mock_resolve_ref.call_args_list, - [mock.call(url3, ref1), mock.call(url3, ref2)], + [mock.call(url3, ref1, None), mock.call(url3, ref2, None)], ) @mock.patch("pungi.util.resolve_git_url") @@ -227,7 +242,7 @@ class TestGitRefResolver(unittest.TestCase): resolver(url) with self.assertRaises(util.GitUrlResolveError): resolver(url) - self.assertEqual(mock_resolve.call_args_list, [mock.call(url)]) + self.assertEqual(mock_resolve.call_args_list, [mock.call(url, None)]) class TestGetVariantData(unittest.TestCase):