From f8932bc1f408a2222c20b971f76d293f2a7be9df Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lubom=C3=ADr=20Sedl=C3=A1=C5=99?= Date: Tue, 9 Jul 2024 13:59:46 +0200 Subject: [PATCH] scm: Clone git submodules MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit If the repo contains .gitmodules file, run the commands to clone all submodules. Signed-off-by: Lubomír Sedlář (cherry picked from commit 6d1428ab89de6ffa5c18466a469606887a0300b8) --- pungi/wrappers/scm.py | 11 +++++ tests/test_scm.py | 95 +++++++++++++++++++++++++++++++++++++++++-- 2 files changed, 103 insertions(+), 3 deletions(-) diff --git a/pungi/wrappers/scm.py b/pungi/wrappers/scm.py index d6417c75..991ce169 100644 --- a/pungi/wrappers/scm.py +++ b/pungi/wrappers/scm.py @@ -198,6 +198,17 @@ class GitWrapper(ScmBase): copy_all(destdir, debugdir) raise + if os.path.exists(os.path.join(destdir, ".gitmodules")): + try: + self.log_debug("Cloning submodules") + run(["git", "submodule", "init"], workdir=destdir) + run(["git", "submodule", "update"], workdir=destdir) + except RuntimeError as e: + self.log_error( + "Failed to clone submodules: %s %s", e, getattr(e, "output", "") + ) + # Ignore the error here, there may just be no submodules. + def get_temp_repo_path(self, scm_root, scm_branch): scm_repo = scm_root.split("/")[-1] process_id = os.getpid() diff --git a/tests/test_scm.py b/tests/test_scm.py index 317ade2a..b41108c2 100644 --- a/tests/test_scm.py +++ b/tests/test_scm.py @@ -7,6 +7,8 @@ import random import shutil import tempfile import unittest +import http.server +import threading import six @@ -303,9 +305,9 @@ class GitSCMTestCase(SCMBaseTest): self.assertCalls(run, "git://example.com/git/repo.git", "master", "make") -class GitSCMTestCaseReal(SCMBaseTest): +class GitSCMTestCaseRealBase(SCMBaseTest): def setUp(self): - super(GitSCMTestCaseReal, self).setUp() + super(GitSCMTestCaseRealBase, self).setUp() self.compose = mock.Mock(conf={}) self.gitRepositoryLocation = tempfile.mkdtemp() git_dir = os.path.join(self.gitRepositoryLocation, ".git") @@ -358,9 +360,11 @@ class GitSCMTestCaseReal(SCMBaseTest): ) def tearDown(self): - super(GitSCMTestCaseReal, self).tearDown() + super(GitSCMTestCaseRealBase, self).tearDown() shutil.rmtree(self.gitRepositoryLocation) + +class GitSCMTestCaseReal(GitSCMTestCaseRealBase): def test_get_file_function(self): sourceFileLocation = random.choice(list(self.files.keys())) sourceFilename = os.path.basename(sourceFileLocation) @@ -416,6 +420,91 @@ class GitSCMTestCaseReal(SCMBaseTest): self.assertEqual(sourceFileContent, destinationFileContent) +class GitSCMTestCaseRealSubmodule(GitSCMTestCaseRealBase): + + def setUp(self): + # This gets a little complicated. The test sets up a git repo with a + # submodule and tries to obtain a file from the submodule. However, + # submodules over file:// are restricted for security reasons. The test + # should not modify any global configuration file, so to avoid the + # issues we instead start a one-off HTTP server to serve the repository + # on localhost. + # The server runs in a separate thread. + super(GitSCMTestCaseRealSubmodule, self).setUp() + self.main_repo_path = tempfile.mkdtemp() + submodule_path = self.gitRepositoryLocation + + run(["git", "update-server-info"], workdir=submodule_path) + + class MyHandler(http.server.SimpleHTTPRequestHandler): + def __init__(self, *args, **kwargs): + super(MyHandler, self).__init__( + *args, directory=submodule_path, **kwargs + ) + + self.httpd = http.server.HTTPServer(("", 0), MyHandler) + self.httpd.timeout = 1 + self.url = "http://localhost:%s/.git" % self.httpd.server_port + + self.thread_done = False + + def runner(): + # Repeatedly handle a request until the flag is set. The timeout is + # configured on the self.httpd object. + while not self.thread_done: + self.httpd.handle_request() + + self.t = threading.Thread(target=runner) + self.t.start() + + cmds = [ + ["git", "-c", "init.defaultBranch=master", "init"], + ["git", "submodule", "add", "-b", "master", self.url, "submodule"], + [ + "git", + "-c", + "user.name=Pungi Test Engineer", + "-c", + "user.email=ptestengineer@example.com", + "commit", + "-am", + "Add submodule", + ], + ] + for cmd in cmds: + run(cmd, workdir=self.main_repo_path) + + def tearDown(self): + super(GitSCMTestCaseRealSubmodule, self).tearDown() + self.thread_done = True + self.t.join() + shutil.rmtree(self.main_repo_path) + + def test_get_file(self): + sourceFileLocation = random.choice(list(self.files.keys())) + sourceFilename = os.path.basename(sourceFileLocation) + destinationFileLocation = os.path.join(self.destdir, "other_file.txt") + destinationFileActualLocation = scm.get_file( + { + "scm": "git", + "repo": self.main_repo_path, + "file": os.path.join("submodule", sourceFilename), + }, + destinationFileLocation, + compose=self.compose, + ) + self.assertEqual(destinationFileActualLocation, destinationFileLocation) + self.assertTrue(os.path.isfile(destinationFileActualLocation)) + + # Reading the contents of both files to compare later. + with open(sourceFileLocation) as sourceFileHandle: + sourceFileContent = sourceFileHandle.read() + with open(destinationFileActualLocation) as destinationFileHandle: + destinationFileContent = destinationFileHandle.read() + # Comparing the contents of source to the destination file. + self.assertEqual(sourceFileContent, destinationFileContent) + + class RpmSCMTestCase(SCMBaseTest): def setUp(self): super(RpmSCMTestCase, self).setUp()