Remove repository cloning multiple times

JIRA: RHELCMP-8913
Signed-off-by: Aditya Bisoi <abisoi@redhat.com>
(cherry picked from commit b6296bdfcd)

doc/conf.py

@@ -31,29 +31,29 @@ import os
 extensions = []
 
 # Add any paths that contain templates here, relative to this directory.
-templates_path = ['_templates']
+templates_path = ["_templates"]
 
 # The suffix of source filenames.
-source_suffix = '.rst'
+source_suffix = ".rst"
 
 # The encoding of source files.
 # source_encoding = 'utf-8-sig'
 
 # The master toctree document.
-master_doc = 'index'
+master_doc = "index"
 
 # General information about the project.
-project = u'Pungi'
-copyright = u'2016, Red Hat, Inc.'
+project = "Pungi"
+copyright = "2016, Red Hat, Inc."
 
 # The version info for the project you're documenting, acts as replacement for
 # |version| and |release|, also used in various other places throughout the
 # built documents.
 #
 # The short X.Y version.
-version = '4.4'
+version = "4.4"
 # The full version, including alpha/beta/rc tags.
-release = '4.4.1'
+release = "4.4.1"
 
 # The language for content autogenerated by Sphinx. Refer to documentation
 # for a list of supported languages.
@@ -67,7 +67,7 @@ release = '4.4.1'
 
 # List of patterns, relative to source directory, that match files and
 # directories to ignore when looking for source files.
-exclude_patterns = ['_build']
+exclude_patterns = ["_build"]
 
 # The reST default role (used for this markup: `text`) to use for all
 # documents.
@@ -85,7 +85,7 @@ exclude_patterns = ['_build']
 # show_authors = False
 
 # The name of the Pygments (syntax highlighting) style to use.
-pygments_style = 'sphinx'
+pygments_style = "sphinx"
 
 # A list of ignored prefixes for module index sorting.
 # modindex_common_prefix = []
@@ -98,7 +98,7 @@ pygments_style = 'sphinx'
 
 # The theme to use for HTML and HTML Help pages.  See the documentation for
 # a list of builtin themes.
-html_theme = 'default'
+html_theme = "default"
 
 # Theme options are theme-specific and customize the look and feel of a theme
 # further.  For a list of options available for each theme, see the
@@ -127,7 +127,7 @@ html_theme = 'default'
 # Add any paths that contain custom static files (such as style sheets) here,
 # relative to this directory. They are copied after the builtin static files,
 # so a file named "default.css" will overwrite the builtin "default.css".
-html_static_path = ['_static']
+html_static_path = ["_static"]
 
 # Add any extra paths that contain custom files (such as robots.txt or
 # .htaccess) here, relative to this directory. These files are copied
@@ -176,7 +176,7 @@ html_static_path = ['_static']
 # html_file_suffix = None
 
 # Output file base name for HTML help builder.
-htmlhelp_basename = 'Pungidoc'
+htmlhelp_basename = "Pungidoc"
 
 
 # -- Options for LaTeX output ---------------------------------------------
@@ -184,10 +184,8 @@ htmlhelp_basename = 'Pungidoc'
 latex_elements = {
     # The paper size ('letterpaper' or 'a4paper').
     #'papersize': 'letterpaper',
-
     # The font size ('10pt', '11pt' or '12pt').
     #'pointsize': '10pt',
-
     # Additional stuff for the LaTeX preamble.
     #'preamble': '',
 }
@@ -196,8 +194,7 @@ latex_elements = {
 # (source start file, target name, title,
 #  author, documentclass [howto, manual, or own class]).
 latex_documents = [
-  ('index', 'Pungi.tex', u'Pungi Documentation',
-   u'Daniel Mach', 'manual'),
+    ("index", "Pungi.tex", "Pungi Documentation", "Daniel Mach", "manual"),
 ]
 
 # The name of an image file (relative to this directory) to place at the top of
@@ -225,10 +222,7 @@ latex_documents = [
 
 # One entry per manual page. List of tuples
 # (source start file, name, description, authors, manual section).
-man_pages = [
-    ('index', 'pungi', u'Pungi Documentation',
-     [u'Daniel Mach'], 1)
-]
+man_pages = [("index", "pungi", "Pungi Documentation", ["Daniel Mach"], 1)]
 
 # If true, show URL addresses after external links.
 # man_show_urls = False
@@ -240,9 +234,15 @@ man_pages = [
 # (source start file, target name, title, author,
 #  dir menu entry, description, category)
 texinfo_documents = [
-  ('index', 'Pungi', u'Pungi Documentation',
-   u'Daniel Mach', 'Pungi', 'One line description of project.',
-   'Miscellaneous'),
+    (
+        "index",
+        "Pungi",
+        "Pungi Documentation",
+        "Daniel Mach",
+        "Pungi",
+        "One line description of project.",
+        "Miscellaneous",
+    ),
 ]
 
 # Documents to append as an appendix to all manuals.

pungi/scripts/pungi_koji.py

@@ -23,6 +23,7 @@ from pungi.phases import PHASES_NAMES
 from pungi import get_full_version, util
 from pungi.errors import UnsignedPackagesError
 from pungi.wrappers import kojiwrapper
+from pungi.util import rmtree
 
 
 # force C locales
@@ -699,3 +700,8 @@ def cli_main():
         sys.stdout.flush()
         sys.stderr.flush()
         sys.exit(1)
+    finally:
+        # Remove repositories cloned during ExtraFiles phase
+        process_id = os.getpid()
+        directoy_to_remove = "/tmp/pungi-temp-git-repos-" + str(process_id) + "/"
+        rmtree(directoy_to_remove)

pungi/wrappers/scm.py

@@ -20,6 +20,7 @@ import os
 import shutil
 import glob
 import six
+import threading
 from six.moves import shlex_quote
 from six.moves.urllib.request import urlretrieve
 from fnmatch import fnmatch
@@ -29,6 +30,8 @@ from kobo.shortcuts import run, force_list
 from pungi.util import explode_rpm_package, makedirs, copy_all, temp_dir, retry
 from .kojiwrapper import KojiWrapper
 
+lock = threading.Lock()
+
 
 class ScmBase(kobo.log.LoggingBase):
     def __init__(self, logger=None, command=None, compose=None, options=None):
@@ -195,19 +198,38 @@ class GitWrapper(ScmBase):
                     copy_all(destdir, debugdir)
                 raise
 
-        self.run_process_command(destdir)
+    def get_temp_repo_path(self, scm_root, scm_branch):
+        scm_repo = scm_root.split("/")[-1]
+        process_id = os.getpid()
+        tmp_dir = (
+            "/tmp/pungi-temp-git-repos-"
+            + str(process_id)
+            + "/"
+            + scm_repo
+            + "-"
+            + scm_branch
+        )
+        return tmp_dir
+
+    def setup_repo(self, scm_root, scm_branch):
+        tmp_dir = self.get_temp_repo_path(scm_root, scm_branch)
+        if not os.path.isdir(tmp_dir):
+            makedirs(tmp_dir)
+            self._clone(scm_root, scm_branch, tmp_dir)
+        self.run_process_command(tmp_dir)
+        return tmp_dir
 
     def export_dir(self, scm_root, scm_dir, target_dir, scm_branch=None):
         scm_dir = scm_dir.lstrip("/")
         scm_branch = scm_branch or "master"
 
-        with temp_dir() as tmp_dir:
         self.log_debug(
             "Exporting directory %s from git %s (branch %s)..."
             % (scm_dir, scm_root, scm_branch)
         )
 
-            self._clone(scm_root, scm_branch, tmp_dir)
+        with lock:
+            tmp_dir = self.setup_repo(scm_root, scm_branch)
 
             copy_all(os.path.join(tmp_dir, scm_dir), target_dir)
 
@@ -215,7 +237,6 @@ class GitWrapper(ScmBase):
         scm_file = scm_file.lstrip("/")
         scm_branch = scm_branch or "master"
 
-        with temp_dir() as tmp_dir:
         target_path = os.path.join(target_dir, os.path.basename(scm_file))
 
         self.log_debug(
@@ -223,7 +244,8 @@ class GitWrapper(ScmBase):
             % (scm_file, scm_root, scm_branch)
         )
 
-            self._clone(scm_root, scm_branch, tmp_dir)
+        with lock:
+            tmp_dir = self.setup_repo(scm_root, scm_branch)
 
             makedirs(target_dir)
             shutil.copy2(os.path.join(tmp_dir, scm_file), target_path)

tests/test_scm.py

@@ -115,6 +115,10 @@ CREDENTIALS_CONFIG = {"credential_helper": "!ch"}
 
 
 class GitSCMTestCase(SCMBaseTest):
+    def tearDown(self):
+        shutil.rmtree("/tmp/pungi-temp-git-repos-%s" % os.getpid())
+        super(GitSCMTestCase, self).tearDown()
+
     def assertCalls(self, mock_run, url, branch, command=None, with_creds=False):
         git = GIT_WITH_CREDS if with_creds else ["git"]
         command = [command] if command else []