Create temporary dirs under compose's workdir

Add compose.mkdtemp which creates and returns a unique temporary
directory under <compose_topdir>/work/{global,<arch>}/tmp[-<variant>]/

Change tempfile.mkdtemp in code to compose.mkdtemp, so temporary
dirs are created under the compose's workdir, this makes it more
easier to maintain the temporary dirs/files especially when compose
fails with amount of temporary files left there.

Signed-off-by: Qixiang Wan <qwan@redhat.com>

pungi/compose.py

@@ -209,7 +209,7 @@ class Compose(kobo.log.LoggingBase):
                 scm_dict = os.path.join(self.config_dir, os.path.basename(scm_dict))
 
             self.log_debug(msg)
-            tmp_dir = tempfile.mkdtemp(prefix="variants_file_")
+            tmp_dir = self.mkdtemp(prefix="variants_file_")
             get_file_from_scm(scm_dict, tmp_dir, logger=self._logger)
             shutil.copy2(os.path.join(tmp_dir, file_name), variants_file)
             shutil.rmtree(tmp_dir)
@@ -379,3 +379,11 @@ class Compose(kobo.log.LoggingBase):
             return '%s_%s' % (version, milestone)
 
         return version
+
+    def mkdtemp(self, arch=None, variant=None, suffix="", prefix="tmp"):
+        """
+        Create and return a unique temporary directory under dir of
+        <compose_topdir>/work/{global,<arch>}/tmp[-<variant>]/
+        """
+        path = os.path.join(self.paths.work.tmp_dir(arch=arch, variant=variant))
+        return tempfile.mkdtemp(suffix=suffix, prefix=prefix, dir=path)

pungi/paths.py

@@ -282,16 +282,17 @@ class WorkPaths(object):
             makedirs(path)
         return path
 
-    def tmp_dir(self, arch, variant=None, create_dir=True):
+    def tmp_dir(self, arch=None, variant=None, create_dir=True):
         """
         Examples:
+            work/global/tmp
             work/x86_64/tmp
             work/x86_64/tmp-Server
         """
         dir_name = "tmp"
         if variant:
             dir_name += "-%s" % variant.uid
-        path = os.path.join(self.topdir(arch, create_dir=create_dir), dir_name)
+        path = os.path.join(self.topdir(arch=arch, create_dir=create_dir), dir_name)
         if create_dir:
             makedirs(path)
         return path

pungi/phases/buildinstall.py

@@ -18,7 +18,6 @@ import errno
 import os
 import time
 import pipes
-import tempfile
 import shutil
 import re
 
@@ -162,7 +161,7 @@ class BuildinstallPhase(PhaseBase):
                 volid = get_volid(self.compose, arch, variant, escape_spaces=False, disc_type=disc_type)
                 can_fail = self.compose.can_fail(variant, arch, 'buildinstall')
                 with failable(self.compose, can_fail, variant, arch, 'buildinstall'):
-                    tweak_buildinstall(buildinstall_dir, os_tree, arch, variant.uid, label, volid, kickstart_file)
+                    tweak_buildinstall(self.compose, buildinstall_dir, os_tree, arch, variant.uid, label, volid, kickstart_file)
                     link_boot_iso(self.compose, arch, variant, can_fail)
 
 
@@ -187,7 +186,7 @@ def get_kickstart_file(compose):
         kickstart_name = os.path.basename(scm_dict)
         scm_dict = os.path.join(compose.config_dir, scm_dict)
 
-    tmp_dir = tempfile.mkdtemp(prefix="buildinstall_kickstart_")
+    tmp_dir = compose.mkdtemp(prefix="buildinstall_kickstart_")
     get_file_from_scm(scm_dict, tmp_dir, logger=compose._logger)
     src = os.path.join(tmp_dir, kickstart_name)
     shutil.copy2(src, kickstart_path)
@@ -199,10 +198,10 @@ def get_kickstart_file(compose):
 # * it's quite trivial to replace volids
 # * it's not easy to replace menu titles
 # * we probably need to get this into lorax
-def tweak_buildinstall(src, dst, arch, variant, label, volid, kickstart_file=None):
+def tweak_buildinstall(compose, src, dst, arch, variant, label, volid, kickstart_file=None):
     volid_escaped = volid.replace(" ", r"\x20").replace("\\", "\\\\")
     volid_escaped_2 = volid_escaped.replace("\\", "\\\\")
-    tmp_dir = tempfile.mkdtemp(prefix="tweak_buildinstall_")
+    tmp_dir = compose.mkdtemp(prefix="tweak_buildinstall_")
 
     # verify src
     if not os.path.isdir(src):
@@ -260,7 +259,7 @@ def tweak_buildinstall(src, dst, arch, variant, label, volid, kickstart_file=Non
     for image in images:
         if not os.path.isfile(image):
             continue
-        mount_tmp_dir = tempfile.mkdtemp(prefix="tweak_buildinstall")
+        mount_tmp_dir = compose.mkdtemp(prefix="tweak_buildinstall")
         # use guestmount to mount the image, which doesn't require root privileges
         # LIBGUESTFS_BACKEND=direct: running qemu directly without libvirt
         cmd = ["LIBGUESTFS_BACKEND=direct", "guestmount", "-a", image, "-m", "/dev/sda", mount_tmp_dir]

pungi/phases/createrepo.py

@@ -22,7 +22,6 @@ __all__ = (
 import os
 import glob
 import shutil
-import tempfile
 import threading
 
 from kobo.threads import ThreadPool, WorkerThread
@@ -204,7 +203,7 @@ def get_productids_from_scm(compose):
     msg = "Getting product certificates from SCM..."
     compose.log_info("[BEGIN] %s" % msg)
 
-    tmp_dir = tempfile.mkdtemp(prefix="pungi_")
+    tmp_dir = compose.mkdtemp(prefix="pungi_")
     get_dir_from_scm(product_id, tmp_dir)
 
     for arch in compose.get_arches():

pungi/phases/gather/__init__.py

@@ -15,7 +15,6 @@
 
 
 import os
-import tempfile
 import shutil
 import json
 
@@ -334,7 +333,7 @@ def write_prepopulate_file(compose):
             scm_dict = os.path.join(compose.config_dir, os.path.basename(scm_dict))
 
         compose.log_debug(msg)
-        tmp_dir = tempfile.mkdtemp(prefix="prepopulate_file_")
+        tmp_dir = compose.mkdtemp(prefix="prepopulate_file_")
         get_file_from_scm(scm_dict, tmp_dir, logger=compose._logger)
         shutil.copy2(os.path.join(tmp_dir, file_name), prepopulate_file)
         shutil.rmtree(tmp_dir)

pungi/phases/gather/methods/method_deps.py

@@ -15,7 +15,6 @@
 
 
 import os
-import tempfile
 
 from kobo.shortcuts import run
 
@@ -122,7 +121,7 @@ def resolve_deps(compose, arch, variant):
     cmd = pungi_wrapper.get_pungi_cmd(pungi_conf, destdir=tmp_dir, name=variant.uid, selfhosting=selfhosting, fulltree=fulltree, arch=yum_arch, full_archlist=True, greedy=greedy_method, cache_dir=cache_dir, lookaside_repos=lookaside_repos, multilib_methods=multilib_methods)
     # Use temp working directory directory as workaround for
     # https://bugzilla.redhat.com/show_bug.cgi?id=795137
-    tmp_dir = tempfile.mkdtemp(prefix="pungi_")
+    tmp_dir = compose.mkdtemp(prefix="pungi_")
     try:
         run(cmd, logfile=pungi_log, show_cmd=True, workdir=tmp_dir, env=os.environ)
     finally:

pungi/phases/init.py

@@ -15,7 +15,6 @@
 
 
 import os
-import tempfile
 import shutil
 
 from kobo.shortcuts import run
@@ -78,7 +77,7 @@ def write_global_comps(compose):
             scm_dict = os.path.join(compose.config_dir, scm_dict)
 
         compose.log_debug(msg)
-        tmp_dir = tempfile.mkdtemp(prefix="comps_")
+        tmp_dir = compose.mkdtemp(prefix="comps_")
         get_file_from_scm(scm_dict, tmp_dir, logger=compose._logger)
         shutil.copy2(os.path.join(tmp_dir, comps_name), comps_file_global)
         shutil.rmtree(tmp_dir)

pungi/phases/product_img.py

@@ -38,7 +38,6 @@ run/install/product/pyanaconda/installclasses -> ../installclasses
 
 import os
 import fnmatch
-import tempfile
 import shutil
 import pipes
 
@@ -112,7 +111,7 @@ def create_product_img(compose, arch, variant):
 
     compose.log_info("[BEGIN] %s" % msg)
 
-    product_tmp = tempfile.mkdtemp(prefix="product_img_")
+    product_tmp = compose.mkdtemp(prefix="product_img_")
     install_class = compose.conf["productimg_install_class"].copy()
     install_class["file"] = install_class["file"] % {"variant_id": variant.id.lower()}
     install_dir = os.path.join(product_tmp, "installclasses")
@@ -120,7 +119,7 @@ def create_product_img(compose, arch, variant):
     get_file_from_scm(install_class, target_path=install_dir, logger=None)
 
     po_files = compose.conf["productimg_po_files"]
-    po_tmp = tempfile.mkdtemp(prefix="pofiles_")
+    po_tmp = compose.mkdtemp(prefix="pofiles_")
     get_dir_from_scm(po_files, po_tmp, logger=compose._logger)
     for po_file in os.listdir(po_tmp):
         if not po_file.endswith(".po"):
@@ -132,7 +131,7 @@ def create_product_img(compose, arch, variant):
 
     shutil.rmtree(po_tmp)
 
-    mount_tmp = tempfile.mkdtemp(prefix="product_img_mount_")
+    mount_tmp = compose.mkdtemp(prefix="product_img_mount_")
     cmds = [
         # allocate image
         "dd if=/dev/zero of=%s bs=1k count=5760" % pipes.quote(image),
@@ -186,8 +185,8 @@ def rebuild_boot_iso(compose, arch, variant, package_sets):
     # remove the original boot.iso (created during buildinstall) from the os dir
     os.remove(boot_iso)
 
-    tmp_dir = tempfile.mkdtemp(prefix="boot_iso_")
+    tmp_dir = compose.mkdtemp(prefix="boot_iso_")
-    mount_dir = tempfile.mkdtemp(prefix="boot_iso_mount_")
+    mount_dir = compose.mkdtemp(prefix="boot_iso_mount_")
 
     cmd = "mount -o loop %s %s" % (pipes.quote(buildinstall_boot_iso), pipes.quote(mount_dir))
     run(cmd, logfile=log_file, show_cmd=True)
@@ -244,7 +243,7 @@ def rebuild_boot_iso(compose, arch, variant, package_sets):
 
 
 def explode_anaconda(compose, arch, variant, package_sets):
-    tmp_dir = tempfile.mkdtemp(prefix="anaconda_")
+    tmp_dir = compose.mkdtemp(prefix="anaconda_")
     scm_dict = {
         "scm": "rpm",
         "repo": "anaconda.%s" % arch,

pungi/phases/test.py

@@ -14,7 +14,6 @@
 # along with this program; if not, see <https://gnu.org/licenses/>.
 
 
-import tempfile
 import os
 
 from kobo.shortcuts import run
@@ -70,7 +69,7 @@ def run_repoclosure(compose):
             cmd = repoclosure.get_repoclosure_cmd(repos=repos, lookaside=lookaside, arch=arches)
             # Use temp working directory directory as workaround for
             # https://bugzilla.redhat.com/show_bug.cgi?id=795137
-            tmp_dir = tempfile.mkdtemp(prefix="repoclosure_")
+            tmp_dir = compose.mkdtemp(prefix="repoclosure_")
             try:
                 run(cmd, logfile=compose.paths.log.log_file(arch, "repoclosure-%s" % variant), show_cmd=True, can_fail=True, workdir=tmp_dir)
             finally:
@@ -93,7 +92,7 @@ def run_repoclosure(compose):
     cmd = repoclosure.get_repoclosure_cmd(repos=all_repos, arch=all_arches, builddeps=True)
     # Use temp working directory directory as workaround for
     # https://bugzilla.redhat.com/show_bug.cgi?id=795137
-    tmp_dir = tempfile.mkdtemp(prefix="repoclosure_")
+    tmp_dir = compose.mkdtemp(prefix="repoclosure_")
     try:
         run(cmd, logfile=compose.paths.log.log_file("global", "repoclosure-builddeps"), show_cmd=True, can_fail=True, workdir=tmp_dir)
     finally:

tests/helpers.py

@@ -10,7 +10,7 @@ import tempfile
 import shutil
 import errno
 
-from pungi.util import get_arch_variant_data
+from pungi.util import get_arch_variant_data, makedirs
 from pungi import paths, checks
 
 
@@ -106,6 +106,11 @@ class DummyCompose(object):
             result |= set(variant.arches)
         return sorted(result)
 
+    def mkdtemp(self, suffix="", prefix="tmp"):
+        tmpdir = os.path.join('/tmp', "tmp_" + self.compose_id)
+        makedirs(tmpdir)
+        return tempfile.mkdtemp(suffix=suffix, prefix=prefix, dir=tmpdir)
+
 
 def touch(path, content=None):
     """Helper utility that creates an dummy file in given location. Directories

tests/test_buildinstall.py

@@ -347,13 +347,16 @@ class TestCopyFiles(PungiTestCase):
              mock.call(compose, 'amd64', compose.variants['Server'], escape_spaces=False, disc_type='dvd')])
         self.assertItemsEqual(
             tweak_buildinstall.mock_calls,
-            [mock.call(self.topdir + '/work/x86_64/buildinstall',
+            [mock.call(compose,
+                       self.topdir + '/work/x86_64/buildinstall',
                        self.topdir + '/compose/Server/x86_64/os',
                        'x86_64', 'Server', '', 'Server.x86_64', 'kickstart'),
-             mock.call(self.topdir + '/work/amd64/buildinstall',
+             mock.call(compose,
+                       self.topdir + '/work/amd64/buildinstall',
                        self.topdir + '/compose/Server/amd64/os',
                        'amd64', 'Server', '', 'Server.amd64', 'kickstart'),
-             mock.call(self.topdir + '/work/amd64/buildinstall',
+             mock.call(compose,
+                       self.topdir + '/work/amd64/buildinstall',
                        self.topdir + '/compose/Client/amd64/os',
                        'amd64', 'Client', '', 'Client.amd64', 'kickstart')])
         self.assertItemsEqual(
@@ -389,13 +392,16 @@ class TestCopyFiles(PungiTestCase):
              mock.call(compose, 'amd64', compose.variants['Server'], escape_spaces=False, disc_type='dvd')])
         self.assertItemsEqual(
             tweak_buildinstall.mock_calls,
-            [mock.call(self.topdir + '/work/x86_64/buildinstall/Server',
+            [mock.call(compose,
+                       self.topdir + '/work/x86_64/buildinstall/Server',
                        self.topdir + '/compose/Server/x86_64/os',
                        'x86_64', 'Server', '', 'Server.x86_64', 'kickstart'),
-             mock.call(self.topdir + '/work/amd64/buildinstall/Server',
+             mock.call(compose,
+                       self.topdir + '/work/amd64/buildinstall/Server',
                        self.topdir + '/compose/Server/amd64/os',
                        'amd64', 'Server', '', 'Server.amd64', 'kickstart'),
-             mock.call(self.topdir + '/work/amd64/buildinstall/Client',
+             mock.call(compose,
+                       self.topdir + '/work/amd64/buildinstall/Client',
                        self.topdir + '/compose/Client/amd64/os',
                        'amd64', 'Client', '', 'Client.amd64', 'kickstart')])
         self.assertItemsEqual(
@@ -435,13 +441,16 @@ class TestCopyFiles(PungiTestCase):
              mock.call(compose, 'amd64', compose.variants['Server'], escape_spaces=False, disc_type='dvd')])
         self.assertItemsEqual(
             tweak_buildinstall.mock_calls,
-            [mock.call(self.topdir + '/work/x86_64/buildinstall/Server',
+            [mock.call(compose,
+                       self.topdir + '/work/x86_64/buildinstall/Server',
                        self.topdir + '/compose/Server/x86_64/os',
                        'x86_64', 'Server', '', 'Server.x86_64', 'kickstart'),
-             mock.call(self.topdir + '/work/amd64/buildinstall/Server',
+             mock.call(compose,
+                       self.topdir + '/work/amd64/buildinstall/Server',
                        self.topdir + '/compose/Server/amd64/os',
                        'amd64', 'Server', '', 'Server.amd64', 'kickstart'),
-             mock.call(self.topdir + '/work/amd64/buildinstall/Client',
+             mock.call(compose,
+                       self.topdir + '/work/amd64/buildinstall/Client',
                        self.topdir + '/compose/Client/amd64/os',
                        'amd64', 'Client', '', 'Client.amd64', 'kickstart')])
         self.assertItemsEqual(link_boot_iso.mock_calls, [])

tests/test_compose.py

@@ -217,6 +217,8 @@ class ComposeTestCase(unittest.TestCase):
 
     @mock.patch('pungi.compose.ComposeInfo')
     def test_get_variant_arches_without_filter(self, ci):
+        ci.return_value.compose.id = 'composeid'
+
         conf = ConfigWrapper(
             variants_file={'scm': 'file',
                            'repo': None,
@@ -258,6 +260,8 @@ class ComposeTestCase(unittest.TestCase):
 
     @mock.patch('pungi.compose.ComposeInfo')
     def test_get_variant_arches_with_arch_filter(self, ci):
+        ci.return_value.compose.id = 'composeid'
+
         conf = ConfigWrapper(
             variants_file={'scm': 'file',
                            'repo': None,
@@ -298,6 +302,7 @@ class ComposeTestCase(unittest.TestCase):
 
     @mock.patch('pungi.compose.ComposeInfo')
     def test_get_variant_arches_with_variant_filter(self, ci):
+        ci.return_value.compose.id = 'composeid'
         ci.return_value.compose.respin = 2
         ci.return_value.compose.date = '20160107'
         ci.return_value.compose.type = 'production'
@@ -334,6 +339,7 @@ class ComposeTestCase(unittest.TestCase):
 
     @mock.patch('pungi.compose.ComposeInfo')
     def test_get_variant_arches_with_both_filters(self, ci):
+        ci.return_value.compose.id = 'composeid'
         ci.return_value.compose.respin = 2
         ci.return_value.compose.date = '20160107'
         ci.return_value.compose.type = 'production'
@@ -379,6 +385,28 @@ class ComposeTestCase(unittest.TestCase):
              mock.call('Excluding variant Server-Gluster: filtered by configuration.')]
         )
 
+    @mock.patch('pungi.compose.ComposeInfo')
+    def test_mkdtemp(self, ci):
+        ci.return_value.compose.id = 'composeid'
+        conf = ConfigWrapper(
+            variants_file={'scm': 'file',
+                           'repo': None,
+                           'file': 'variants.xml'},
+            release_name='Test',
+            release_version='1.0',
+            release_short='test',
+            release_type='ga',
+            release_is_layered=False,
+            release_internal=False,
+            tree_variants=['Server', 'Client', 'Server-optional'],
+            tree_arches=['x86_64'],
+        )
+        compose = Compose(conf, self.tmp_dir)
+        d = compose.mkdtemp()
+        self.assertTrue(os.path.isdir(d))
+        d = compose.mkdtemp(prefix='tweak_buildinstall')
+        self.assertTrue(os.path.isdir(d))
+
 
 class StatusTest(unittest.TestCase):
     def setUp(self):