diff --git a/pungi/compose.py b/pungi/compose.py index ceb142c1..9c3fbf3d 100644 --- a/pungi/compose.py +++ b/pungi/compose.py @@ -209,7 +209,7 @@ class Compose(kobo.log.LoggingBase): scm_dict = os.path.join(self.config_dir, os.path.basename(scm_dict)) self.log_debug(msg) - tmp_dir = tempfile.mkdtemp(prefix="variants_file_") + tmp_dir = self.mkdtemp(prefix="variants_file_") get_file_from_scm(scm_dict, tmp_dir, logger=self._logger) shutil.copy2(os.path.join(tmp_dir, file_name), variants_file) shutil.rmtree(tmp_dir) @@ -379,3 +379,11 @@ class Compose(kobo.log.LoggingBase): return '%s_%s' % (version, milestone) return version + + def mkdtemp(self, arch=None, variant=None, suffix="", prefix="tmp"): + """ + Create and return a unique temporary directory under dir of + /work/{global,}/tmp[-]/ + """ + path = os.path.join(self.paths.work.tmp_dir(arch=arch, variant=variant)) + return tempfile.mkdtemp(suffix=suffix, prefix=prefix, dir=path) diff --git a/pungi/paths.py b/pungi/paths.py index 1183b93c..39db9569 100644 --- a/pungi/paths.py +++ b/pungi/paths.py @@ -282,16 +282,17 @@ class WorkPaths(object): makedirs(path) return path - def tmp_dir(self, arch, variant=None, create_dir=True): + def tmp_dir(self, arch=None, variant=None, create_dir=True): """ Examples: + work/global/tmp work/x86_64/tmp work/x86_64/tmp-Server """ dir_name = "tmp" if variant: dir_name += "-%s" % variant.uid - path = os.path.join(self.topdir(arch, create_dir=create_dir), dir_name) + path = os.path.join(self.topdir(arch=arch, create_dir=create_dir), dir_name) if create_dir: makedirs(path) return path diff --git a/pungi/phases/buildinstall.py b/pungi/phases/buildinstall.py index a1b2bde7..a9562bb1 100644 --- a/pungi/phases/buildinstall.py +++ b/pungi/phases/buildinstall.py @@ -18,7 +18,6 @@ import errno import os import time import pipes -import tempfile import shutil import re @@ -162,7 +161,7 @@ class BuildinstallPhase(PhaseBase): volid = get_volid(self.compose, arch, variant, escape_spaces=False, disc_type=disc_type) can_fail = self.compose.can_fail(variant, arch, 'buildinstall') with failable(self.compose, can_fail, variant, arch, 'buildinstall'): - tweak_buildinstall(buildinstall_dir, os_tree, arch, variant.uid, label, volid, kickstart_file) + tweak_buildinstall(self.compose, buildinstall_dir, os_tree, arch, variant.uid, label, volid, kickstart_file) link_boot_iso(self.compose, arch, variant, can_fail) @@ -187,7 +186,7 @@ def get_kickstart_file(compose): kickstart_name = os.path.basename(scm_dict) scm_dict = os.path.join(compose.config_dir, scm_dict) - tmp_dir = tempfile.mkdtemp(prefix="buildinstall_kickstart_") + tmp_dir = compose.mkdtemp(prefix="buildinstall_kickstart_") get_file_from_scm(scm_dict, tmp_dir, logger=compose._logger) src = os.path.join(tmp_dir, kickstart_name) shutil.copy2(src, kickstart_path) @@ -199,10 +198,10 @@ def get_kickstart_file(compose): # * it's quite trivial to replace volids # * it's not easy to replace menu titles # * we probably need to get this into lorax -def tweak_buildinstall(src, dst, arch, variant, label, volid, kickstart_file=None): +def tweak_buildinstall(compose, src, dst, arch, variant, label, volid, kickstart_file=None): volid_escaped = volid.replace(" ", r"\x20").replace("\\", "\\\\") volid_escaped_2 = volid_escaped.replace("\\", "\\\\") - tmp_dir = tempfile.mkdtemp(prefix="tweak_buildinstall_") + tmp_dir = compose.mkdtemp(prefix="tweak_buildinstall_") # verify src if not os.path.isdir(src): @@ -260,7 +259,7 @@ def tweak_buildinstall(src, dst, arch, variant, label, volid, kickstart_file=Non for image in images: if not os.path.isfile(image): continue - mount_tmp_dir = tempfile.mkdtemp(prefix="tweak_buildinstall") + mount_tmp_dir = compose.mkdtemp(prefix="tweak_buildinstall") # use guestmount to mount the image, which doesn't require root privileges # LIBGUESTFS_BACKEND=direct: running qemu directly without libvirt cmd = ["LIBGUESTFS_BACKEND=direct", "guestmount", "-a", image, "-m", "/dev/sda", mount_tmp_dir] diff --git a/pungi/phases/createrepo.py b/pungi/phases/createrepo.py index c751cede..05ffe4da 100644 --- a/pungi/phases/createrepo.py +++ b/pungi/phases/createrepo.py @@ -22,7 +22,6 @@ __all__ = ( import os import glob import shutil -import tempfile import threading from kobo.threads import ThreadPool, WorkerThread @@ -204,7 +203,7 @@ def get_productids_from_scm(compose): msg = "Getting product certificates from SCM..." compose.log_info("[BEGIN] %s" % msg) - tmp_dir = tempfile.mkdtemp(prefix="pungi_") + tmp_dir = compose.mkdtemp(prefix="pungi_") get_dir_from_scm(product_id, tmp_dir) for arch in compose.get_arches(): diff --git a/pungi/phases/gather/__init__.py b/pungi/phases/gather/__init__.py index 4eab85db..6b8da82d 100644 --- a/pungi/phases/gather/__init__.py +++ b/pungi/phases/gather/__init__.py @@ -15,7 +15,6 @@ import os -import tempfile import shutil import json @@ -334,7 +333,7 @@ def write_prepopulate_file(compose): scm_dict = os.path.join(compose.config_dir, os.path.basename(scm_dict)) compose.log_debug(msg) - tmp_dir = tempfile.mkdtemp(prefix="prepopulate_file_") + tmp_dir = compose.mkdtemp(prefix="prepopulate_file_") get_file_from_scm(scm_dict, tmp_dir, logger=compose._logger) shutil.copy2(os.path.join(tmp_dir, file_name), prepopulate_file) shutil.rmtree(tmp_dir) diff --git a/pungi/phases/gather/methods/method_deps.py b/pungi/phases/gather/methods/method_deps.py index 6d75d38f..b2e8f9a8 100644 --- a/pungi/phases/gather/methods/method_deps.py +++ b/pungi/phases/gather/methods/method_deps.py @@ -15,7 +15,6 @@ import os -import tempfile from kobo.shortcuts import run @@ -122,7 +121,7 @@ def resolve_deps(compose, arch, variant): cmd = pungi_wrapper.get_pungi_cmd(pungi_conf, destdir=tmp_dir, name=variant.uid, selfhosting=selfhosting, fulltree=fulltree, arch=yum_arch, full_archlist=True, greedy=greedy_method, cache_dir=cache_dir, lookaside_repos=lookaside_repos, multilib_methods=multilib_methods) # Use temp working directory directory as workaround for # https://bugzilla.redhat.com/show_bug.cgi?id=795137 - tmp_dir = tempfile.mkdtemp(prefix="pungi_") + tmp_dir = compose.mkdtemp(prefix="pungi_") try: run(cmd, logfile=pungi_log, show_cmd=True, workdir=tmp_dir, env=os.environ) finally: diff --git a/pungi/phases/init.py b/pungi/phases/init.py index 85360a72..44fc87b8 100644 --- a/pungi/phases/init.py +++ b/pungi/phases/init.py @@ -15,7 +15,6 @@ import os -import tempfile import shutil from kobo.shortcuts import run @@ -78,7 +77,7 @@ def write_global_comps(compose): scm_dict = os.path.join(compose.config_dir, scm_dict) compose.log_debug(msg) - tmp_dir = tempfile.mkdtemp(prefix="comps_") + tmp_dir = compose.mkdtemp(prefix="comps_") get_file_from_scm(scm_dict, tmp_dir, logger=compose._logger) shutil.copy2(os.path.join(tmp_dir, comps_name), comps_file_global) shutil.rmtree(tmp_dir) diff --git a/pungi/phases/product_img.py b/pungi/phases/product_img.py index bab7fb09..72827726 100644 --- a/pungi/phases/product_img.py +++ b/pungi/phases/product_img.py @@ -38,7 +38,6 @@ run/install/product/pyanaconda/installclasses -> ../installclasses import os import fnmatch -import tempfile import shutil import pipes @@ -112,7 +111,7 @@ def create_product_img(compose, arch, variant): compose.log_info("[BEGIN] %s" % msg) - product_tmp = tempfile.mkdtemp(prefix="product_img_") + product_tmp = compose.mkdtemp(prefix="product_img_") install_class = compose.conf["productimg_install_class"].copy() install_class["file"] = install_class["file"] % {"variant_id": variant.id.lower()} install_dir = os.path.join(product_tmp, "installclasses") @@ -120,7 +119,7 @@ def create_product_img(compose, arch, variant): get_file_from_scm(install_class, target_path=install_dir, logger=None) po_files = compose.conf["productimg_po_files"] - po_tmp = tempfile.mkdtemp(prefix="pofiles_") + po_tmp = compose.mkdtemp(prefix="pofiles_") get_dir_from_scm(po_files, po_tmp, logger=compose._logger) for po_file in os.listdir(po_tmp): if not po_file.endswith(".po"): @@ -132,7 +131,7 @@ def create_product_img(compose, arch, variant): shutil.rmtree(po_tmp) - mount_tmp = tempfile.mkdtemp(prefix="product_img_mount_") + mount_tmp = compose.mkdtemp(prefix="product_img_mount_") cmds = [ # allocate image "dd if=/dev/zero of=%s bs=1k count=5760" % pipes.quote(image), @@ -186,8 +185,8 @@ def rebuild_boot_iso(compose, arch, variant, package_sets): # remove the original boot.iso (created during buildinstall) from the os dir os.remove(boot_iso) - tmp_dir = tempfile.mkdtemp(prefix="boot_iso_") - mount_dir = tempfile.mkdtemp(prefix="boot_iso_mount_") + tmp_dir = compose.mkdtemp(prefix="boot_iso_") + mount_dir = compose.mkdtemp(prefix="boot_iso_mount_") cmd = "mount -o loop %s %s" % (pipes.quote(buildinstall_boot_iso), pipes.quote(mount_dir)) run(cmd, logfile=log_file, show_cmd=True) @@ -244,7 +243,7 @@ def rebuild_boot_iso(compose, arch, variant, package_sets): def explode_anaconda(compose, arch, variant, package_sets): - tmp_dir = tempfile.mkdtemp(prefix="anaconda_") + tmp_dir = compose.mkdtemp(prefix="anaconda_") scm_dict = { "scm": "rpm", "repo": "anaconda.%s" % arch, diff --git a/pungi/phases/test.py b/pungi/phases/test.py index e1e8b97e..2e9653f4 100644 --- a/pungi/phases/test.py +++ b/pungi/phases/test.py @@ -14,7 +14,6 @@ # along with this program; if not, see . -import tempfile import os from kobo.shortcuts import run @@ -70,7 +69,7 @@ def run_repoclosure(compose): cmd = repoclosure.get_repoclosure_cmd(repos=repos, lookaside=lookaside, arch=arches) # Use temp working directory directory as workaround for # https://bugzilla.redhat.com/show_bug.cgi?id=795137 - tmp_dir = tempfile.mkdtemp(prefix="repoclosure_") + tmp_dir = compose.mkdtemp(prefix="repoclosure_") try: run(cmd, logfile=compose.paths.log.log_file(arch, "repoclosure-%s" % variant), show_cmd=True, can_fail=True, workdir=tmp_dir) finally: @@ -93,7 +92,7 @@ def run_repoclosure(compose): cmd = repoclosure.get_repoclosure_cmd(repos=all_repos, arch=all_arches, builddeps=True) # Use temp working directory directory as workaround for # https://bugzilla.redhat.com/show_bug.cgi?id=795137 - tmp_dir = tempfile.mkdtemp(prefix="repoclosure_") + tmp_dir = compose.mkdtemp(prefix="repoclosure_") try: run(cmd, logfile=compose.paths.log.log_file("global", "repoclosure-builddeps"), show_cmd=True, can_fail=True, workdir=tmp_dir) finally: diff --git a/tests/helpers.py b/tests/helpers.py index c3983079..585eeec2 100644 --- a/tests/helpers.py +++ b/tests/helpers.py @@ -10,7 +10,7 @@ import tempfile import shutil import errno -from pungi.util import get_arch_variant_data +from pungi.util import get_arch_variant_data, makedirs from pungi import paths, checks @@ -106,6 +106,11 @@ class DummyCompose(object): result |= set(variant.arches) return sorted(result) + def mkdtemp(self, suffix="", prefix="tmp"): + tmpdir = os.path.join('/tmp', "tmp_" + self.compose_id) + makedirs(tmpdir) + return tempfile.mkdtemp(suffix=suffix, prefix=prefix, dir=tmpdir) + def touch(path, content=None): """Helper utility that creates an dummy file in given location. Directories diff --git a/tests/test_buildinstall.py b/tests/test_buildinstall.py index bea5616c..ce5439bc 100644 --- a/tests/test_buildinstall.py +++ b/tests/test_buildinstall.py @@ -347,13 +347,16 @@ class TestCopyFiles(PungiTestCase): mock.call(compose, 'amd64', compose.variants['Server'], escape_spaces=False, disc_type='dvd')]) self.assertItemsEqual( tweak_buildinstall.mock_calls, - [mock.call(self.topdir + '/work/x86_64/buildinstall', + [mock.call(compose, + self.topdir + '/work/x86_64/buildinstall', self.topdir + '/compose/Server/x86_64/os', 'x86_64', 'Server', '', 'Server.x86_64', 'kickstart'), - mock.call(self.topdir + '/work/amd64/buildinstall', + mock.call(compose, + self.topdir + '/work/amd64/buildinstall', self.topdir + '/compose/Server/amd64/os', 'amd64', 'Server', '', 'Server.amd64', 'kickstart'), - mock.call(self.topdir + '/work/amd64/buildinstall', + mock.call(compose, + self.topdir + '/work/amd64/buildinstall', self.topdir + '/compose/Client/amd64/os', 'amd64', 'Client', '', 'Client.amd64', 'kickstart')]) self.assertItemsEqual( @@ -389,13 +392,16 @@ class TestCopyFiles(PungiTestCase): mock.call(compose, 'amd64', compose.variants['Server'], escape_spaces=False, disc_type='dvd')]) self.assertItemsEqual( tweak_buildinstall.mock_calls, - [mock.call(self.topdir + '/work/x86_64/buildinstall/Server', + [mock.call(compose, + self.topdir + '/work/x86_64/buildinstall/Server', self.topdir + '/compose/Server/x86_64/os', 'x86_64', 'Server', '', 'Server.x86_64', 'kickstart'), - mock.call(self.topdir + '/work/amd64/buildinstall/Server', + mock.call(compose, + self.topdir + '/work/amd64/buildinstall/Server', self.topdir + '/compose/Server/amd64/os', 'amd64', 'Server', '', 'Server.amd64', 'kickstart'), - mock.call(self.topdir + '/work/amd64/buildinstall/Client', + mock.call(compose, + self.topdir + '/work/amd64/buildinstall/Client', self.topdir + '/compose/Client/amd64/os', 'amd64', 'Client', '', 'Client.amd64', 'kickstart')]) self.assertItemsEqual( @@ -435,13 +441,16 @@ class TestCopyFiles(PungiTestCase): mock.call(compose, 'amd64', compose.variants['Server'], escape_spaces=False, disc_type='dvd')]) self.assertItemsEqual( tweak_buildinstall.mock_calls, - [mock.call(self.topdir + '/work/x86_64/buildinstall/Server', + [mock.call(compose, + self.topdir + '/work/x86_64/buildinstall/Server', self.topdir + '/compose/Server/x86_64/os', 'x86_64', 'Server', '', 'Server.x86_64', 'kickstart'), - mock.call(self.topdir + '/work/amd64/buildinstall/Server', + mock.call(compose, + self.topdir + '/work/amd64/buildinstall/Server', self.topdir + '/compose/Server/amd64/os', 'amd64', 'Server', '', 'Server.amd64', 'kickstart'), - mock.call(self.topdir + '/work/amd64/buildinstall/Client', + mock.call(compose, + self.topdir + '/work/amd64/buildinstall/Client', self.topdir + '/compose/Client/amd64/os', 'amd64', 'Client', '', 'Client.amd64', 'kickstart')]) self.assertItemsEqual(link_boot_iso.mock_calls, []) diff --git a/tests/test_compose.py b/tests/test_compose.py index bc7cacce..f67a5a21 100644 --- a/tests/test_compose.py +++ b/tests/test_compose.py @@ -217,6 +217,8 @@ class ComposeTestCase(unittest.TestCase): @mock.patch('pungi.compose.ComposeInfo') def test_get_variant_arches_without_filter(self, ci): + ci.return_value.compose.id = 'composeid' + conf = ConfigWrapper( variants_file={'scm': 'file', 'repo': None, @@ -258,6 +260,8 @@ class ComposeTestCase(unittest.TestCase): @mock.patch('pungi.compose.ComposeInfo') def test_get_variant_arches_with_arch_filter(self, ci): + ci.return_value.compose.id = 'composeid' + conf = ConfigWrapper( variants_file={'scm': 'file', 'repo': None, @@ -298,6 +302,7 @@ class ComposeTestCase(unittest.TestCase): @mock.patch('pungi.compose.ComposeInfo') def test_get_variant_arches_with_variant_filter(self, ci): + ci.return_value.compose.id = 'composeid' ci.return_value.compose.respin = 2 ci.return_value.compose.date = '20160107' ci.return_value.compose.type = 'production' @@ -334,6 +339,7 @@ class ComposeTestCase(unittest.TestCase): @mock.patch('pungi.compose.ComposeInfo') def test_get_variant_arches_with_both_filters(self, ci): + ci.return_value.compose.id = 'composeid' ci.return_value.compose.respin = 2 ci.return_value.compose.date = '20160107' ci.return_value.compose.type = 'production' @@ -379,6 +385,28 @@ class ComposeTestCase(unittest.TestCase): mock.call('Excluding variant Server-Gluster: filtered by configuration.')] ) + @mock.patch('pungi.compose.ComposeInfo') + def test_mkdtemp(self, ci): + ci.return_value.compose.id = 'composeid' + conf = ConfigWrapper( + variants_file={'scm': 'file', + 'repo': None, + 'file': 'variants.xml'}, + release_name='Test', + release_version='1.0', + release_short='test', + release_type='ga', + release_is_layered=False, + release_internal=False, + tree_variants=['Server', 'Client', 'Server-optional'], + tree_arches=['x86_64'], + ) + compose = Compose(conf, self.tmp_dir) + d = compose.mkdtemp() + self.assertTrue(os.path.isdir(d)) + d = compose.mkdtemp(prefix='tweak_buildinstall') + self.assertTrue(os.path.isdir(d)) + class StatusTest(unittest.TestCase): def setUp(self):