almalinux/pungi
14
1
Fork 0
Code Issues Pull Requests Releases Activity

Create temporary dirs under compose's workdir

Browse Source 
Add compose.mkdtemp which creates and returns a unique temporary
directory under <compose_topdir>/work/{global,<arch>}/tmp[-<variant>]/

Change tempfile.mkdtemp in code to compose.mkdtemp, so temporary
dirs are created under the compose's workdir, this makes it more
easier to maintain the temporary dirs/files especially when compose
fails with amount of temporary files left there.

Signed-off-by: Qixiang Wan <qwan@redhat.com>
This commit is contained in:
Qixiang Wan 2017-01-09 15:40:24 +08:00
parent f6f84de1fb
commit 6fbf1e8f59
12 changed files with 81 additions and 37 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

10
pungi/compose.py
View File

@ -209,7 +209,7 @@ class Compose(kobo.log.LoggingBase):
scm_dict = os.path.join(self.config_dir, os.path.basename(scm_dict))
self.log_debug(msg)
tmp_dir = tempfile.mkdtemp(prefix="variants_file_")
tmp_dir = self.mkdtemp(prefix="variants_file_")
get_file_from_scm(scm_dict, tmp_dir, logger=self._logger)
shutil.copy2(os.path.join(tmp_dir, file_name), variants_file)
shutil.rmtree(tmp_dir)
@ -379,3 +379,11 @@ class Compose(kobo.log.LoggingBase):
return '%s_%s' % (version, milestone)
return version
def mkdtemp(self, arch=None, variant=None, suffix="", prefix="tmp"):
"""
Create and return a unique temporary directory under dir of
<compose_topdir>/work/{global,<arch>}/tmp[-<variant>]/
"""
path = os.path.join(self.paths.work.tmp_dir(arch=arch, variant=variant))
return tempfile.mkdtemp(suffix=suffix, prefix=prefix, dir=path)

5
pungi/paths.py
View File

@ -282,16 +282,17 @@ class WorkPaths(object):
makedirs(path)
return path
def tmp_dir(self, arch, variant=None, create_dir=True):
def tmp_dir(self, arch=None, variant=None, create_dir=True):
"""
Examples:
work/global/tmp
work/x86_64/tmp
work/x86_64/tmp-Server
"""
dir_name = "tmp"
if variant:
dir_name += "-%s" % variant.uid
path = os.path.join(self.topdir(arch, create_dir=create_dir), dir_name)
path = os.path.join(self.topdir(arch=arch, create_dir=create_dir), dir_name)
if create_dir:
makedirs(path)
return path

11
pungi/phases/buildinstall.py
View File

@ -18,7 +18,6 @@ import errno
import os
import time
import pipes
import tempfile
import shutil
import re
@ -162,7 +161,7 @@ class BuildinstallPhase(PhaseBase):
volid = get_volid(self.compose, arch, variant, escape_spaces=False, disc_type=disc_type)
can_fail = self.compose.can_fail(variant, arch, 'buildinstall')
with failable(self.compose, can_fail, variant, arch, 'buildinstall'):
tweak_buildinstall(buildinstall_dir, os_tree, arch, variant.uid, label, volid, kickstart_file)
tweak_buildinstall(self.compose, buildinstall_dir, os_tree, arch, variant.uid, label, volid, kickstart_file)
link_boot_iso(self.compose, arch, variant, can_fail)
@ -187,7 +186,7 @@ def get_kickstart_file(compose):
kickstart_name = os.path.basename(scm_dict)
scm_dict = os.path.join(compose.config_dir, scm_dict)
tmp_dir = tempfile.mkdtemp(prefix="buildinstall_kickstart_")
tmp_dir = compose.mkdtemp(prefix="buildinstall_kickstart_")
get_file_from_scm(scm_dict, tmp_dir, logger=compose._logger)
src = os.path.join(tmp_dir, kickstart_name)
shutil.copy2(src, kickstart_path)
@ -199,10 +198,10 @@ def get_kickstart_file(compose):
# * it's quite trivial to replace volids
# * it's not easy to replace menu titles
# * we probably need to get this into lorax
def tweak_buildinstall(src, dst, arch, variant, label, volid, kickstart_file=None):
def tweak_buildinstall(compose, src, dst, arch, variant, label, volid, kickstart_file=None):
volid_escaped = volid.replace(" ", r"\x20").replace("\\", "\\\\")
volid_escaped_2 = volid_escaped.replace("\\", "\\\\")
tmp_dir = tempfile.mkdtemp(prefix="tweak_buildinstall_")
tmp_dir = compose.mkdtemp(prefix="tweak_buildinstall_")
# verify src
if not os.path.isdir(src):
@ -260,7 +259,7 @@ def tweak_buildinstall(src, dst, arch, variant, label, volid, kickstart_file=Non
for image in images:
if not os.path.isfile(image):
continue
mount_tmp_dir = tempfile.mkdtemp(prefix="tweak_buildinstall")
mount_tmp_dir = compose.mkdtemp(prefix="tweak_buildinstall")
# use guestmount to mount the image, which doesn't require root privileges
# LIBGUESTFS_BACKEND=direct: running qemu directly without libvirt
cmd = ["LIBGUESTFS_BACKEND=direct", "guestmount", "-a", image, "-m", "/dev/sda", mount_tmp_dir]

3
pungi/phases/createrepo.py
View File

@ -22,7 +22,6 @@ __all__ = (
import os
import glob
import shutil
import tempfile
import threading
from kobo.threads import ThreadPool, WorkerThread
@ -204,7 +203,7 @@ def get_productids_from_scm(compose):
msg = "Getting product certificates from SCM..."
compose.log_info("[BEGIN] %s" % msg)
tmp_dir = tempfile.mkdtemp(prefix="pungi_")
tmp_dir = compose.mkdtemp(prefix="pungi_")
get_dir_from_scm(product_id, tmp_dir)
for arch in compose.get_arches():

3
pungi/phases/gather/__init__.py
View File

@ -15,7 +15,6 @@
import os
import tempfile
import shutil
import json
@ -334,7 +333,7 @@ def write_prepopulate_file(compose):
scm_dict = os.path.join(compose.config_dir, os.path.basename(scm_dict))
compose.log_debug(msg)
tmp_dir = tempfile.mkdtemp(prefix="prepopulate_file_")
tmp_dir = compose.mkdtemp(prefix="prepopulate_file_")
get_file_from_scm(scm_dict, tmp_dir, logger=compose._logger)
shutil.copy2(os.path.join(tmp_dir, file_name), prepopulate_file)
shutil.rmtree(tmp_dir)

3
pungi/phases/gather/methods/method_deps.py
View File

@ -15,7 +15,6 @@
import os
import tempfile
from kobo.shortcuts import run
@ -122,7 +121,7 @@ def resolve_deps(compose, arch, variant):
cmd = pungi_wrapper.get_pungi_cmd(pungi_conf, destdir=tmp_dir, name=variant.uid, selfhosting=selfhosting, fulltree=fulltree, arch=yum_arch, full_archlist=True, greedy=greedy_method, cache_dir=cache_dir, lookaside_repos=lookaside_repos, multilib_methods=multilib_methods)
# Use temp working directory directory as workaround for
# https://bugzilla.redhat.com/show_bug.cgi?id=795137
tmp_dir = tempfile.mkdtemp(prefix="pungi_")
tmp_dir = compose.mkdtemp(prefix="pungi_")
try:
run(cmd, logfile=pungi_log, show_cmd=True, workdir=tmp_dir, env=os.environ)
finally:

3
pungi/phases/init.py
View File

@ -15,7 +15,6 @@
import os
import tempfile
import shutil
from kobo.shortcuts import run
@ -78,7 +77,7 @@ def write_global_comps(compose):
scm_dict = os.path.join(compose.config_dir, scm_dict)
compose.log_debug(msg)
tmp_dir = tempfile.mkdtemp(prefix="comps_")
tmp_dir = compose.mkdtemp(prefix="comps_")
get_file_from_scm(scm_dict, tmp_dir, logger=compose._logger)
shutil.copy2(os.path.join(tmp_dir, comps_name), comps_file_global)
shutil.rmtree(tmp_dir)

13
pungi/phases/product_img.py
View File

@ -38,7 +38,6 @@ run/install/product/pyanaconda/installclasses -> ../installclasses
import os
import fnmatch
import tempfile
import shutil
import pipes
@ -112,7 +111,7 @@ def create_product_img(compose, arch, variant):
compose.log_info("[BEGIN] %s" % msg)
product_tmp = tempfile.mkdtemp(prefix="product_img_")
product_tmp = compose.mkdtemp(prefix="product_img_")
install_class = compose.conf["productimg_install_class"].copy()
install_class["file"] = install_class["file"] % {"variant_id": variant.id.lower()}
install_dir = os.path.join(product_tmp, "installclasses")
@ -120,7 +119,7 @@ def create_product_img(compose, arch, variant):
get_file_from_scm(install_class, target_path=install_dir, logger=None)
po_files = compose.conf["productimg_po_files"]
po_tmp = tempfile.mkdtemp(prefix="pofiles_")
po_tmp = compose.mkdtemp(prefix="pofiles_")
get_dir_from_scm(po_files, po_tmp, logger=compose._logger)
for po_file in os.listdir(po_tmp):
if not po_file.endswith(".po"):
@ -132,7 +131,7 @@ def create_product_img(compose, arch, variant):
shutil.rmtree(po_tmp)
mount_tmp = tempfile.mkdtemp(prefix="product_img_mount_")
mount_tmp = compose.mkdtemp(prefix="product_img_mount_")
cmds = [
# allocate image
"dd if=/dev/zero of=%s bs=1k count=5760" % pipes.quote(image),
@ -186,8 +185,8 @@ def rebuild_boot_iso(compose, arch, variant, package_sets):
# remove the original boot.iso (created during buildinstall) from the os dir
os.remove(boot_iso)
tmp_dir = tempfile.mkdtemp(prefix="boot_iso_")
mount_dir = tempfile.mkdtemp(prefix="boot_iso_mount_")
tmp_dir = compose.mkdtemp(prefix="boot_iso_")
mount_dir = compose.mkdtemp(prefix="boot_iso_mount_")
cmd = "mount -o loop %s %s" % (pipes.quote(buildinstall_boot_iso), pipes.quote(mount_dir))
run(cmd, logfile=log_file, show_cmd=True)
@ -244,7 +243,7 @@ def rebuild_boot_iso(compose, arch, variant, package_sets):
def explode_anaconda(compose, arch, variant, package_sets):
tmp_dir = tempfile.mkdtemp(prefix="anaconda_")
tmp_dir = compose.mkdtemp(prefix="anaconda_")
scm_dict = {
"scm": "rpm",
"repo": "anaconda.%s" % arch,

5
pungi/phases/test.py
View File

@ -14,7 +14,6 @@
# along with this program; if not, see <https://gnu.org/licenses/>.
import tempfile
import os
from kobo.shortcuts import run
@ -70,7 +69,7 @@ def run_repoclosure(compose):
cmd = repoclosure.get_repoclosure_cmd(repos=repos, lookaside=lookaside, arch=arches)
# Use temp working directory directory as workaround for
# https://bugzilla.redhat.com/show_bug.cgi?id=795137
tmp_dir = tempfile.mkdtemp(prefix="repoclosure_")
tmp_dir = compose.mkdtemp(prefix="repoclosure_")
try:
run(cmd, logfile=compose.paths.log.log_file(arch, "repoclosure-%s" % variant), show_cmd=True, can_fail=True, workdir=tmp_dir)
finally:
@ -93,7 +92,7 @@ def run_repoclosure(compose):
cmd = repoclosure.get_repoclosure_cmd(repos=all_repos, arch=all_arches, builddeps=True)
# Use temp working directory directory as workaround for
# https://bugzilla.redhat.com/show_bug.cgi?id=795137
tmp_dir = tempfile.mkdtemp(prefix="repoclosure_")
tmp_dir = compose.mkdtemp(prefix="repoclosure_")
try:
run(cmd, logfile=compose.paths.log.log_file("global", "repoclosure-builddeps"), show_cmd=True, can_fail=True, workdir=tmp_dir)
finally:

7
tests/helpers.py
View File

@ -10,7 +10,7 @@ import tempfile
import shutil
import errno
from pungi.util import get_arch_variant_data
from pungi.util import get_arch_variant_data, makedirs
from pungi import paths, checks
@ -106,6 +106,11 @@ class DummyCompose(object):
result |= set(variant.arches)
return sorted(result)
def mkdtemp(self, suffix="", prefix="tmp"):
tmpdir = os.path.join('/tmp', "tmp_" + self.compose_id)
makedirs(tmpdir)
return tempfile.mkdtemp(suffix=suffix, prefix=prefix, dir=tmpdir)
def touch(path, content=None):
"""Helper utility that creates an dummy file in given location. Directories

27
tests/test_buildinstall.py
View File

@ -347,13 +347,16 @@ class TestCopyFiles(PungiTestCase):
mock.call(compose, 'amd64', compose.variants['Server'], escape_spaces=False, disc_type='dvd')])
self.assertItemsEqual(
tweak_buildinstall.mock_calls,
[mock.call(self.topdir + '/work/x86_64/buildinstall',
[mock.call(compose,
self.topdir + '/work/x86_64/buildinstall',
self.topdir + '/compose/Server/x86_64/os',
'x86_64', 'Server', '', 'Server.x86_64', 'kickstart'),
mock.call(self.topdir + '/work/amd64/buildinstall',
mock.call(compose,
self.topdir + '/work/amd64/buildinstall',
self.topdir + '/compose/Server/amd64/os',
'amd64', 'Server', '', 'Server.amd64', 'kickstart'),
mock.call(self.topdir + '/work/amd64/buildinstall',
mock.call(compose,
self.topdir + '/work/amd64/buildinstall',
self.topdir + '/compose/Client/amd64/os',
'amd64', 'Client', '', 'Client.amd64', 'kickstart')])
self.assertItemsEqual(
@ -389,13 +392,16 @@ class TestCopyFiles(PungiTestCase):
mock.call(compose, 'amd64', compose.variants['Server'], escape_spaces=False, disc_type='dvd')])
self.assertItemsEqual(
tweak_buildinstall.mock_calls,
[mock.call(self.topdir + '/work/x86_64/buildinstall/Server',
[mock.call(compose,
self.topdir + '/work/x86_64/buildinstall/Server',
self.topdir + '/compose/Server/x86_64/os',
'x86_64', 'Server', '', 'Server.x86_64', 'kickstart'),
mock.call(self.topdir + '/work/amd64/buildinstall/Server',
mock.call(compose,
self.topdir + '/work/amd64/buildinstall/Server',
self.topdir + '/compose/Server/amd64/os',
'amd64', 'Server', '', 'Server.amd64', 'kickstart'),
mock.call(self.topdir + '/work/amd64/buildinstall/Client',
mock.call(compose,
self.topdir + '/work/amd64/buildinstall/Client',
self.topdir + '/compose/Client/amd64/os',
'amd64', 'Client', '', 'Client.amd64', 'kickstart')])
self.assertItemsEqual(
@ -435,13 +441,16 @@ class TestCopyFiles(PungiTestCase):
mock.call(compose, 'amd64', compose.variants['Server'], escape_spaces=False, disc_type='dvd')])
self.assertItemsEqual(
tweak_buildinstall.mock_calls,
[mock.call(self.topdir + '/work/x86_64/buildinstall/Server',
[mock.call(compose,
self.topdir + '/work/x86_64/buildinstall/Server',
self.topdir + '/compose/Server/x86_64/os',
'x86_64', 'Server', '', 'Server.x86_64', 'kickstart'),
mock.call(self.topdir + '/work/amd64/buildinstall/Server',
mock.call(compose,
self.topdir + '/work/amd64/buildinstall/Server',
self.topdir + '/compose/Server/amd64/os',
'amd64', 'Server', '', 'Server.amd64', 'kickstart'),
mock.call(self.topdir + '/work/amd64/buildinstall/Client',
mock.call(compose,
self.topdir + '/work/amd64/buildinstall/Client',
self.topdir + '/compose/Client/amd64/os',
'amd64', 'Client', '', 'Client.amd64', 'kickstart')])
self.assertItemsEqual(link_boot_iso.mock_calls, [])

28
tests/test_compose.py
View File

@ -217,6 +217,8 @@ class ComposeTestCase(unittest.TestCase):
@mock.patch('pungi.compose.ComposeInfo')
def test_get_variant_arches_without_filter(self, ci):
ci.return_value.compose.id = 'composeid'
conf = ConfigWrapper(
variants_file={'scm': 'file',
'repo': None,
@ -258,6 +260,8 @@ class ComposeTestCase(unittest.TestCase):
@mock.patch('pungi.compose.ComposeInfo')
def test_get_variant_arches_with_arch_filter(self, ci):
ci.return_value.compose.id = 'composeid'
conf = ConfigWrapper(
variants_file={'scm': 'file',
'repo': None,
@ -298,6 +302,7 @@ class ComposeTestCase(unittest.TestCase):
@mock.patch('pungi.compose.ComposeInfo')
def test_get_variant_arches_with_variant_filter(self, ci):
ci.return_value.compose.id = 'composeid'
ci.return_value.compose.respin = 2
ci.return_value.compose.date = '20160107'
ci.return_value.compose.type = 'production'
@ -334,6 +339,7 @@ class ComposeTestCase(unittest.TestCase):
@mock.patch('pungi.compose.ComposeInfo')
def test_get_variant_arches_with_both_filters(self, ci):
ci.return_value.compose.id = 'composeid'
ci.return_value.compose.respin = 2
ci.return_value.compose.date = '20160107'
ci.return_value.compose.type = 'production'
@ -379,6 +385,28 @@ class ComposeTestCase(unittest.TestCase):
mock.call('Excluding variant Server-Gluster: filtered by configuration.')]
)
@mock.patch('pungi.compose.ComposeInfo')
def test_mkdtemp(self, ci):
ci.return_value.compose.id = 'composeid'
conf = ConfigWrapper(
variants_file={'scm': 'file',
'repo': None,
'file': 'variants.xml'},
release_name='Test',
release_version='1.0',
release_short='test',
release_type='ga',
release_is_layered=False,
release_internal=False,
tree_variants=['Server', 'Client', 'Server-optional'],
tree_arches=['x86_64'],
)
compose = Compose(conf, self.tmp_dir)
d = compose.mkdtemp()
self.assertTrue(os.path.isdir(d))
d = compose.mkdtemp(prefix='tweak_buildinstall')
self.assertTrue(os.path.isdir(d))
class StatusTest(unittest.TestCase):
def setUp(self):