diff --git a/pungi/phases/test.py b/pungi/phases/test.py
index ddc641fc..da7f0562 100644
--- a/pungi/phases/test.py
+++ b/pungi/phases/test.py
@@ -22,7 +22,7 @@ from pungi.wrappers import repoclosure
 from pungi.arch import get_valid_arches
 from pungi.phases.base import PhaseBase
 from pungi.phases.gather import get_lookaside_repos
-from pungi.util import rmtree, is_arch_multilib, failable
+from pungi.util import is_arch_multilib, failable, temp_dir
 
 
 class TestPhase(PhaseBase):
@@ -63,12 +63,16 @@ def run_repoclosure(compose):
                                                   repos=repos, lookaside=lookaside, arch=arches)
             # Use temp working directory directory as workaround for
             # https://bugzilla.redhat.com/show_bug.cgi?id=795137
-            tmp_dir = compose.mkdtemp(prefix="repoclosure_")
-            try:
-                run(cmd, logfile=compose.paths.log.log_file(arch, "repoclosure-%s" % variant),
-                    show_cmd=True, can_fail=True, workdir=tmp_dir)
-            finally:
-                rmtree(tmp_dir)
+            with temp_dir(prefix='repoclosure_') as tmp_dir:
+                # Ideally we would want show_cmd=True here to include the
+                # command in the logfile, but due to a bug in Kobo that would
+                # cause any error to be printed directly to stderr.
+                #  https://github.com/release-engineering/kobo/pull/26
+                try:
+                    run(cmd, logfile=compose.paths.log.log_file(arch, "repoclosure-%s" % variant),
+                        workdir=tmp_dir)
+                except RuntimeError as exc:
+                    compose.log_warning('Repoclosure failed for %s.%s\n%s' % (variant.uid, arch, exc))
 
     compose.log_info("[DONE ] %s" % msg)