Split repoclosure into separate phase

Move repoclosure out from test phase into its own phase and
run parallel with image building phases(osbs, imagebuild, ...)
to speed things up.

JIRA: RHELCMP-8
Signed-off-by: Haibo Lin <hlin@redhat.com>

doc/_static/phases.svg

@@ -11,12 +11,12 @@
    inkscape:export-xdpi="90"
    inkscape:export-filename="/home/lsedlar/repos/pungi/doc/_static/phases.png"
    sodipodi:docname="phases.svg"
-   inkscape:version="0.92.4 (unknown)"
+   inkscape:version="0.92.2 5c3e80d, 2017-08-06"
    version="1.1"
    id="svg2"
-   viewBox="0 0 771.66458 221.50019"
+   viewBox="0 0 669.66458 255.18195"
-   height="221.50018"
+   height="255.18195"
-   width="771.66455">
+   width="669.66455">
   <sodipodi:namedview
      fit-margin-bottom="0"
      fit-margin-right="0"
@@ -27,16 +27,16 @@
      inkscape:document-rotation="0"
      units="px"
      inkscape:window-maximized="1"
-     inkscape:window-y="0"
+     inkscape:window-y="1"
      inkscape:window-x="0"
-     inkscape:window-height="1015"
+     inkscape:window-height="1035"
      inkscape:window-width="1920"
      showgrid="false"
-     inkscape:current-layer="g3668"
+     inkscape:current-layer="layer1"
      inkscape:document-units="px"
-     inkscape:cy="137.85275"
+     inkscape:cy="127.3243"
-     inkscape:cx="228.82868"
+     inkscape:cx="420.82921"
-     inkscape:zoom="3.3064935"
+     inkscape:zoom="1.5"
      inkscape:pageshadow="2"
      inkscape:pageopacity="1"
      borderopacity="1.0"
@@ -69,7 +69,7 @@
         <dc:format>image/svg+xml</dc:format>
         <dc:type
            rdf:resource="http://purl.org/dc/dcmitype/StillImage" />
-        <dc:title />
+        <dc:title></dc:title>
       </cc:Work>
     </rdf:RDF>
   </metadata>
@@ -77,7 +77,7 @@
      inkscape:label="Vrstva 1"
      inkscape:groupmode="layer"
      id="layer1"
-     transform="matrix(1.066667,0,0,1.066667,-2.473231,-910.85239)">
+     transform="matrix(1.066667,0,0,1.066667,-78.473216,-910.85239)">
     <g
        id="g3411"
        transform="translate(71.99326,-80.817124)">
@@ -95,7 +95,7 @@
          x="51.554729"
          style="font-style:normal;font-weight:normal;line-height:0%;font-family:sans-serif;letter-spacing:0px;word-spacing:0px;fill:#000000;fill-opacity:1;stroke:none;stroke-width:1px;stroke-linecap:butt;stroke-linejoin:miter;stroke-opacity:1"
          xml:space="preserve"><tspan
-           style="font-size:13.1479px;line-height:1.25"
+           style="font-size:13.14789963px;line-height:1.25"
            y="970.26605"
            x="51.554729"
            id="tspan3362"
@@ -118,7 +118,7 @@
          x="556.95709"
          style="font-style:normal;font-variant:normal;font-weight:normal;font-stretch:normal;line-height:0%;font-family:sans-serif;-inkscape-font-specification:'sans-serif, Normal';text-align:start;letter-spacing:0px;word-spacing:0px;writing-mode:lr-tb;text-anchor:start;fill:#000000;fill-opacity:1;stroke:none;stroke-width:1px;stroke-linecap:butt;stroke-linejoin:miter;stroke-opacity:1"
          xml:space="preserve"><tspan
-           style="font-size:13.1475px;line-height:1.25"
+           style="font-size:13.14750004px;line-height:1.25"
            id="tspan3391"
            sodipodi:role="line"
            x="556.95709"
@@ -141,7 +141,7 @@
          x="557.61566"
          style="font-style:normal;font-weight:normal;line-height:0%;font-family:sans-serif;letter-spacing:0px;word-spacing:0px;fill:#ffffff;fill-opacity:1;stroke:none;stroke-width:1px;stroke-linecap:butt;stroke-linejoin:miter;stroke-opacity:1"
          xml:space="preserve"><tspan
-           style="font-size:13.1479px;line-height:1.25"
+           style="font-size:13.14789963px;line-height:1.25"
            y="971.33813"
            x="557.61566"
            id="tspan3398"
@@ -164,7 +164,7 @@
          x="6.2600794"
          style="font-style:normal;font-weight:normal;line-height:0%;font-family:sans-serif;letter-spacing:0px;word-spacing:0px;fill:#000000;fill-opacity:1;stroke:none;stroke-width:1px;stroke-linecap:butt;stroke-linejoin:miter;stroke-opacity:1"
          xml:space="preserve"><tspan
-           style="font-size:13.1479px;line-height:1.25"
+           style="font-size:13.14789963px;line-height:1.25"
            y="891.1604"
            x="6.2600794"
            id="tspan3358"
@@ -198,7 +198,7 @@
              id="tspan3366"
              x="105.76799"
              y="891.06732"
-             style="font-size:13.1479px;line-height:1.25">Buildinstall</tspan></text>
+             style="font-size:13.14789963px;line-height:1.25">Buildinstall</tspan></text>
       </g>
       <g
          id="g3639">
@@ -220,7 +220,7 @@
              id="tspan3370"
              x="106.1384"
              y="923.25934"
-             style="font-size:13.1479px;line-height:1.25">Gather</tspan></text>
+             style="font-size:13.14789963px;line-height:1.25">Gather</tspan></text>
       </g>
       <g
          id="g3647"
@@ -246,7 +246,7 @@
              id="tspan3374"
              x="165.23042"
              y="923.25934"
-             style="font-size:13.14789963px;line-height:1.25">ExtraFiles</tspan></text>
+             style="font-size:13.1479px;line-height:1.25">ExtraFiles</tspan></text>
       </g>
       <g
          id="g3658"
@@ -269,7 +269,7 @@
              id="tspan3378"
              x="243.95874"
              y="921.86945"
-             style="font-size:13.14789963px;line-height:1.25">Createrepo</tspan></text>
+             style="font-size:13.1479px;line-height:1.25">Createrepo</tspan></text>
       </g>
       <g
          transform="translate(-150.564,114.11662)"
@@ -292,7 +292,7 @@
              x="256.90588"
              sodipodi:role="line"
              id="tspan3406"
-             style="font-size:13.1479px;line-height:1.25">OSTree</tspan></text>
+             style="font-size:13.14789963px;line-height:1.25">OSTree</tspan></text>
       </g>
       <g
          id="g288"
@@ -323,11 +323,10 @@
       </g>
     </g>
     <g
-       transform="translate(-29.683562,-0.34408888)"
+       id="g1061">
-       id="g236">
       <g
          id="g3458"
-         transform="translate(28.723958,-80.473035)">
+         transform="translate(-0.959604,-80.817124)">
         <rect
            y="420.13605"
            x="953.49097"
@@ -342,14 +341,14 @@
            x="422.99252"
            y="971.54041"
            id="text3384"><tspan
-             style="font-size:13.1479px;line-height:1.25"
+             style="font-size:13.14789963px;line-height:1.25"
              sodipodi:role="line"
              id="tspan3386"
              x="422.99252"
              y="971.54041">Createiso</tspan></text>
       </g>
       <g
-         transform="translate(28.467511,-84.181232)"
+         transform="translate(-1.216051,-84.525321)"
          id="g3453">
         <rect
            style="fill:#73d216;fill-rule:evenodd;stroke:none;stroke-width:1px;stroke-linecap:butt;stroke-linejoin:miter;stroke-opacity:1"
@@ -365,14 +364,14 @@
            x="422.69772"
            y="1006.4276"
            id="text3388"><tspan
-             style="font-size:13.1479px;line-height:1.25"
+             style="font-size:13.14789963px;line-height:1.25"
              sodipodi:role="line"
              id="tspan3390"
              x="422.69772"
              y="1006.4276">LiveImages</tspan></text>
       </g>
       <g
-         transform="translate(28.467511,-88.141877)"
+         transform="translate(-1.216051,-88.485966)"
          id="g3448">
         <rect
            style="fill:#f57900;fill-rule:evenodd;stroke:none;stroke-width:1px;stroke-linecap:butt;stroke-linejoin:miter;stroke-opacity:1"
@@ -388,14 +387,14 @@
            x="422.69772"
            y="1042.8416"
            id="text3392"><tspan
-             style="font-size:13.1479px;line-height:1.25"
+             style="font-size:13.14789963px;line-height:1.25"
              sodipodi:role="line"
              id="tspan3394"
              x="422.69772"
              y="1042.8416">ImageBuild</tspan></text>
       </g>
       <g
-         transform="translate(27.760419,-92.458101)"
+         transform="translate(-1.923143,-92.80219)"
          id="g3443">
         <rect
            transform="matrix(0,1,1,0,0,0)"
@@ -411,38 +410,37 @@
            x="423.40482"
            style="font-style:normal;font-weight:normal;line-height:0%;font-family:sans-serif;text-align:start;letter-spacing:0px;word-spacing:0px;writing-mode:lr-tb;text-anchor:start;fill:#000000;fill-opacity:1;stroke:none;stroke-width:1px;stroke-linecap:butt;stroke-linejoin:miter;stroke-opacity:1"
            xml:space="preserve"><tspan
-             style="font-size:13.1479px;line-height:1.25"
+             style="font-size:13.14789963px;line-height:1.25"
              y="1079.6111"
              x="423.40482"
              sodipodi:role="line"
              id="tspan3434">LiveMedia</tspan></text>
       </g>
       <g
-         id="g306"
+         id="g204">
-         transform="translate(-41.476764,-46.1016)">
         <rect
            transform="matrix(0,1,1,0,0,0)"
-           y="490.33765"
+           y="419.17731"
-           x="1048.9327"
+           x="1002.4871"
            height="101.85102"
            width="26.295755"
            id="rect290"
            style="fill:#c17d11;fill-rule:evenodd;stroke:none;stroke-width:1px;stroke-linecap:butt;stroke-linejoin:miter;stroke-opacity:1" />
         <text
            id="text294"
-           y="1065.7078"
+           y="1019.2621"
-           x="492.642"
+           x="421.48166"
            style="font-style:normal;font-variant:normal;font-weight:normal;font-stretch:normal;line-height:0%;font-family:sans-serif;-inkscape-font-specification:'sans-serif, Normal';text-align:start;letter-spacing:0px;word-spacing:0px;writing-mode:lr-tb;text-anchor:start;fill:#000000;fill-opacity:1;stroke:none;stroke-width:1px;stroke-linecap:butt;stroke-linejoin:miter;stroke-opacity:1"
            xml:space="preserve"><tspan
              style="font-size:12px;line-height:0"
              id="tspan301"
              sodipodi:role="line"
-             x="492.642"
+             x="421.48166"
-             y="1065.7078">OSBS</tspan></text>
+             y="1019.2621">OSBS</tspan></text>
       </g>
       <g
          id="g3819"
-         transform="translate(0,-16.949078)">
+         transform="translate(-29.683562,-17.293167)">
         <rect
            transform="matrix(0,1,1,0,0,0)"
            y="448.86087"
@@ -461,7 +459,51 @@
              y="1069.0087"
              x="451.16522"
              sodipodi:role="line"
-             style="font-size:13.1479px;line-height:1.25">ExtraIsos</tspan></text>
+             style="font-size:13.14789963px;line-height:1.25">ExtraIsos</tspan></text>
+      </g>
+      <g
+         id="g1031">
+        <rect
+           transform="matrix(0,1,1,0,0,0)"
+           style="fill:#5ed4ec;fill-opacity:1;fill-rule:evenodd;stroke:none;stroke-width:1px;stroke-linecap:butt;stroke-linejoin:miter;stroke-opacity:1"
+           id="rect206"
+           width="26.295755"
+           height="102.36562"
+           x="1066.8611"
+           y="418.66275" />
+        <text
+           id="text210"
+           y="1084.9105"
+           x="421.51923"
+           style="font-style:normal;font-weight:normal;line-height:0%;font-family:sans-serif;letter-spacing:0px;word-spacing:0px;fill:#000000;fill-opacity:1;stroke:none;stroke-width:1px;stroke-linecap:butt;stroke-linejoin:miter;stroke-opacity:1"
+           xml:space="preserve"><tspan
+             y="1084.9105"
+             x="421.51923"
+             id="tspan208"
+             sodipodi:role="line"
+             style="font-size:13.14789963px;line-height:1.25">Repoclosure</tspan></text>
+      </g>
+      <g
+         id="g1031">
+        <rect
+           transform="matrix(0,1,1,0,0,0)"
+           style="fill:#5ed4ec;fill-opacity:1;fill-rule:evenodd;stroke:none;stroke-width:1px;stroke-linecap:butt;stroke-linejoin:miter;stroke-opacity:1"
+           id="rect206"
+           width="26.295755"
+           height="102.36562"
+           x="1066.8611"
+           y="418.66275" />
+        <text
+           id="text210"
+           y="1084.9105"
+           x="421.51923"
+           style="font-style:normal;font-weight:normal;line-height:0%;font-family:sans-serif;letter-spacing:0px;word-spacing:0px;fill:#000000;fill-opacity:1;stroke:none;stroke-width:1px;stroke-linecap:butt;stroke-linejoin:miter;stroke-opacity:1"
+           xml:space="preserve"><tspan
+             y="1084.9105"
+             x="421.51923"
+             id="tspan208"
+             sodipodi:role="line"
+             style="font-size:13.1479px;line-height:1.25">Repoclosure</tspan></text>
       </g>
     </g>
   </g>

doc/phases.rst

@@ -132,6 +132,13 @@ Creates bootable media that carry an ostree repository as a payload. These
 images are created by running ``lorax`` with special templates. Again it runs
 in Koji runroot.
 
+Repoclosure
+-----------
+
+Run ``repoclosure`` on each repository. By default errors are only reported
+in the log, the compose will still be considered a success. The actual error
+has to be looked up in the compose logs directory. Configuration allows customizing this.
+
 ImageChecksum
 -------------
 
@@ -145,12 +152,7 @@ Test
 
 This phase is supposed to run some sanity checks on the finished compose.
 
-The first test is to run ``repoclosure`` on each repository. By default errors
+The only test is to check all images listed the metadata and verify that they
-are only reported in the log, the compose will still be considered a success.
-The actual error has to be looked up in the compose logs directory.
-Configuration allows customizing this.
-
-The other test is to check all images listed the metadata and verify that they
 look sane. For ISO files headers are checked to verify the format is correct,
 and for bootable media a check is run to verify they have properties that allow
 booting.

pungi/phases/__init__.py

@@ -27,6 +27,7 @@ from .createiso import CreateisoPhase  # noqa
 from .extra_isos import ExtraIsosPhase  # noqa
 from .live_images import LiveImagesPhase  # noqa
 from .image_build import ImageBuildPhase  # noqa
+from .repoclosure import RepoclosurePhase  # noqa
 from .test import TestPhase  # noqa
 from .image_checksum import ImageChecksumPhase  # noqa
 from .livemedia_phase import LiveMediaPhase  # noqa

pungi/phases/repoclosure.py

@@ -0,0 +1,141 @@
+# -*- coding: utf-8 -*-
+
+
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; version 2 of the License.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Library General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, see <https://gnu.org/licenses/>.
+
+
+import glob
+import os
+import shutil
+
+from kobo.shortcuts import run
+
+from pungi.wrappers import repoclosure
+from pungi.arch import get_valid_arches
+from pungi.phases.base import PhaseBase
+from pungi.phases.gather import get_lookaside_repos, get_gather_methods
+from pungi.util import is_arch_multilib, temp_dir, get_arch_variant_data
+
+
+class RepoclosurePhase(PhaseBase):
+    name = "repoclosure"
+
+    def run(self):
+        run_repoclosure(self.compose)
+
+
+def run_repoclosure(compose):
+    msg = "Running repoclosure"
+    compose.log_info("[BEGIN] %s" % msg)
+
+    # Variant repos
+    for arch in compose.get_arches():
+        is_multilib = is_arch_multilib(compose.conf, arch)
+        arches = get_valid_arches(arch, is_multilib)
+        for variant in compose.get_variants(arch=arch):
+            if variant.is_empty:
+                continue
+
+            conf = get_arch_variant_data(
+                compose.conf, "repoclosure_strictness", arch, variant
+            )
+            if conf and conf[-1] == "off":
+                continue
+
+            prefix = "%s-repoclosure" % compose.compose_id
+            lookaside = {}
+            if variant.parent:
+                repo_id = "%s-%s.%s" % (prefix, variant.parent.uid, arch)
+                repo_dir = compose.paths.compose.repository(
+                    arch=arch, variant=variant.parent
+                )
+                lookaside[repo_id] = repo_dir
+
+            repos = {}
+            repo_id = "%s-%s.%s" % (prefix, variant.uid, arch)
+            repo_dir = compose.paths.compose.repository(arch=arch, variant=variant)
+            repos[repo_id] = repo_dir
+
+            for i, lookaside_url in enumerate(
+                get_lookaside_repos(compose, arch, variant)
+            ):
+                lookaside[
+                    "%s-lookaside-%s.%s-%s" % (compose.compose_id, variant.uid, arch, i)
+                ] = lookaside_url
+
+            logfile = compose.paths.log.log_file(arch, "repoclosure-%s" % variant)
+
+            try:
+                _, methods = get_gather_methods(compose, variant)
+                if methods == "hybrid":
+                    # Using hybrid solver, no repoclosure command is available.
+                    pattern = compose.paths.log.log_file(
+                        arch, "hybrid-depsolver-%s-iter-*" % variant
+                    )
+                    fus_logs = sorted(glob.glob(pattern))
+                    repoclosure.extract_from_fus_logs(fus_logs, logfile)
+                else:
+                    _run_repoclosure_cmd(compose, repos, lookaside, arches, logfile)
+            except RuntimeError as exc:
+                if conf and conf[-1] == "fatal":
+                    raise
+                else:
+                    compose.log_warning(
+                        "Repoclosure failed for %s.%s\n%s" % (variant.uid, arch, exc)
+                    )
+            finally:
+                if methods != "hybrid":
+                    _delete_repoclosure_cache_dirs(compose)
+
+    compose.log_info("[DONE ] %s" % msg)
+
+
+def _delete_repoclosure_cache_dirs(compose):
+    if "dnf" == compose.conf["repoclosure_backend"]:
+        from dnf.const import SYSTEM_CACHEDIR
+        from dnf.util import am_i_root
+        from dnf.yum.misc import getCacheDir
+
+        if am_i_root():
+            top_cache_dir = SYSTEM_CACHEDIR
+        else:
+            top_cache_dir = getCacheDir()
+    else:
+        from yum.misc import getCacheDir
+
+        top_cache_dir = getCacheDir()
+
+    for name in os.listdir(top_cache_dir):
+        if name.startswith(compose.compose_id):
+            cache_path = os.path.join(top_cache_dir, name)
+            if os.path.isdir(cache_path):
+                shutil.rmtree(cache_path)
+            else:
+                os.remove(cache_path)
+
+
+def _run_repoclosure_cmd(compose, repos, lookaside, arches, logfile):
+    cmd = repoclosure.get_repoclosure_cmd(
+        backend=compose.conf["repoclosure_backend"],
+        repos=repos,
+        lookaside=lookaside,
+        arch=arches,
+    )
+    # Use temp working directory directory as workaround for
+    # https://bugzilla.redhat.com/show_bug.cgi?id=795137
+    with temp_dir(prefix="repoclosure_") as tmp_dir:
+        # Ideally we would want show_cmd=True here to include the
+        # command in the logfile, but due to a bug in Kobo that would
+        # cause any error to be printed directly to stderr.
+        #  https://github.com/release-engineering/kobo/pull/26
+        run(cmd, logfile=logfile, workdir=tmp_dir, show_cmd=True)

pungi/phases/test.py

@@ -14,134 +14,19 @@
 # along with this program; if not, see <https://gnu.org/licenses/>.
 
 
-import glob
 import os
-import shutil
 
-from kobo.shortcuts import run
-
-from pungi.wrappers import repoclosure
-from pungi.arch import get_valid_arches
 from pungi.phases.base import PhaseBase
-from pungi.phases.gather import get_lookaside_repos, get_gather_methods
+from pungi.util import failable, get_arch_variant_data
-from pungi.util import is_arch_multilib, failable, temp_dir, get_arch_variant_data
 
 
 class TestPhase(PhaseBase):
     name = "test"
 
     def run(self):
-        run_repoclosure(self.compose)
         check_image_sanity(self.compose)
 
 
-def run_repoclosure(compose):
-    msg = "Running repoclosure"
-    compose.log_info("[BEGIN] %s" % msg)
-
-    # Variant repos
-    for arch in compose.get_arches():
-        is_multilib = is_arch_multilib(compose.conf, arch)
-        arches = get_valid_arches(arch, is_multilib)
-        for variant in compose.get_variants(arch=arch):
-            if variant.is_empty:
-                continue
-
-            conf = get_arch_variant_data(
-                compose.conf, "repoclosure_strictness", arch, variant
-            )
-            if conf and conf[-1] == "off":
-                continue
-
-            prefix = "%s-repoclosure" % compose.compose_id
-            lookaside = {}
-            if variant.parent:
-                repo_id = "%s-%s.%s" % (prefix, variant.parent.uid, arch)
-                repo_dir = compose.paths.compose.repository(
-                    arch=arch, variant=variant.parent
-                )
-                lookaside[repo_id] = repo_dir
-
-            repos = {}
-            repo_id = "%s-%s.%s" % (prefix, variant.uid, arch)
-            repo_dir = compose.paths.compose.repository(arch=arch, variant=variant)
-            repos[repo_id] = repo_dir
-
-            for i, lookaside_url in enumerate(
-                get_lookaside_repos(compose, arch, variant)
-            ):
-                lookaside[
-                    "%s-lookaside-%s.%s-%s" % (compose.compose_id, variant.uid, arch, i)
-                ] = lookaside_url
-
-            logfile = compose.paths.log.log_file(arch, "repoclosure-%s" % variant)
-
-            try:
-                _, methods = get_gather_methods(compose, variant)
-                if methods == "hybrid":
-                    # Using hybrid solver, no repoclosure command is available.
-                    pattern = compose.paths.log.log_file(
-                        arch, "hybrid-depsolver-%s-iter-*" % variant
-                    )
-                    fus_logs = sorted(glob.glob(pattern))
-                    repoclosure.extract_from_fus_logs(fus_logs, logfile)
-                else:
-                    _run_repoclosure_cmd(compose, repos, lookaside, arches, logfile)
-            except RuntimeError as exc:
-                if conf and conf[-1] == "fatal":
-                    raise
-                else:
-                    compose.log_warning(
-                        "Repoclosure failed for %s.%s\n%s" % (variant.uid, arch, exc)
-                    )
-            finally:
-                if methods != "hybrid":
-                    _delete_repoclosure_cache_dirs(compose)
-
-    compose.log_info("[DONE ] %s" % msg)
-
-
-def _delete_repoclosure_cache_dirs(compose):
-    if "dnf" == compose.conf["repoclosure_backend"]:
-        from dnf.const import SYSTEM_CACHEDIR
-        from dnf.util import am_i_root
-        from dnf.yum.misc import getCacheDir
-
-        if am_i_root():
-            top_cache_dir = SYSTEM_CACHEDIR
-        else:
-            top_cache_dir = getCacheDir()
-    else:
-        from yum.misc import getCacheDir
-
-        top_cache_dir = getCacheDir()
-
-    for name in os.listdir(top_cache_dir):
-        if name.startswith(compose.compose_id):
-            cache_path = os.path.join(top_cache_dir, name)
-            if os.path.isdir(cache_path):
-                shutil.rmtree(cache_path)
-            else:
-                os.remove(cache_path)
-
-
-def _run_repoclosure_cmd(compose, repos, lookaside, arches, logfile):
-    cmd = repoclosure.get_repoclosure_cmd(
-        backend=compose.conf["repoclosure_backend"],
-        repos=repos,
-        lookaside=lookaside,
-        arch=arches,
-    )
-    # Use temp working directory directory as workaround for
-    # https://bugzilla.redhat.com/show_bug.cgi?id=795137
-    with temp_dir(prefix="repoclosure_") as tmp_dir:
-        # Ideally we would want show_cmd=True here to include the
-        # command in the logfile, but due to a bug in Kobo that would
-        # cause any error to be printed directly to stderr.
-        #  https://github.com/release-engineering/kobo/pull/26
-        run(cmd, logfile=logfile, workdir=tmp_dir, show_cmd=True)
-
-
 def check_image_sanity(compose):
     """
     Go through all images in manifest and make basic sanity tests on them. If

pungi/scripts/pungi_koji.py

@@ -349,6 +349,7 @@ def run_compose(
     image_build_phase = pungi.phases.ImageBuildPhase(compose)
     osbs_phase = pungi.phases.OSBSPhase(compose)
     image_checksum_phase = pungi.phases.ImageChecksumPhase(compose)
+    repoclosure_phase = pungi.phases.RepoclosurePhase(compose)
     test_phase = pungi.phases.TestPhase(compose)
 
     # check if all config options are set
@@ -467,6 +468,7 @@ def run_compose(
         image_build_phase,
         livemedia_phase,
         osbs_phase,
+        repoclosure_phase,
     )
     compose_images_phase = pungi.phases.WeaverPhase(compose, compose_images_schema)
     compose_images_phase.start()

tests/test_repoclosure_phase.py

@@ -0,0 +1,236 @@
+# -*- coding: utf-8 -*-
+
+
+try:
+    import unittest2 as unittest
+except ImportError:
+    import unittest
+
+import mock
+import six
+
+import pungi.phases.repoclosure as repoclosure_phase
+from tests.helpers import DummyCompose, PungiTestCase, mk_boom
+
+try:
+    import dnf  # noqa: F401
+
+    HAS_DNF = True
+except ImportError:
+    HAS_DNF = False
+
+try:
+    import yum  # noqa: F401
+
+    HAS_YUM = True
+except ImportError:
+    HAS_YUM = False
+
+
+class TestRepoclosure(PungiTestCase):
+    def setUp(self):
+        super(TestRepoclosure, self).setUp()
+        self.maxDiff = None
+
+    def _get_repo(self, compose_id, variant, arch, path=None):
+        path = path or arch + "/os"
+        return {
+            "%s-repoclosure-%s.%s" % (compose_id, variant, arch): self.topdir
+            + "/compose/%s/%s" % (variant, path)
+        }
+
+    @mock.patch("pungi.wrappers.repoclosure.get_repoclosure_cmd")
+    @mock.patch("pungi.phases.repoclosure.run")
+    def test_repoclosure_skip_if_disabled(self, mock_run, mock_grc):
+        compose = DummyCompose(
+            self.topdir, {"repoclosure_strictness": [("^.*$", {"*": "off"})]}
+        )
+        repoclosure_phase.run_repoclosure(compose)
+
+        self.assertEqual(mock_grc.call_args_list, [])
+
+    @unittest.skipUnless(HAS_YUM, "YUM is not available")
+    @mock.patch("pungi.wrappers.repoclosure.get_repoclosure_cmd")
+    @mock.patch("pungi.phases.repoclosure.run")
+    def test_repoclosure_default_backend(self, mock_run, mock_grc):
+        with mock.patch("six.PY2", new=True):
+            compose = DummyCompose(self.topdir, {})
+
+        repoclosure_phase.run_repoclosure(compose)
+
+        six.assertCountEqual(
+            self,
+            mock_grc.call_args_list,
+            [
+                mock.call(
+                    backend="yum",
+                    arch=["amd64", "x86_64", "noarch"],
+                    lookaside={},
+                    repos=self._get_repo(compose.compose_id, "Everything", "amd64"),
+                ),
+                mock.call(
+                    backend="yum",
+                    arch=["amd64", "x86_64", "noarch"],
+                    lookaside={},
+                    repos=self._get_repo(compose.compose_id, "Client", "amd64"),
+                ),
+                mock.call(
+                    backend="yum",
+                    arch=["amd64", "x86_64", "noarch"],
+                    lookaside={},
+                    repos=self._get_repo(compose.compose_id, "Server", "amd64"),
+                ),
+                mock.call(
+                    backend="yum",
+                    arch=["x86_64", "noarch"],
+                    lookaside={},
+                    repos=self._get_repo(compose.compose_id, "Server", "x86_64"),
+                ),
+                mock.call(
+                    backend="yum",
+                    arch=["x86_64", "noarch"],
+                    lookaside={},
+                    repos=self._get_repo(compose.compose_id, "Everything", "x86_64"),
+                ),
+            ],
+        )
+
+    @unittest.skipUnless(HAS_DNF, "DNF is not available")
+    @mock.patch("pungi.wrappers.repoclosure.get_repoclosure_cmd")
+    @mock.patch("pungi.phases.repoclosure.run")
+    def test_repoclosure_dnf_backend(self, mock_run, mock_grc):
+        compose = DummyCompose(self.topdir, {"repoclosure_backend": "dnf"})
+        repoclosure_phase.run_repoclosure(compose)
+
+        six.assertCountEqual(
+            self,
+            mock_grc.call_args_list,
+            [
+                mock.call(
+                    backend="dnf",
+                    arch=["amd64", "x86_64", "noarch"],
+                    lookaside={},
+                    repos=self._get_repo(compose.compose_id, "Everything", "amd64"),
+                ),
+                mock.call(
+                    backend="dnf",
+                    arch=["amd64", "x86_64", "noarch"],
+                    lookaside={},
+                    repos=self._get_repo(compose.compose_id, "Client", "amd64"),
+                ),
+                mock.call(
+                    backend="dnf",
+                    arch=["amd64", "x86_64", "noarch"],
+                    lookaside={},
+                    repos=self._get_repo(compose.compose_id, "Server", "amd64"),
+                ),
+                mock.call(
+                    backend="dnf",
+                    arch=["x86_64", "noarch"],
+                    lookaside={},
+                    repos=self._get_repo(compose.compose_id, "Server", "x86_64"),
+                ),
+                mock.call(
+                    backend="dnf",
+                    arch=["x86_64", "noarch"],
+                    lookaside={},
+                    repos=self._get_repo(compose.compose_id, "Everything", "x86_64"),
+                ),
+            ],
+        )
+
+    @mock.patch("glob.glob")
+    @mock.patch("pungi.wrappers.repoclosure.extract_from_fus_logs")
+    @mock.patch("pungi.wrappers.repoclosure.get_repoclosure_cmd")
+    @mock.patch("pungi.phases.repoclosure.run")
+    def test_repoclosure_hybrid_variant(self, mock_run, mock_grc, effl, glob):
+        compose = DummyCompose(
+            self.topdir, {"repoclosure_backend": "dnf", "gather_method": "hybrid"}
+        )
+        f = mock.Mock()
+        glob.return_value = [f]
+
+        def _log(a, v):
+            return compose.paths.log.log_file(a, "repoclosure-%s" % compose.variants[v])
+
+        repoclosure_phase.run_repoclosure(compose)
+
+        self.assertEqual(mock_grc.call_args_list, [])
+        six.assertCountEqual(
+            self,
+            effl.call_args_list,
+            [
+                mock.call([f], _log("amd64", "Everything")),
+                mock.call([f], _log("amd64", "Client")),
+                mock.call([f], _log("amd64", "Server")),
+                mock.call([f], _log("x86_64", "Server")),
+                mock.call([f], _log("x86_64", "Everything")),
+            ],
+        )
+
+    @mock.patch("pungi.wrappers.repoclosure.get_repoclosure_cmd")
+    @mock.patch("pungi.phases.repoclosure.run")
+    def test_repoclosure_report_error(self, mock_run, mock_grc):
+        compose = DummyCompose(
+            self.topdir, {"repoclosure_strictness": [("^.*$", {"*": "fatal"})]}
+        )
+        mock_run.side_effect = mk_boom(cls=RuntimeError)
+
+        with self.assertRaises(RuntimeError):
+            repoclosure_phase.run_repoclosure(compose)
+
+    @unittest.skipUnless(HAS_DNF, "DNF is not available")
+    @mock.patch("pungi.wrappers.repoclosure.get_repoclosure_cmd")
+    @mock.patch("pungi.phases.repoclosure.run")
+    def test_repoclosure_overwrite_options_creates_correct_commands(
+        self, mock_run, mock_grc
+    ):
+        compose = DummyCompose(
+            self.topdir,
+            {
+                "repoclosure_backend": "dnf",
+                "repoclosure_strictness": [
+                    ("^.*$", {"*": "off"}),
+                    ("^Server$", {"*": "fatal"}),
+                ],
+            },
+        )
+        repoclosure_phase.run_repoclosure(compose)
+
+        six.assertCountEqual(
+            self,
+            mock_grc.call_args_list,
+            [
+                mock.call(
+                    backend="dnf",
+                    arch=["amd64", "x86_64", "noarch"],
+                    lookaside={},
+                    repos=self._get_repo(compose.compose_id, "Server", "amd64"),
+                ),
+                mock.call(
+                    backend="dnf",
+                    arch=["x86_64", "noarch"],
+                    lookaside={},
+                    repos=self._get_repo(compose.compose_id, "Server", "x86_64"),
+                ),
+            ],
+        )
+
+    @mock.patch("pungi.phases.repoclosure._delete_repoclosure_cache_dirs")
+    @mock.patch("pungi.wrappers.repoclosure.get_repoclosure_cmd")
+    @mock.patch("pungi.phases.repoclosure.run")
+    def test_repoclosure_uses_correct_behaviour(self, mock_run, mock_grc, mock_del):
+        compose = DummyCompose(
+            self.topdir,
+            {
+                "repoclosure_backend": "dnf",
+                "repoclosure_strictness": [
+                    ("^.*$", {"*": "off"}),
+                    ("^Server$", {"*": "fatal"}),
+                ],
+            },
+        )
+        mock_run.side_effect = mk_boom(cls=RuntimeError)
+
+        with self.assertRaises(RuntimeError):
+            repoclosure_phase.run_repoclosure(compose)

tests/test_test_phase.py

@@ -1,17 +1,10 @@
 # -*- coding: utf-8 -*-
 
-
-try:
-    import unittest2 as unittest
-except ImportError:
-    import unittest
-
 import mock
 import os
-import six
 
 import pungi.phases.test as test_phase
-from tests.helpers import DummyCompose, PungiTestCase, touch, mk_boom
+from tests.helpers import DummyCompose, PungiTestCase, touch
 
 try:
     import dnf  # noqa: F401
@@ -311,212 +304,3 @@ class TestCheckImageSanity(PungiTestCase):
         test_phase.check_image_sanity(compose)
 
         self.assertEqual(compose.log_warning.call_args_list, [])
-
-
-class TestRepoclosure(PungiTestCase):
-    def setUp(self):
-        super(TestRepoclosure, self).setUp()
-        self.maxDiff = None
-
-    def _get_repo(self, compose_id, variant, arch, path=None):
-        path = path or arch + "/os"
-        return {
-            "%s-repoclosure-%s.%s" % (compose_id, variant, arch): self.topdir
-            + "/compose/%s/%s" % (variant, path)
-        }
-
-    @mock.patch("pungi.wrappers.repoclosure.get_repoclosure_cmd")
-    @mock.patch("pungi.phases.test.run")
-    def test_repoclosure_skip_if_disabled(self, mock_run, mock_grc):
-        compose = DummyCompose(
-            self.topdir, {"repoclosure_strictness": [("^.*$", {"*": "off"})]}
-        )
-        test_phase.run_repoclosure(compose)
-
-        self.assertEqual(mock_grc.call_args_list, [])
-
-    @unittest.skipUnless(HAS_YUM, "YUM is not available")
-    @mock.patch("pungi.wrappers.repoclosure.get_repoclosure_cmd")
-    @mock.patch("pungi.phases.test.run")
-    def test_repoclosure_default_backend(self, mock_run, mock_grc):
-        with mock.patch("six.PY2", new=True):
-            compose = DummyCompose(self.topdir, {})
-
-        test_phase.run_repoclosure(compose)
-
-        six.assertCountEqual(
-            self,
-            mock_grc.call_args_list,
-            [
-                mock.call(
-                    backend="yum",
-                    arch=["amd64", "x86_64", "noarch"],
-                    lookaside={},
-                    repos=self._get_repo(compose.compose_id, "Everything", "amd64"),
-                ),
-                mock.call(
-                    backend="yum",
-                    arch=["amd64", "x86_64", "noarch"],
-                    lookaside={},
-                    repos=self._get_repo(compose.compose_id, "Client", "amd64"),
-                ),
-                mock.call(
-                    backend="yum",
-                    arch=["amd64", "x86_64", "noarch"],
-                    lookaside={},
-                    repos=self._get_repo(compose.compose_id, "Server", "amd64"),
-                ),
-                mock.call(
-                    backend="yum",
-                    arch=["x86_64", "noarch"],
-                    lookaside={},
-                    repos=self._get_repo(compose.compose_id, "Server", "x86_64"),
-                ),
-                mock.call(
-                    backend="yum",
-                    arch=["x86_64", "noarch"],
-                    lookaside={},
-                    repos=self._get_repo(compose.compose_id, "Everything", "x86_64"),
-                ),
-            ],
-        )
-
-    @unittest.skipUnless(HAS_DNF, "DNF is not available")
-    @mock.patch("pungi.wrappers.repoclosure.get_repoclosure_cmd")
-    @mock.patch("pungi.phases.test.run")
-    def test_repoclosure_dnf_backend(self, mock_run, mock_grc):
-        compose = DummyCompose(self.topdir, {"repoclosure_backend": "dnf"})
-        test_phase.run_repoclosure(compose)
-
-        six.assertCountEqual(
-            self,
-            mock_grc.call_args_list,
-            [
-                mock.call(
-                    backend="dnf",
-                    arch=["amd64", "x86_64", "noarch"],
-                    lookaside={},
-                    repos=self._get_repo(compose.compose_id, "Everything", "amd64"),
-                ),
-                mock.call(
-                    backend="dnf",
-                    arch=["amd64", "x86_64", "noarch"],
-                    lookaside={},
-                    repos=self._get_repo(compose.compose_id, "Client", "amd64"),
-                ),
-                mock.call(
-                    backend="dnf",
-                    arch=["amd64", "x86_64", "noarch"],
-                    lookaside={},
-                    repos=self._get_repo(compose.compose_id, "Server", "amd64"),
-                ),
-                mock.call(
-                    backend="dnf",
-                    arch=["x86_64", "noarch"],
-                    lookaside={},
-                    repos=self._get_repo(compose.compose_id, "Server", "x86_64"),
-                ),
-                mock.call(
-                    backend="dnf",
-                    arch=["x86_64", "noarch"],
-                    lookaside={},
-                    repos=self._get_repo(compose.compose_id, "Everything", "x86_64"),
-                ),
-            ],
-        )
-
-    @mock.patch("glob.glob")
-    @mock.patch("pungi.wrappers.repoclosure.extract_from_fus_logs")
-    @mock.patch("pungi.wrappers.repoclosure.get_repoclosure_cmd")
-    @mock.patch("pungi.phases.test.run")
-    def test_repoclosure_hybrid_variant(self, mock_run, mock_grc, effl, glob):
-        compose = DummyCompose(
-            self.topdir, {"repoclosure_backend": "dnf", "gather_method": "hybrid"}
-        )
-        f = mock.Mock()
-        glob.return_value = [f]
-
-        def _log(a, v):
-            return compose.paths.log.log_file(a, "repoclosure-%s" % compose.variants[v])
-
-        test_phase.run_repoclosure(compose)
-
-        self.assertEqual(mock_grc.call_args_list, [])
-        six.assertCountEqual(
-            self,
-            effl.call_args_list,
-            [
-                mock.call([f], _log("amd64", "Everything")),
-                mock.call([f], _log("amd64", "Client")),
-                mock.call([f], _log("amd64", "Server")),
-                mock.call([f], _log("x86_64", "Server")),
-                mock.call([f], _log("x86_64", "Everything")),
-            ],
-        )
-
-    @mock.patch("pungi.wrappers.repoclosure.get_repoclosure_cmd")
-    @mock.patch("pungi.phases.test.run")
-    def test_repoclosure_report_error(self, mock_run, mock_grc):
-        compose = DummyCompose(
-            self.topdir, {"repoclosure_strictness": [("^.*$", {"*": "fatal"})]}
-        )
-        mock_run.side_effect = mk_boom(cls=RuntimeError)
-
-        with self.assertRaises(RuntimeError):
-            test_phase.run_repoclosure(compose)
-
-    @unittest.skipUnless(HAS_DNF, "DNF is not available")
-    @mock.patch("pungi.wrappers.repoclosure.get_repoclosure_cmd")
-    @mock.patch("pungi.phases.test.run")
-    def test_repoclosure_overwrite_options_creates_correct_commands(
-        self, mock_run, mock_grc
-    ):
-        compose = DummyCompose(
-            self.topdir,
-            {
-                "repoclosure_backend": "dnf",
-                "repoclosure_strictness": [
-                    ("^.*$", {"*": "off"}),
-                    ("^Server$", {"*": "fatal"}),
-                ],
-            },
-        )
-        test_phase.run_repoclosure(compose)
-
-        six.assertCountEqual(
-            self,
-            mock_grc.call_args_list,
-            [
-                mock.call(
-                    backend="dnf",
-                    arch=["amd64", "x86_64", "noarch"],
-                    lookaside={},
-                    repos=self._get_repo(compose.compose_id, "Server", "amd64"),
-                ),
-                mock.call(
-                    backend="dnf",
-                    arch=["x86_64", "noarch"],
-                    lookaside={},
-                    repos=self._get_repo(compose.compose_id, "Server", "x86_64"),
-                ),
-            ],
-        )
-
-    @mock.patch("pungi.phases.test._delete_repoclosure_cache_dirs")
-    @mock.patch("pungi.wrappers.repoclosure.get_repoclosure_cmd")
-    @mock.patch("pungi.phases.test.run")
-    def test_repoclosure_uses_correct_behaviour(self, mock_run, mock_grc, mock_del):
-        compose = DummyCompose(
-            self.topdir,
-            {
-                "repoclosure_backend": "dnf",
-                "repoclosure_strictness": [
-                    ("^.*$", {"*": "off"}),
-                    ("^Server$", {"*": "fatal"}),
-                ],
-            },
-        )
-        mock_run.side_effect = mk_boom(cls=RuntimeError)
-
-        with self.assertRaises(RuntimeError):
-            test_phase.run_repoclosure(compose)