Split repoclosure into separate phase

Move repoclosure out from test phase into its own phase and
run parallel with image building phases(osbs, imagebuild, ...)
to speed things up.

JIRA: RHELCMP-8
Signed-off-by: Haibo Lin <hlin@redhat.com>
This commit is contained in:
Haibo Lin 2020-04-13 14:50:00 +08:00
parent e187b5ea79
commit 477b43d4e9
9 changed files with 473 additions and 380 deletions

BIN
doc/_static/phases.png vendored

Binary file not shown.

Before

Width:  |  Height:  |  Size: 24 KiB

After

Width:  |  Height:  |  Size: 24 KiB

124
doc/_static/phases.svg vendored
View File

@ -11,12 +11,12 @@
inkscape:export-xdpi="90" inkscape:export-xdpi="90"
inkscape:export-filename="/home/lsedlar/repos/pungi/doc/_static/phases.png" inkscape:export-filename="/home/lsedlar/repos/pungi/doc/_static/phases.png"
sodipodi:docname="phases.svg" sodipodi:docname="phases.svg"
inkscape:version="0.92.4 (unknown)" inkscape:version="0.92.2 5c3e80d, 2017-08-06"
version="1.1" version="1.1"
id="svg2" id="svg2"
viewBox="0 0 771.66458 221.50019" viewBox="0 0 669.66458 255.18195"
height="221.50018" height="255.18195"
width="771.66455"> width="669.66455">
<sodipodi:namedview <sodipodi:namedview
fit-margin-bottom="0" fit-margin-bottom="0"
fit-margin-right="0" fit-margin-right="0"
@ -27,16 +27,16 @@
inkscape:document-rotation="0" inkscape:document-rotation="0"
units="px" units="px"
inkscape:window-maximized="1" inkscape:window-maximized="1"
inkscape:window-y="0" inkscape:window-y="1"
inkscape:window-x="0" inkscape:window-x="0"
inkscape:window-height="1015" inkscape:window-height="1035"
inkscape:window-width="1920" inkscape:window-width="1920"
showgrid="false" showgrid="false"
inkscape:current-layer="g3668" inkscape:current-layer="layer1"
inkscape:document-units="px" inkscape:document-units="px"
inkscape:cy="137.85275" inkscape:cy="127.3243"
inkscape:cx="228.82868" inkscape:cx="420.82921"
inkscape:zoom="3.3064935" inkscape:zoom="1.5"
inkscape:pageshadow="2" inkscape:pageshadow="2"
inkscape:pageopacity="1" inkscape:pageopacity="1"
borderopacity="1.0" borderopacity="1.0"
@ -69,7 +69,7 @@
<dc:format>image/svg+xml</dc:format> <dc:format>image/svg+xml</dc:format>
<dc:type <dc:type
rdf:resource="http://purl.org/dc/dcmitype/StillImage" /> rdf:resource="http://purl.org/dc/dcmitype/StillImage" />
<dc:title /> <dc:title></dc:title>
</cc:Work> </cc:Work>
</rdf:RDF> </rdf:RDF>
</metadata> </metadata>
@ -77,7 +77,7 @@
inkscape:label="Vrstva 1" inkscape:label="Vrstva 1"
inkscape:groupmode="layer" inkscape:groupmode="layer"
id="layer1" id="layer1"
transform="matrix(1.066667,0,0,1.066667,-2.473231,-910.85239)"> transform="matrix(1.066667,0,0,1.066667,-78.473216,-910.85239)">
<g <g
id="g3411" id="g3411"
transform="translate(71.99326,-80.817124)"> transform="translate(71.99326,-80.817124)">
@ -95,7 +95,7 @@
x="51.554729" x="51.554729"
style="font-style:normal;font-weight:normal;line-height:0%;font-family:sans-serif;letter-spacing:0px;word-spacing:0px;fill:#000000;fill-opacity:1;stroke:none;stroke-width:1px;stroke-linecap:butt;stroke-linejoin:miter;stroke-opacity:1" style="font-style:normal;font-weight:normal;line-height:0%;font-family:sans-serif;letter-spacing:0px;word-spacing:0px;fill:#000000;fill-opacity:1;stroke:none;stroke-width:1px;stroke-linecap:butt;stroke-linejoin:miter;stroke-opacity:1"
xml:space="preserve"><tspan xml:space="preserve"><tspan
style="font-size:13.1479px;line-height:1.25" style="font-size:13.14789963px;line-height:1.25"
y="970.26605" y="970.26605"
x="51.554729" x="51.554729"
id="tspan3362" id="tspan3362"
@ -118,7 +118,7 @@
x="556.95709" x="556.95709"
style="font-style:normal;font-variant:normal;font-weight:normal;font-stretch:normal;line-height:0%;font-family:sans-serif;-inkscape-font-specification:'sans-serif, Normal';text-align:start;letter-spacing:0px;word-spacing:0px;writing-mode:lr-tb;text-anchor:start;fill:#000000;fill-opacity:1;stroke:none;stroke-width:1px;stroke-linecap:butt;stroke-linejoin:miter;stroke-opacity:1" style="font-style:normal;font-variant:normal;font-weight:normal;font-stretch:normal;line-height:0%;font-family:sans-serif;-inkscape-font-specification:'sans-serif, Normal';text-align:start;letter-spacing:0px;word-spacing:0px;writing-mode:lr-tb;text-anchor:start;fill:#000000;fill-opacity:1;stroke:none;stroke-width:1px;stroke-linecap:butt;stroke-linejoin:miter;stroke-opacity:1"
xml:space="preserve"><tspan xml:space="preserve"><tspan
style="font-size:13.1475px;line-height:1.25" style="font-size:13.14750004px;line-height:1.25"
id="tspan3391" id="tspan3391"
sodipodi:role="line" sodipodi:role="line"
x="556.95709" x="556.95709"
@ -141,7 +141,7 @@
x="557.61566" x="557.61566"
style="font-style:normal;font-weight:normal;line-height:0%;font-family:sans-serif;letter-spacing:0px;word-spacing:0px;fill:#ffffff;fill-opacity:1;stroke:none;stroke-width:1px;stroke-linecap:butt;stroke-linejoin:miter;stroke-opacity:1" style="font-style:normal;font-weight:normal;line-height:0%;font-family:sans-serif;letter-spacing:0px;word-spacing:0px;fill:#ffffff;fill-opacity:1;stroke:none;stroke-width:1px;stroke-linecap:butt;stroke-linejoin:miter;stroke-opacity:1"
xml:space="preserve"><tspan xml:space="preserve"><tspan
style="font-size:13.1479px;line-height:1.25" style="font-size:13.14789963px;line-height:1.25"
y="971.33813" y="971.33813"
x="557.61566" x="557.61566"
id="tspan3398" id="tspan3398"
@ -164,7 +164,7 @@
x="6.2600794" x="6.2600794"
style="font-style:normal;font-weight:normal;line-height:0%;font-family:sans-serif;letter-spacing:0px;word-spacing:0px;fill:#000000;fill-opacity:1;stroke:none;stroke-width:1px;stroke-linecap:butt;stroke-linejoin:miter;stroke-opacity:1" style="font-style:normal;font-weight:normal;line-height:0%;font-family:sans-serif;letter-spacing:0px;word-spacing:0px;fill:#000000;fill-opacity:1;stroke:none;stroke-width:1px;stroke-linecap:butt;stroke-linejoin:miter;stroke-opacity:1"
xml:space="preserve"><tspan xml:space="preserve"><tspan
style="font-size:13.1479px;line-height:1.25" style="font-size:13.14789963px;line-height:1.25"
y="891.1604" y="891.1604"
x="6.2600794" x="6.2600794"
id="tspan3358" id="tspan3358"
@ -198,7 +198,7 @@
id="tspan3366" id="tspan3366"
x="105.76799" x="105.76799"
y="891.06732" y="891.06732"
style="font-size:13.1479px;line-height:1.25">Buildinstall</tspan></text> style="font-size:13.14789963px;line-height:1.25">Buildinstall</tspan></text>
</g> </g>
<g <g
id="g3639"> id="g3639">
@ -220,7 +220,7 @@
id="tspan3370" id="tspan3370"
x="106.1384" x="106.1384"
y="923.25934" y="923.25934"
style="font-size:13.1479px;line-height:1.25">Gather</tspan></text> style="font-size:13.14789963px;line-height:1.25">Gather</tspan></text>
</g> </g>
<g <g
id="g3647" id="g3647"
@ -246,7 +246,7 @@
id="tspan3374" id="tspan3374"
x="165.23042" x="165.23042"
y="923.25934" y="923.25934"
style="font-size:13.14789963px;line-height:1.25">ExtraFiles</tspan></text> style="font-size:13.1479px;line-height:1.25">ExtraFiles</tspan></text>
</g> </g>
<g <g
id="g3658" id="g3658"
@ -269,7 +269,7 @@
id="tspan3378" id="tspan3378"
x="243.95874" x="243.95874"
y="921.86945" y="921.86945"
style="font-size:13.14789963px;line-height:1.25">Createrepo</tspan></text> style="font-size:13.1479px;line-height:1.25">Createrepo</tspan></text>
</g> </g>
<g <g
transform="translate(-150.564,114.11662)" transform="translate(-150.564,114.11662)"
@ -292,7 +292,7 @@
x="256.90588" x="256.90588"
sodipodi:role="line" sodipodi:role="line"
id="tspan3406" id="tspan3406"
style="font-size:13.1479px;line-height:1.25">OSTree</tspan></text> style="font-size:13.14789963px;line-height:1.25">OSTree</tspan></text>
</g> </g>
<g <g
id="g288" id="g288"
@ -323,11 +323,10 @@
</g> </g>
</g> </g>
<g <g
transform="translate(-29.683562,-0.34408888)" id="g1061">
id="g236">
<g <g
id="g3458" id="g3458"
transform="translate(28.723958,-80.473035)"> transform="translate(-0.959604,-80.817124)">
<rect <rect
y="420.13605" y="420.13605"
x="953.49097" x="953.49097"
@ -342,14 +341,14 @@
x="422.99252" x="422.99252"
y="971.54041" y="971.54041"
id="text3384"><tspan id="text3384"><tspan
style="font-size:13.1479px;line-height:1.25" style="font-size:13.14789963px;line-height:1.25"
sodipodi:role="line" sodipodi:role="line"
id="tspan3386" id="tspan3386"
x="422.99252" x="422.99252"
y="971.54041">Createiso</tspan></text> y="971.54041">Createiso</tspan></text>
</g> </g>
<g <g
transform="translate(28.467511,-84.181232)" transform="translate(-1.216051,-84.525321)"
id="g3453"> id="g3453">
<rect <rect
style="fill:#73d216;fill-rule:evenodd;stroke:none;stroke-width:1px;stroke-linecap:butt;stroke-linejoin:miter;stroke-opacity:1" style="fill:#73d216;fill-rule:evenodd;stroke:none;stroke-width:1px;stroke-linecap:butt;stroke-linejoin:miter;stroke-opacity:1"
@ -365,14 +364,14 @@
x="422.69772" x="422.69772"
y="1006.4276" y="1006.4276"
id="text3388"><tspan id="text3388"><tspan
style="font-size:13.1479px;line-height:1.25" style="font-size:13.14789963px;line-height:1.25"
sodipodi:role="line" sodipodi:role="line"
id="tspan3390" id="tspan3390"
x="422.69772" x="422.69772"
y="1006.4276">LiveImages</tspan></text> y="1006.4276">LiveImages</tspan></text>
</g> </g>
<g <g
transform="translate(28.467511,-88.141877)" transform="translate(-1.216051,-88.485966)"
id="g3448"> id="g3448">
<rect <rect
style="fill:#f57900;fill-rule:evenodd;stroke:none;stroke-width:1px;stroke-linecap:butt;stroke-linejoin:miter;stroke-opacity:1" style="fill:#f57900;fill-rule:evenodd;stroke:none;stroke-width:1px;stroke-linecap:butt;stroke-linejoin:miter;stroke-opacity:1"
@ -388,14 +387,14 @@
x="422.69772" x="422.69772"
y="1042.8416" y="1042.8416"
id="text3392"><tspan id="text3392"><tspan
style="font-size:13.1479px;line-height:1.25" style="font-size:13.14789963px;line-height:1.25"
sodipodi:role="line" sodipodi:role="line"
id="tspan3394" id="tspan3394"
x="422.69772" x="422.69772"
y="1042.8416">ImageBuild</tspan></text> y="1042.8416">ImageBuild</tspan></text>
</g> </g>
<g <g
transform="translate(27.760419,-92.458101)" transform="translate(-1.923143,-92.80219)"
id="g3443"> id="g3443">
<rect <rect
transform="matrix(0,1,1,0,0,0)" transform="matrix(0,1,1,0,0,0)"
@ -411,38 +410,37 @@
x="423.40482" x="423.40482"
style="font-style:normal;font-weight:normal;line-height:0%;font-family:sans-serif;text-align:start;letter-spacing:0px;word-spacing:0px;writing-mode:lr-tb;text-anchor:start;fill:#000000;fill-opacity:1;stroke:none;stroke-width:1px;stroke-linecap:butt;stroke-linejoin:miter;stroke-opacity:1" style="font-style:normal;font-weight:normal;line-height:0%;font-family:sans-serif;text-align:start;letter-spacing:0px;word-spacing:0px;writing-mode:lr-tb;text-anchor:start;fill:#000000;fill-opacity:1;stroke:none;stroke-width:1px;stroke-linecap:butt;stroke-linejoin:miter;stroke-opacity:1"
xml:space="preserve"><tspan xml:space="preserve"><tspan
style="font-size:13.1479px;line-height:1.25" style="font-size:13.14789963px;line-height:1.25"
y="1079.6111" y="1079.6111"
x="423.40482" x="423.40482"
sodipodi:role="line" sodipodi:role="line"
id="tspan3434">LiveMedia</tspan></text> id="tspan3434">LiveMedia</tspan></text>
</g> </g>
<g <g
id="g306" id="g204">
transform="translate(-41.476764,-46.1016)">
<rect <rect
transform="matrix(0,1,1,0,0,0)" transform="matrix(0,1,1,0,0,0)"
y="490.33765" y="419.17731"
x="1048.9327" x="1002.4871"
height="101.85102" height="101.85102"
width="26.295755" width="26.295755"
id="rect290" id="rect290"
style="fill:#c17d11;fill-rule:evenodd;stroke:none;stroke-width:1px;stroke-linecap:butt;stroke-linejoin:miter;stroke-opacity:1" /> style="fill:#c17d11;fill-rule:evenodd;stroke:none;stroke-width:1px;stroke-linecap:butt;stroke-linejoin:miter;stroke-opacity:1" />
<text <text
id="text294" id="text294"
y="1065.7078" y="1019.2621"
x="492.642" x="421.48166"
style="font-style:normal;font-variant:normal;font-weight:normal;font-stretch:normal;line-height:0%;font-family:sans-serif;-inkscape-font-specification:'sans-serif, Normal';text-align:start;letter-spacing:0px;word-spacing:0px;writing-mode:lr-tb;text-anchor:start;fill:#000000;fill-opacity:1;stroke:none;stroke-width:1px;stroke-linecap:butt;stroke-linejoin:miter;stroke-opacity:1" style="font-style:normal;font-variant:normal;font-weight:normal;font-stretch:normal;line-height:0%;font-family:sans-serif;-inkscape-font-specification:'sans-serif, Normal';text-align:start;letter-spacing:0px;word-spacing:0px;writing-mode:lr-tb;text-anchor:start;fill:#000000;fill-opacity:1;stroke:none;stroke-width:1px;stroke-linecap:butt;stroke-linejoin:miter;stroke-opacity:1"
xml:space="preserve"><tspan xml:space="preserve"><tspan
style="font-size:12px;line-height:0" style="font-size:12px;line-height:0"
id="tspan301" id="tspan301"
sodipodi:role="line" sodipodi:role="line"
x="492.642" x="421.48166"
y="1065.7078">OSBS</tspan></text> y="1019.2621">OSBS</tspan></text>
</g> </g>
<g <g
id="g3819" id="g3819"
transform="translate(0,-16.949078)"> transform="translate(-29.683562,-17.293167)">
<rect <rect
transform="matrix(0,1,1,0,0,0)" transform="matrix(0,1,1,0,0,0)"
y="448.86087" y="448.86087"
@ -461,7 +459,51 @@
y="1069.0087" y="1069.0087"
x="451.16522" x="451.16522"
sodipodi:role="line" sodipodi:role="line"
style="font-size:13.1479px;line-height:1.25">ExtraIsos</tspan></text> style="font-size:13.14789963px;line-height:1.25">ExtraIsos</tspan></text>
</g>
<g
id="g1031">
<rect
transform="matrix(0,1,1,0,0,0)"
style="fill:#5ed4ec;fill-opacity:1;fill-rule:evenodd;stroke:none;stroke-width:1px;stroke-linecap:butt;stroke-linejoin:miter;stroke-opacity:1"
id="rect206"
width="26.295755"
height="102.36562"
x="1066.8611"
y="418.66275" />
<text
id="text210"
y="1084.9105"
x="421.51923"
style="font-style:normal;font-weight:normal;line-height:0%;font-family:sans-serif;letter-spacing:0px;word-spacing:0px;fill:#000000;fill-opacity:1;stroke:none;stroke-width:1px;stroke-linecap:butt;stroke-linejoin:miter;stroke-opacity:1"
xml:space="preserve"><tspan
y="1084.9105"
x="421.51923"
id="tspan208"
sodipodi:role="line"
style="font-size:13.14789963px;line-height:1.25">Repoclosure</tspan></text>
</g>
<g
id="g1031">
<rect
transform="matrix(0,1,1,0,0,0)"
style="fill:#5ed4ec;fill-opacity:1;fill-rule:evenodd;stroke:none;stroke-width:1px;stroke-linecap:butt;stroke-linejoin:miter;stroke-opacity:1"
id="rect206"
width="26.295755"
height="102.36562"
x="1066.8611"
y="418.66275" />
<text
id="text210"
y="1084.9105"
x="421.51923"
style="font-style:normal;font-weight:normal;line-height:0%;font-family:sans-serif;letter-spacing:0px;word-spacing:0px;fill:#000000;fill-opacity:1;stroke:none;stroke-width:1px;stroke-linecap:butt;stroke-linejoin:miter;stroke-opacity:1"
xml:space="preserve"><tspan
y="1084.9105"
x="421.51923"
id="tspan208"
sodipodi:role="line"
style="font-size:13.1479px;line-height:1.25">Repoclosure</tspan></text>
</g> </g>
</g> </g>
</g> </g>

Before

Width:  |  Height:  |  Size: 19 KiB

After

Width:  |  Height:  |  Size: 21 KiB

View File

@ -132,6 +132,13 @@ Creates bootable media that carry an ostree repository as a payload. These
images are created by running ``lorax`` with special templates. Again it runs images are created by running ``lorax`` with special templates. Again it runs
in Koji runroot. in Koji runroot.
Repoclosure
-----------
Run ``repoclosure`` on each repository. By default errors are only reported
in the log, the compose will still be considered a success. The actual error
has to be looked up in the compose logs directory. Configuration allows customizing this.
ImageChecksum ImageChecksum
------------- -------------
@ -145,12 +152,7 @@ Test
This phase is supposed to run some sanity checks on the finished compose. This phase is supposed to run some sanity checks on the finished compose.
The first test is to run ``repoclosure`` on each repository. By default errors The only test is to check all images listed the metadata and verify that they
are only reported in the log, the compose will still be considered a success.
The actual error has to be looked up in the compose logs directory.
Configuration allows customizing this.
The other test is to check all images listed the metadata and verify that they
look sane. For ISO files headers are checked to verify the format is correct, look sane. For ISO files headers are checked to verify the format is correct,
and for bootable media a check is run to verify they have properties that allow and for bootable media a check is run to verify they have properties that allow
booting. booting.

View File

@ -27,6 +27,7 @@ from .createiso import CreateisoPhase # noqa
from .extra_isos import ExtraIsosPhase # noqa from .extra_isos import ExtraIsosPhase # noqa
from .live_images import LiveImagesPhase # noqa from .live_images import LiveImagesPhase # noqa
from .image_build import ImageBuildPhase # noqa from .image_build import ImageBuildPhase # noqa
from .repoclosure import RepoclosurePhase # noqa
from .test import TestPhase # noqa from .test import TestPhase # noqa
from .image_checksum import ImageChecksumPhase # noqa from .image_checksum import ImageChecksumPhase # noqa
from .livemedia_phase import LiveMediaPhase # noqa from .livemedia_phase import LiveMediaPhase # noqa

141
pungi/phases/repoclosure.py Normal file
View File

@ -0,0 +1,141 @@
# -*- coding: utf-8 -*-
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation; version 2 of the License.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Library General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program; if not, see <https://gnu.org/licenses/>.
import glob
import os
import shutil
from kobo.shortcuts import run
from pungi.wrappers import repoclosure
from pungi.arch import get_valid_arches
from pungi.phases.base import PhaseBase
from pungi.phases.gather import get_lookaside_repos, get_gather_methods
from pungi.util import is_arch_multilib, temp_dir, get_arch_variant_data
class RepoclosurePhase(PhaseBase):
name = "repoclosure"
def run(self):
run_repoclosure(self.compose)
def run_repoclosure(compose):
msg = "Running repoclosure"
compose.log_info("[BEGIN] %s" % msg)
# Variant repos
for arch in compose.get_arches():
is_multilib = is_arch_multilib(compose.conf, arch)
arches = get_valid_arches(arch, is_multilib)
for variant in compose.get_variants(arch=arch):
if variant.is_empty:
continue
conf = get_arch_variant_data(
compose.conf, "repoclosure_strictness", arch, variant
)
if conf and conf[-1] == "off":
continue
prefix = "%s-repoclosure" % compose.compose_id
lookaside = {}
if variant.parent:
repo_id = "%s-%s.%s" % (prefix, variant.parent.uid, arch)
repo_dir = compose.paths.compose.repository(
arch=arch, variant=variant.parent
)
lookaside[repo_id] = repo_dir
repos = {}
repo_id = "%s-%s.%s" % (prefix, variant.uid, arch)
repo_dir = compose.paths.compose.repository(arch=arch, variant=variant)
repos[repo_id] = repo_dir
for i, lookaside_url in enumerate(
get_lookaside_repos(compose, arch, variant)
):
lookaside[
"%s-lookaside-%s.%s-%s" % (compose.compose_id, variant.uid, arch, i)
] = lookaside_url
logfile = compose.paths.log.log_file(arch, "repoclosure-%s" % variant)
try:
_, methods = get_gather_methods(compose, variant)
if methods == "hybrid":
# Using hybrid solver, no repoclosure command is available.
pattern = compose.paths.log.log_file(
arch, "hybrid-depsolver-%s-iter-*" % variant
)
fus_logs = sorted(glob.glob(pattern))
repoclosure.extract_from_fus_logs(fus_logs, logfile)
else:
_run_repoclosure_cmd(compose, repos, lookaside, arches, logfile)
except RuntimeError as exc:
if conf and conf[-1] == "fatal":
raise
else:
compose.log_warning(
"Repoclosure failed for %s.%s\n%s" % (variant.uid, arch, exc)
)
finally:
if methods != "hybrid":
_delete_repoclosure_cache_dirs(compose)
compose.log_info("[DONE ] %s" % msg)
def _delete_repoclosure_cache_dirs(compose):
if "dnf" == compose.conf["repoclosure_backend"]:
from dnf.const import SYSTEM_CACHEDIR
from dnf.util import am_i_root
from dnf.yum.misc import getCacheDir
if am_i_root():
top_cache_dir = SYSTEM_CACHEDIR
else:
top_cache_dir = getCacheDir()
else:
from yum.misc import getCacheDir
top_cache_dir = getCacheDir()
for name in os.listdir(top_cache_dir):
if name.startswith(compose.compose_id):
cache_path = os.path.join(top_cache_dir, name)
if os.path.isdir(cache_path):
shutil.rmtree(cache_path)
else:
os.remove(cache_path)
def _run_repoclosure_cmd(compose, repos, lookaside, arches, logfile):
cmd = repoclosure.get_repoclosure_cmd(
backend=compose.conf["repoclosure_backend"],
repos=repos,
lookaside=lookaside,
arch=arches,
)
# Use temp working directory directory as workaround for
# https://bugzilla.redhat.com/show_bug.cgi?id=795137
with temp_dir(prefix="repoclosure_") as tmp_dir:
# Ideally we would want show_cmd=True here to include the
# command in the logfile, but due to a bug in Kobo that would
# cause any error to be printed directly to stderr.
# https://github.com/release-engineering/kobo/pull/26
run(cmd, logfile=logfile, workdir=tmp_dir, show_cmd=True)

View File

@ -14,134 +14,19 @@
# along with this program; if not, see <https://gnu.org/licenses/>. # along with this program; if not, see <https://gnu.org/licenses/>.
import glob
import os import os
import shutil
from kobo.shortcuts import run
from pungi.wrappers import repoclosure
from pungi.arch import get_valid_arches
from pungi.phases.base import PhaseBase from pungi.phases.base import PhaseBase
from pungi.phases.gather import get_lookaside_repos, get_gather_methods from pungi.util import failable, get_arch_variant_data
from pungi.util import is_arch_multilib, failable, temp_dir, get_arch_variant_data
class TestPhase(PhaseBase): class TestPhase(PhaseBase):
name = "test" name = "test"
def run(self): def run(self):
run_repoclosure(self.compose)
check_image_sanity(self.compose) check_image_sanity(self.compose)
def run_repoclosure(compose):
msg = "Running repoclosure"
compose.log_info("[BEGIN] %s" % msg)
# Variant repos
for arch in compose.get_arches():
is_multilib = is_arch_multilib(compose.conf, arch)
arches = get_valid_arches(arch, is_multilib)
for variant in compose.get_variants(arch=arch):
if variant.is_empty:
continue
conf = get_arch_variant_data(
compose.conf, "repoclosure_strictness", arch, variant
)
if conf and conf[-1] == "off":
continue
prefix = "%s-repoclosure" % compose.compose_id
lookaside = {}
if variant.parent:
repo_id = "%s-%s.%s" % (prefix, variant.parent.uid, arch)
repo_dir = compose.paths.compose.repository(
arch=arch, variant=variant.parent
)
lookaside[repo_id] = repo_dir
repos = {}
repo_id = "%s-%s.%s" % (prefix, variant.uid, arch)
repo_dir = compose.paths.compose.repository(arch=arch, variant=variant)
repos[repo_id] = repo_dir
for i, lookaside_url in enumerate(
get_lookaside_repos(compose, arch, variant)
):
lookaside[
"%s-lookaside-%s.%s-%s" % (compose.compose_id, variant.uid, arch, i)
] = lookaside_url
logfile = compose.paths.log.log_file(arch, "repoclosure-%s" % variant)
try:
_, methods = get_gather_methods(compose, variant)
if methods == "hybrid":
# Using hybrid solver, no repoclosure command is available.
pattern = compose.paths.log.log_file(
arch, "hybrid-depsolver-%s-iter-*" % variant
)
fus_logs = sorted(glob.glob(pattern))
repoclosure.extract_from_fus_logs(fus_logs, logfile)
else:
_run_repoclosure_cmd(compose, repos, lookaside, arches, logfile)
except RuntimeError as exc:
if conf and conf[-1] == "fatal":
raise
else:
compose.log_warning(
"Repoclosure failed for %s.%s\n%s" % (variant.uid, arch, exc)
)
finally:
if methods != "hybrid":
_delete_repoclosure_cache_dirs(compose)
compose.log_info("[DONE ] %s" % msg)
def _delete_repoclosure_cache_dirs(compose):
if "dnf" == compose.conf["repoclosure_backend"]:
from dnf.const import SYSTEM_CACHEDIR
from dnf.util import am_i_root
from dnf.yum.misc import getCacheDir
if am_i_root():
top_cache_dir = SYSTEM_CACHEDIR
else:
top_cache_dir = getCacheDir()
else:
from yum.misc import getCacheDir
top_cache_dir = getCacheDir()
for name in os.listdir(top_cache_dir):
if name.startswith(compose.compose_id):
cache_path = os.path.join(top_cache_dir, name)
if os.path.isdir(cache_path):
shutil.rmtree(cache_path)
else:
os.remove(cache_path)
def _run_repoclosure_cmd(compose, repos, lookaside, arches, logfile):
cmd = repoclosure.get_repoclosure_cmd(
backend=compose.conf["repoclosure_backend"],
repos=repos,
lookaside=lookaside,
arch=arches,
)
# Use temp working directory directory as workaround for
# https://bugzilla.redhat.com/show_bug.cgi?id=795137
with temp_dir(prefix="repoclosure_") as tmp_dir:
# Ideally we would want show_cmd=True here to include the
# command in the logfile, but due to a bug in Kobo that would
# cause any error to be printed directly to stderr.
# https://github.com/release-engineering/kobo/pull/26
run(cmd, logfile=logfile, workdir=tmp_dir, show_cmd=True)
def check_image_sanity(compose): def check_image_sanity(compose):
""" """
Go through all images in manifest and make basic sanity tests on them. If Go through all images in manifest and make basic sanity tests on them. If

View File

@ -349,6 +349,7 @@ def run_compose(
image_build_phase = pungi.phases.ImageBuildPhase(compose) image_build_phase = pungi.phases.ImageBuildPhase(compose)
osbs_phase = pungi.phases.OSBSPhase(compose) osbs_phase = pungi.phases.OSBSPhase(compose)
image_checksum_phase = pungi.phases.ImageChecksumPhase(compose) image_checksum_phase = pungi.phases.ImageChecksumPhase(compose)
repoclosure_phase = pungi.phases.RepoclosurePhase(compose)
test_phase = pungi.phases.TestPhase(compose) test_phase = pungi.phases.TestPhase(compose)
# check if all config options are set # check if all config options are set
@ -467,6 +468,7 @@ def run_compose(
image_build_phase, image_build_phase,
livemedia_phase, livemedia_phase,
osbs_phase, osbs_phase,
repoclosure_phase,
) )
compose_images_phase = pungi.phases.WeaverPhase(compose, compose_images_schema) compose_images_phase = pungi.phases.WeaverPhase(compose, compose_images_schema)
compose_images_phase.start() compose_images_phase.start()

View File

@ -0,0 +1,236 @@
# -*- coding: utf-8 -*-
try:
import unittest2 as unittest
except ImportError:
import unittest
import mock
import six
import pungi.phases.repoclosure as repoclosure_phase
from tests.helpers import DummyCompose, PungiTestCase, mk_boom
try:
import dnf # noqa: F401
HAS_DNF = True
except ImportError:
HAS_DNF = False
try:
import yum # noqa: F401
HAS_YUM = True
except ImportError:
HAS_YUM = False
class TestRepoclosure(PungiTestCase):
def setUp(self):
super(TestRepoclosure, self).setUp()
self.maxDiff = None
def _get_repo(self, compose_id, variant, arch, path=None):
path = path or arch + "/os"
return {
"%s-repoclosure-%s.%s" % (compose_id, variant, arch): self.topdir
+ "/compose/%s/%s" % (variant, path)
}
@mock.patch("pungi.wrappers.repoclosure.get_repoclosure_cmd")
@mock.patch("pungi.phases.repoclosure.run")
def test_repoclosure_skip_if_disabled(self, mock_run, mock_grc):
compose = DummyCompose(
self.topdir, {"repoclosure_strictness": [("^.*$", {"*": "off"})]}
)
repoclosure_phase.run_repoclosure(compose)
self.assertEqual(mock_grc.call_args_list, [])
@unittest.skipUnless(HAS_YUM, "YUM is not available")
@mock.patch("pungi.wrappers.repoclosure.get_repoclosure_cmd")
@mock.patch("pungi.phases.repoclosure.run")
def test_repoclosure_default_backend(self, mock_run, mock_grc):
with mock.patch("six.PY2", new=True):
compose = DummyCompose(self.topdir, {})
repoclosure_phase.run_repoclosure(compose)
six.assertCountEqual(
self,
mock_grc.call_args_list,
[
mock.call(
backend="yum",
arch=["amd64", "x86_64", "noarch"],
lookaside={},
repos=self._get_repo(compose.compose_id, "Everything", "amd64"),
),
mock.call(
backend="yum",
arch=["amd64", "x86_64", "noarch"],
lookaside={},
repos=self._get_repo(compose.compose_id, "Client", "amd64"),
),
mock.call(
backend="yum",
arch=["amd64", "x86_64", "noarch"],
lookaside={},
repos=self._get_repo(compose.compose_id, "Server", "amd64"),
),
mock.call(
backend="yum",
arch=["x86_64", "noarch"],
lookaside={},
repos=self._get_repo(compose.compose_id, "Server", "x86_64"),
),
mock.call(
backend="yum",
arch=["x86_64", "noarch"],
lookaside={},
repos=self._get_repo(compose.compose_id, "Everything", "x86_64"),
),
],
)
@unittest.skipUnless(HAS_DNF, "DNF is not available")
@mock.patch("pungi.wrappers.repoclosure.get_repoclosure_cmd")
@mock.patch("pungi.phases.repoclosure.run")
def test_repoclosure_dnf_backend(self, mock_run, mock_grc):
compose = DummyCompose(self.topdir, {"repoclosure_backend": "dnf"})
repoclosure_phase.run_repoclosure(compose)
six.assertCountEqual(
self,
mock_grc.call_args_list,
[
mock.call(
backend="dnf",
arch=["amd64", "x86_64", "noarch"],
lookaside={},
repos=self._get_repo(compose.compose_id, "Everything", "amd64"),
),
mock.call(
backend="dnf",
arch=["amd64", "x86_64", "noarch"],
lookaside={},
repos=self._get_repo(compose.compose_id, "Client", "amd64"),
),
mock.call(
backend="dnf",
arch=["amd64", "x86_64", "noarch"],
lookaside={},
repos=self._get_repo(compose.compose_id, "Server", "amd64"),
),
mock.call(
backend="dnf",
arch=["x86_64", "noarch"],
lookaside={},
repos=self._get_repo(compose.compose_id, "Server", "x86_64"),
),
mock.call(
backend="dnf",
arch=["x86_64", "noarch"],
lookaside={},
repos=self._get_repo(compose.compose_id, "Everything", "x86_64"),
),
],
)
@mock.patch("glob.glob")
@mock.patch("pungi.wrappers.repoclosure.extract_from_fus_logs")
@mock.patch("pungi.wrappers.repoclosure.get_repoclosure_cmd")
@mock.patch("pungi.phases.repoclosure.run")
def test_repoclosure_hybrid_variant(self, mock_run, mock_grc, effl, glob):
compose = DummyCompose(
self.topdir, {"repoclosure_backend": "dnf", "gather_method": "hybrid"}
)
f = mock.Mock()
glob.return_value = [f]
def _log(a, v):
return compose.paths.log.log_file(a, "repoclosure-%s" % compose.variants[v])
repoclosure_phase.run_repoclosure(compose)
self.assertEqual(mock_grc.call_args_list, [])
six.assertCountEqual(
self,
effl.call_args_list,
[
mock.call([f], _log("amd64", "Everything")),
mock.call([f], _log("amd64", "Client")),
mock.call([f], _log("amd64", "Server")),
mock.call([f], _log("x86_64", "Server")),
mock.call([f], _log("x86_64", "Everything")),
],
)
@mock.patch("pungi.wrappers.repoclosure.get_repoclosure_cmd")
@mock.patch("pungi.phases.repoclosure.run")
def test_repoclosure_report_error(self, mock_run, mock_grc):
compose = DummyCompose(
self.topdir, {"repoclosure_strictness": [("^.*$", {"*": "fatal"})]}
)
mock_run.side_effect = mk_boom(cls=RuntimeError)
with self.assertRaises(RuntimeError):
repoclosure_phase.run_repoclosure(compose)
@unittest.skipUnless(HAS_DNF, "DNF is not available")
@mock.patch("pungi.wrappers.repoclosure.get_repoclosure_cmd")
@mock.patch("pungi.phases.repoclosure.run")
def test_repoclosure_overwrite_options_creates_correct_commands(
self, mock_run, mock_grc
):
compose = DummyCompose(
self.topdir,
{
"repoclosure_backend": "dnf",
"repoclosure_strictness": [
("^.*$", {"*": "off"}),
("^Server$", {"*": "fatal"}),
],
},
)
repoclosure_phase.run_repoclosure(compose)
six.assertCountEqual(
self,
mock_grc.call_args_list,
[
mock.call(
backend="dnf",
arch=["amd64", "x86_64", "noarch"],
lookaside={},
repos=self._get_repo(compose.compose_id, "Server", "amd64"),
),
mock.call(
backend="dnf",
arch=["x86_64", "noarch"],
lookaside={},
repos=self._get_repo(compose.compose_id, "Server", "x86_64"),
),
],
)
@mock.patch("pungi.phases.repoclosure._delete_repoclosure_cache_dirs")
@mock.patch("pungi.wrappers.repoclosure.get_repoclosure_cmd")
@mock.patch("pungi.phases.repoclosure.run")
def test_repoclosure_uses_correct_behaviour(self, mock_run, mock_grc, mock_del):
compose = DummyCompose(
self.topdir,
{
"repoclosure_backend": "dnf",
"repoclosure_strictness": [
("^.*$", {"*": "off"}),
("^Server$", {"*": "fatal"}),
],
},
)
mock_run.side_effect = mk_boom(cls=RuntimeError)
with self.assertRaises(RuntimeError):
repoclosure_phase.run_repoclosure(compose)

View File

@ -1,17 +1,10 @@
# -*- coding: utf-8 -*- # -*- coding: utf-8 -*-
try:
import unittest2 as unittest
except ImportError:
import unittest
import mock import mock
import os import os
import six
import pungi.phases.test as test_phase import pungi.phases.test as test_phase
from tests.helpers import DummyCompose, PungiTestCase, touch, mk_boom from tests.helpers import DummyCompose, PungiTestCase, touch
try: try:
import dnf # noqa: F401 import dnf # noqa: F401
@ -311,212 +304,3 @@ class TestCheckImageSanity(PungiTestCase):
test_phase.check_image_sanity(compose) test_phase.check_image_sanity(compose)
self.assertEqual(compose.log_warning.call_args_list, []) self.assertEqual(compose.log_warning.call_args_list, [])
class TestRepoclosure(PungiTestCase):
def setUp(self):
super(TestRepoclosure, self).setUp()
self.maxDiff = None
def _get_repo(self, compose_id, variant, arch, path=None):
path = path or arch + "/os"
return {
"%s-repoclosure-%s.%s" % (compose_id, variant, arch): self.topdir
+ "/compose/%s/%s" % (variant, path)
}
@mock.patch("pungi.wrappers.repoclosure.get_repoclosure_cmd")
@mock.patch("pungi.phases.test.run")
def test_repoclosure_skip_if_disabled(self, mock_run, mock_grc):
compose = DummyCompose(
self.topdir, {"repoclosure_strictness": [("^.*$", {"*": "off"})]}
)
test_phase.run_repoclosure(compose)
self.assertEqual(mock_grc.call_args_list, [])
@unittest.skipUnless(HAS_YUM, "YUM is not available")
@mock.patch("pungi.wrappers.repoclosure.get_repoclosure_cmd")
@mock.patch("pungi.phases.test.run")
def test_repoclosure_default_backend(self, mock_run, mock_grc):
with mock.patch("six.PY2", new=True):
compose = DummyCompose(self.topdir, {})
test_phase.run_repoclosure(compose)
six.assertCountEqual(
self,
mock_grc.call_args_list,
[
mock.call(
backend="yum",
arch=["amd64", "x86_64", "noarch"],
lookaside={},
repos=self._get_repo(compose.compose_id, "Everything", "amd64"),
),
mock.call(
backend="yum",
arch=["amd64", "x86_64", "noarch"],
lookaside={},
repos=self._get_repo(compose.compose_id, "Client", "amd64"),
),
mock.call(
backend="yum",
arch=["amd64", "x86_64", "noarch"],
lookaside={},
repos=self._get_repo(compose.compose_id, "Server", "amd64"),
),
mock.call(
backend="yum",
arch=["x86_64", "noarch"],
lookaside={},
repos=self._get_repo(compose.compose_id, "Server", "x86_64"),
),
mock.call(
backend="yum",
arch=["x86_64", "noarch"],
lookaside={},
repos=self._get_repo(compose.compose_id, "Everything", "x86_64"),
),
],
)
@unittest.skipUnless(HAS_DNF, "DNF is not available")
@mock.patch("pungi.wrappers.repoclosure.get_repoclosure_cmd")
@mock.patch("pungi.phases.test.run")
def test_repoclosure_dnf_backend(self, mock_run, mock_grc):
compose = DummyCompose(self.topdir, {"repoclosure_backend": "dnf"})
test_phase.run_repoclosure(compose)
six.assertCountEqual(
self,
mock_grc.call_args_list,
[
mock.call(
backend="dnf",
arch=["amd64", "x86_64", "noarch"],
lookaside={},
repos=self._get_repo(compose.compose_id, "Everything", "amd64"),
),
mock.call(
backend="dnf",
arch=["amd64", "x86_64", "noarch"],
lookaside={},
repos=self._get_repo(compose.compose_id, "Client", "amd64"),
),
mock.call(
backend="dnf",
arch=["amd64", "x86_64", "noarch"],
lookaside={},
repos=self._get_repo(compose.compose_id, "Server", "amd64"),
),
mock.call(
backend="dnf",
arch=["x86_64", "noarch"],
lookaside={},
repos=self._get_repo(compose.compose_id, "Server", "x86_64"),
),
mock.call(
backend="dnf",
arch=["x86_64", "noarch"],
lookaside={},
repos=self._get_repo(compose.compose_id, "Everything", "x86_64"),
),
],
)
@mock.patch("glob.glob")
@mock.patch("pungi.wrappers.repoclosure.extract_from_fus_logs")
@mock.patch("pungi.wrappers.repoclosure.get_repoclosure_cmd")
@mock.patch("pungi.phases.test.run")
def test_repoclosure_hybrid_variant(self, mock_run, mock_grc, effl, glob):
compose = DummyCompose(
self.topdir, {"repoclosure_backend": "dnf", "gather_method": "hybrid"}
)
f = mock.Mock()
glob.return_value = [f]
def _log(a, v):
return compose.paths.log.log_file(a, "repoclosure-%s" % compose.variants[v])
test_phase.run_repoclosure(compose)
self.assertEqual(mock_grc.call_args_list, [])
six.assertCountEqual(
self,
effl.call_args_list,
[
mock.call([f], _log("amd64", "Everything")),
mock.call([f], _log("amd64", "Client")),
mock.call([f], _log("amd64", "Server")),
mock.call([f], _log("x86_64", "Server")),
mock.call([f], _log("x86_64", "Everything")),
],
)
@mock.patch("pungi.wrappers.repoclosure.get_repoclosure_cmd")
@mock.patch("pungi.phases.test.run")
def test_repoclosure_report_error(self, mock_run, mock_grc):
compose = DummyCompose(
self.topdir, {"repoclosure_strictness": [("^.*$", {"*": "fatal"})]}
)
mock_run.side_effect = mk_boom(cls=RuntimeError)
with self.assertRaises(RuntimeError):
test_phase.run_repoclosure(compose)
@unittest.skipUnless(HAS_DNF, "DNF is not available")
@mock.patch("pungi.wrappers.repoclosure.get_repoclosure_cmd")
@mock.patch("pungi.phases.test.run")
def test_repoclosure_overwrite_options_creates_correct_commands(
self, mock_run, mock_grc
):
compose = DummyCompose(
self.topdir,
{
"repoclosure_backend": "dnf",
"repoclosure_strictness": [
("^.*$", {"*": "off"}),
("^Server$", {"*": "fatal"}),
],
},
)
test_phase.run_repoclosure(compose)
six.assertCountEqual(
self,
mock_grc.call_args_list,
[
mock.call(
backend="dnf",
arch=["amd64", "x86_64", "noarch"],
lookaside={},
repos=self._get_repo(compose.compose_id, "Server", "amd64"),
),
mock.call(
backend="dnf",
arch=["x86_64", "noarch"],
lookaside={},
repos=self._get_repo(compose.compose_id, "Server", "x86_64"),
),
],
)
@mock.patch("pungi.phases.test._delete_repoclosure_cache_dirs")
@mock.patch("pungi.wrappers.repoclosure.get_repoclosure_cmd")
@mock.patch("pungi.phases.test.run")
def test_repoclosure_uses_correct_behaviour(self, mock_run, mock_grc, mock_del):
compose = DummyCompose(
self.topdir,
{
"repoclosure_backend": "dnf",
"repoclosure_strictness": [
("^.*$", {"*": "off"}),
("^Server$", {"*": "fatal"}),
],
},
)
mock_run.side_effect = mk_boom(cls=RuntimeError)
with self.assertRaises(RuntimeError):
test_phase.run_repoclosure(compose)