almalinux/pungi
14
1
Fork 0
Code Issues Pull Requests Releases Activity

Move ostree phase and pipelines for running phases

Browse Source 
Signed-off-by: Ondrej Nosek <onosek@redhat.com>

Related: https://pagure.io/pungi/issue/778
This commit is contained in:
Ondrej Nosek 2018-02-16 13:40:54 +01:00
parent 5c081cb545
commit 660c04368b
6 changed files with 389 additions and 238 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

46
bin/pungi-koji
View File

@ -358,41 +358,27 @@ def run_compose(compose, create_latest_link=True, latest_link_status=None):
# Store the password # Store the password
compose.conf["signing_key_password"] = signing_key_password compose.conf["signing_key_password"] = signing_key_password
# INIT phase
init_phase.start() init_phase.start()
init_phase.stop() init_phase.stop()
# PKGSET phase
pkgset_phase.start() pkgset_phase.start()
pkgset_phase.stop() pkgset_phase.stop()
# BUILDINSTALL phase - start, we can run gathering, extra files and # WEAVER phase - launches other phases which can safely run in parallel
# createrepo while buildinstall is in progress. essentials_schema = (
buildinstall_phase.start() buildinstall_phase,
(gather_phase, extrafiles_phase, createrepo_phase),
# If any of the following three phases fail, we must ensure that (ostree_phase, ostree_installer_phase),
# buildinstall is stopped. Otherwise the whole process will hang. )
try: essentials_phase = pungi.phases.WeaverPhase(compose, essentials_schema)
gather_phase.start() essentials_phase.start()
gather_phase.stop() essentials_phase.stop()
extrafiles_phase.start()
extrafiles_phase.stop()
createrepo_phase.start()
createrepo_phase.stop()
finally:
buildinstall_phase.stop()
if not buildinstall_phase.skip(): if not buildinstall_phase.skip():
buildinstall_phase.copy_files() buildinstall_phase.copy_files()
ostree_phase.start() productimg_phase.start()
ostree_phase.stop() productimg_phase.stop()
pungi.phases.run_all([productimg_phase,
ostree_installer_phase])
# write treeinfo before ISOs are created # write treeinfo before ISOs are created
for variant in compose.get_variants(): for variant in compose.get_variants():
@ -408,11 +394,16 @@ def run_compose(compose, create_latest_link=True, latest_link_status=None):
pungi.metadata.write_media_repo(compose, arch, variant, timestamp) pungi.metadata.write_media_repo(compose, arch, variant, timestamp)
# Start all phases for image artifacts # Start all phases for image artifacts
pungi.phases.run_all([createiso_phase, compose_images_schema = (
createiso_phase,
liveimages_phase, liveimages_phase,
image_build_phase, image_build_phase,
livemedia_phase, livemedia_phase,
osbs_phase]) osbs_phase,
)
compose_images_phase = pungi.phases.WeaverPhase(compose, compose_images_schema)
compose_images_phase.start()
compose_images_phase.stop()
image_checksum_phase.start() image_checksum_phase.start()
image_checksum_phase.stop() image_checksum_phase.stop()
@ -421,7 +412,6 @@ def run_compose(compose, create_latest_link=True, latest_link_status=None):
compose.im.dump(compose.paths.compose.metadata("images.json")) compose.im.dump(compose.paths.compose.metadata("images.json"))
osbs_phase.dump_metadata() osbs_phase.dump_metadata()
# TEST phase
test_phase.start() test_phase.start()
test_phase.stop() test_phase.stop()

BIN
doc/_static/phases.png vendored
View File

Binary file not shown.
Side by Side Swipe Overlay

Before

Width:  |  Height:  |  Size: 23 KiB

After

Width:  |  Height:  |  Size: 24 KiB

351
doc/_static/phases.svg vendored
View File

@ -14,7 +14,7 @@
viewBox="0 0 839.33334 220.33335" viewBox="0 0 839.33334 220.33335"
id="svg2" id="svg2"
version="1.1" version="1.1"
inkscape:version="0.92+devel unknown" inkscape:version="0.91 r13725"
sodipodi:docname="phases.svg" sodipodi:docname="phases.svg"
inkscape:export-filename="/home/lsedlar/repos/pungi/doc/_static/phases.png" inkscape:export-filename="/home/lsedlar/repos/pungi/doc/_static/phases.png"
inkscape:export-xdpi="90" inkscape:export-xdpi="90"
@ -26,16 +26,16 @@
borderopacity="1.0" borderopacity="1.0"
inkscape:pageopacity="1" inkscape:pageopacity="1"
inkscape:pageshadow="2" inkscape:pageshadow="2"
inkscape:zoom="0.8266234" inkscape:zoom="1.6532468"
inkscape:cx="226.63066" inkscape:cx="337.4932"
inkscape:cy="201.60264" inkscape:cy="70.825454"
inkscape:document-units="px" inkscape:document-units="px"
inkscape:current-layer="layer1" inkscape:current-layer="layer1"
showgrid="false" showgrid="false"
inkscape:window-width="1920" inkscape:window-width="1920"
inkscape:window-height="1016" inkscape:window-height="1020"
inkscape:window-x="0" inkscape:window-x="1920"
inkscape:window-y="27" inkscape:window-y="31"
inkscape:window-maximized="1" inkscape:window-maximized="1"
units="px" units="px"
inkscape:document-rotation="0" inkscape:document-rotation="0"
@ -67,7 +67,7 @@
<dc:format>image/svg+xml</dc:format> <dc:format>image/svg+xml</dc:format>
<dc:type <dc:type
rdf:resource="http://purl.org/dc/dcmitype/StillImage" /> rdf:resource="http://purl.org/dc/dcmitype/StillImage" />
<dc:title></dc:title> <dc:title />
</cc:Work> </cc:Work>
</rdf:RDF> </rdf:RDF>
</metadata> </metadata>
@ -100,87 +100,7 @@
style="font-size:13.14787769px;line-height:1.25">Pkgset</tspan></text> style="font-size:13.14787769px;line-height:1.25">Pkgset</tspan></text>
</g> </g>
<g <g
transform="translate(-4.582059,-80.47309)" transform="translate(3.704976,-80.47309)"
id="g3416">
<rect
style="fill:#fcaf3e;fill-rule:evenodd;stroke:none;stroke-width:1px;stroke-linecap:butt;stroke-linejoin:miter;stroke-opacity:1"
id="rect3342"
width="26.295755"
height="231.47725"
x="953.49097"
y="108.04571"
transform="matrix(0,1,1,0,0,0)" />
<text
xml:space="preserve"
style="font-style:normal;font-weight:normal;line-height:0%;font-family:sans-serif;letter-spacing:0px;word-spacing:0px;fill:#000000;fill-opacity:1;stroke:none;stroke-width:1px;stroke-linecap:butt;stroke-linejoin:miter;stroke-opacity:1"
x="110.35005"
y="971.54041"
id="text3364"><tspan
sodipodi:role="line"
id="tspan3366"
x="110.35005"
y="971.54041"
style="font-size:13.14787769px;line-height:1.25">Buildinstall</tspan></text>
<rect
style="fill:#729fcf;fill-rule:evenodd;stroke:none;stroke-width:1px;stroke-linecap:butt;stroke-linejoin:miter;stroke-opacity:1"
id="rect3344"
width="26.295755"
height="54.197887"
x="989.65247"
y="112.96759"
transform="matrix(0,1,1,0,0,0)" />
<text
xml:space="preserve"
style="font-style:normal;font-weight:normal;line-height:0%;font-family:sans-serif;letter-spacing:0px;word-spacing:0px;fill:#000000;fill-opacity:1;stroke:none;stroke-width:1px;stroke-linecap:butt;stroke-linejoin:miter;stroke-opacity:1"
x="115.82405"
y="1007.7019"
id="text3368"><tspan
sodipodi:role="line"
id="tspan3370"
x="115.82405"
y="1007.7019"
style="font-size:13.14787769px;line-height:1.25">Gather</tspan></text>
<rect
style="fill:#ad7fa8;fill-rule:evenodd;stroke:none;stroke-width:1px;stroke-linecap:butt;stroke-linejoin:miter;stroke-opacity:1"
id="rect3346"
width="26.295755"
height="72.729973"
x="989.65247"
y="172.61172"
transform="matrix(0,1,1,0,0,0)" />
<text
xml:space="preserve"
style="font-style:normal;font-weight:normal;line-height:0%;font-family:sans-serif;letter-spacing:0px;word-spacing:0px;fill:#000000;fill-opacity:1;stroke:none;stroke-width:1px;stroke-linecap:butt;stroke-linejoin:miter;stroke-opacity:1"
x="174.91608"
y="1007.7019"
id="text3372"><tspan
sodipodi:role="line"
id="tspan3374"
x="174.91608"
y="1007.7019"
style="font-size:13.14787769px;line-height:1.25">ExtraFiles</tspan></text>
<rect
style="fill:#e9b96e;fill-rule:evenodd;stroke:none;stroke-width:1px;stroke-linecap:butt;stroke-linejoin:miter;stroke-opacity:1"
id="rect3348"
width="26.295755"
height="78.636055"
x="989.65247"
y="250.78795"
transform="matrix(0,1,1,0,0,0)" />
<text
xml:space="preserve"
style="font-style:normal;font-weight:normal;line-height:0%;font-family:sans-serif;letter-spacing:0px;word-spacing:0px;fill:#000000;fill-opacity:1;stroke:none;stroke-width:1px;stroke-linecap:butt;stroke-linejoin:miter;stroke-opacity:1"
x="253.64439"
y="1006.312"
id="text3376"><tspan
sodipodi:role="line"
id="tspan3378"
x="253.64439"
y="1006.312"
style="font-size:13.14787769px;line-height:1.25">Createrepo</tspan></text>
</g>
<g
transform="translate(63.81395,-80.47309)"
id="g3446"> id="g3446">
<rect <rect
y="554.10059" y="554.10059"
@ -195,7 +115,8 @@
style="font-style:normal;font-variant:normal;font-weight:normal;font-stretch:normal;line-height:0%;font-family:sans-serif;-inkscape-font-specification:'sans-serif, Normal';text-align:start;letter-spacing:0px;word-spacing:0px;writing-mode:lr-tb;text-anchor:start;fill:#000000;fill-opacity:1;stroke:none;stroke-width:1px;stroke-linecap:butt;stroke-linejoin:miter;stroke-opacity:1" style="font-style:normal;font-variant:normal;font-weight:normal;font-stretch:normal;line-height:0%;font-family:sans-serif;-inkscape-font-specification:'sans-serif, Normal';text-align:start;letter-spacing:0px;word-spacing:0px;writing-mode:lr-tb;text-anchor:start;fill:#000000;fill-opacity:1;stroke:none;stroke-width:1px;stroke-linecap:butt;stroke-linejoin:miter;stroke-opacity:1"
x="556.95709" x="556.95709"
y="971.54041" y="971.54041"
id="text3384-0"><tspan id="text3384-0"
sodipodi:linespacing="0%"><tspan
y="971.54041" y="971.54041"
x="556.95709" x="556.95709"
sodipodi:role="line" sodipodi:role="line"
@ -203,7 +124,7 @@
style="font-size:13.14749908px;line-height:1.25">ImageChecksum</tspan></text> style="font-size:13.14749908px;line-height:1.25">ImageChecksum</tspan></text>
</g> </g>
<g <g
transform="translate(185.8629,-80.47309)" transform="translate(125.75393,-80.47309)"
id="g3398"> id="g3398">
<rect <rect
y="553.98242" y="553.98242"
@ -218,7 +139,8 @@
style="font-style:normal;font-weight:normal;line-height:0%;font-family:sans-serif;letter-spacing:0px;word-spacing:0px;fill:#ffffff;fill-opacity:1;stroke:none;stroke-width:1px;stroke-linecap:butt;stroke-linejoin:miter;stroke-opacity:1" style="font-style:normal;font-weight:normal;line-height:0%;font-family:sans-serif;letter-spacing:0px;word-spacing:0px;fill:#ffffff;fill-opacity:1;stroke:none;stroke-width:1px;stroke-linecap:butt;stroke-linejoin:miter;stroke-opacity:1"
x="557.61566" x="557.61566"
y="971.33813" y="971.33813"
id="text3396"><tspan id="text3396"
sodipodi:linespacing="0%"><tspan
sodipodi:role="line" sodipodi:role="line"
id="tspan3398" id="tspan3398"
x="557.61566" x="557.61566"
@ -226,107 +148,209 @@
style="font-size:13.14787769px;line-height:1.25">Test</tspan></text> style="font-size:13.14787769px;line-height:1.25">Test</tspan></text>
</g> </g>
<g <g
transform="translate(2.318656,-80.47309)" id="g3720">
id="g3406">
<rect <rect
style="fill:#fce94f;fill-rule:evenodd;stroke:none;stroke-width:1px;stroke-linecap:butt;stroke-linejoin:miter;stroke-opacity:1" style="fill:#fce94f;fill-rule:evenodd;stroke:none;stroke-width:1px;stroke-linecap:butt;stroke-linejoin:miter;stroke-opacity:1"
id="rect3336" id="rect3336"
width="26.295755" width="26.295755"
height="39.669899" height="39.669899"
x="953.49097" x="873.01788"
y="-2.7716319e-06" y="2.3186533"
transform="matrix(0,1,1,0,0,0)" /> transform="matrix(0,1,1,0,0,0)" />
<text <text
xml:space="preserve" xml:space="preserve"
style="font-style:normal;font-weight:normal;line-height:0%;font-family:sans-serif;letter-spacing:0px;word-spacing:0px;fill:#000000;fill-opacity:1;stroke:none;stroke-width:1px;stroke-linecap:butt;stroke-linejoin:miter;stroke-opacity:1" style="font-style:normal;font-weight:normal;line-height:0%;font-family:sans-serif;letter-spacing:0px;word-spacing:0px;fill:#000000;fill-opacity:1;stroke:none;stroke-width:1px;stroke-linecap:butt;stroke-linejoin:miter;stroke-opacity:1"
x="1.1060941" x="6.2600794"
y="971.63348" y="891.1604"
id="text3356"><tspan id="text3356"
sodipodi:linespacing="0%"><tspan
sodipodi:role="line" sodipodi:role="line"
id="tspan3358" id="tspan3358"
x="1.1060941" x="6.2600794"
y="971.63348" y="891.1604"
style="font-size:13.14787769px;line-height:1.25">Init</tspan></text> style="font-size:13.14787769px;line-height:1.25">Init</tspan></text>
</g> </g>
<path <path
inkscape:connector-curvature="0" inkscape:connector-curvature="0"
id="path3642" id="path3642"
d="M 100.90864,859.8891 H 712.42335" d="m 100.90864,859.8891 553.31842,0"
style="fill:none;fill-rule:evenodd;stroke:#000000;stroke-width:1.23489845px;stroke-linecap:butt;stroke-linejoin:miter;stroke-opacity:1;marker-end:url(#Arrow1Lend)" /> style="fill:none;fill-rule:evenodd;stroke:#000000;stroke-width:1.17466855px;stroke-linecap:butt;stroke-linejoin:miter;stroke-opacity:1;marker-end:url(#Arrow1Lend)" />
<g <g
transform="translate(86.469501,49.471116)" id="g241"
id="g3408"> transform="translate(-60.108974,42.1407)">
<rect <rect
style="fill:#729fcf;fill-rule:evenodd;stroke:none;stroke-width:1px;stroke-linecap:butt;stroke-linejoin:miter;stroke-opacity:1" style="fill:#a40000;fill-rule:evenodd;stroke:none;stroke-width:1.10477591px;stroke-linecap:butt;stroke-linejoin:miter;stroke-opacity:1"
id="rect3350-3" id="rect3350"
width="26.295755" width="26.295755"
height="53.653927" height="101.73411"
x="823.54675" x="830.8772"
y="254.60153" y="400.8551"
transform="matrix(0,1,1,0,0,0)" /> transform="matrix(0,1,1,0,0,0)" />
<text <text
sodipodi:linespacing="0%"
xml:space="preserve" xml:space="preserve"
style="font-style:normal;font-weight:normal;line-height:0%;font-family:sans-serif;text-align:start;letter-spacing:0px;word-spacing:0px;writing-mode:lr-tb;text-anchor:start;fill:#ffffff;fill-opacity:1;stroke:none;stroke-width:1px;stroke-linecap:butt;stroke-linejoin:miter;stroke-opacity:1"
x="256.90588"
y="840.3219"
id="text3380-2"><tspan
y="840.3219"
x="256.90588"
sodipodi:role="line"
id="tspan3406"
style="font-size:13.14787769px;line-height:1.25">OSTree</tspan></text>
</g>
<g
id="g251"
transform="translate(0,42.1407)">
<g
id="g241">
<rect
transform="matrix(0,1,1,0,0,0)"
y="400.8551"
x="830.8772"
height="101.73411"
width="26.295755"
id="rect3350"
style="fill:#a40000;fill-rule:evenodd;stroke:none;stroke-width:1.10477591px;stroke-linecap:butt;stroke-linejoin:miter;stroke-opacity:1" />
<text
id="text3380"
y="847.65234"
x="403.15945"
style="font-style:normal;font-weight:normal;line-height:0%;font-family:sans-serif;letter-spacing:0px;word-spacing:0px;fill:#ffffff;fill-opacity:1;stroke:none;stroke-width:1px;stroke-linecap:butt;stroke-linejoin:miter;stroke-opacity:1" style="font-style:normal;font-weight:normal;line-height:0%;font-family:sans-serif;letter-spacing:0px;word-spacing:0px;fill:#ffffff;fill-opacity:1;stroke:none;stroke-width:1px;stroke-linecap:butt;stroke-linejoin:miter;stroke-opacity:1"
xml:space="preserve"><tspan
style="font-size:13.14787769px;line-height:1.25"
y="847.65234"
x="403.15945" x="403.15945"
y="847.65234"
id="text3380"><tspan
sodipodi:role="line"
id="tspan3382" id="tspan3382"
sodipodi:role="line">Productimg</tspan></text> x="403.15945"
y="847.65234"
style="font-size:13.14787769px;line-height:1.25">Productimg</tspan></text>
</g> </g>
<g <g
transform="translate(-89.482556,-154.87768)" id="g3668">
id="g288"> <g
id="g3663">
<rect <rect
transform="matrix(0,1,1,0,0,0)" transform="matrix(0,1,1,0,0,0)"
y="490.33765" y="103.46365"
x="1022.637" x="873.01788"
height="101.85102" height="231.47725"
width="26.295755" width="26.295755"
id="rect3428" id="rect3342"
style="fill:#fcaf3e;fill-rule:evenodd;stroke:none;stroke-width:1px;stroke-linecap:butt;stroke-linejoin:miter;stroke-opacity:1" /> style="fill:#fcaf3e;fill-rule:evenodd;stroke:none;stroke-width:1px;stroke-linecap:butt;stroke-linejoin:miter;stroke-opacity:1" />
<text <text
id="text3430" sodipodi:linespacing="0%"
y="1039.4121" id="text3364"
x="492.642" y="891.06732"
style="font-style:normal;font-variant:normal;font-weight:normal;font-stretch:normal;font-size:11.99999714px;line-height:0%;font-family:sans-serif;-inkscape-font-specification:'sans-serif, Normal';font-variant-ligatures:normal;font-variant-caps:normal;font-variant-numeric:normal;font-feature-settings:normal;text-align:start;letter-spacing:0px;word-spacing:0px;writing-mode:lr-tb;text-anchor:start;fill:#000000;fill-opacity:1;stroke:none;stroke-width:1px;stroke-linecap:butt;stroke-linejoin:miter;stroke-opacity:1" x="105.76799"
style="font-style:normal;font-weight:normal;line-height:0%;font-family:sans-serif;letter-spacing:0px;word-spacing:0px;fill:#000000;fill-opacity:1;stroke:none;stroke-width:1px;stroke-linecap:butt;stroke-linejoin:miter;stroke-opacity:1"
xml:space="preserve"><tspan xml:space="preserve"><tspan
y="1039.4121" style="font-size:13.14787769px;line-height:1.25"
x="492.642" y="891.06732"
x="105.76799"
id="tspan3366"
sodipodi:role="line">Buildinstall</tspan></text>
</g>
<g
id="g3639">
<rect
transform="matrix(0,1,1,0,0,0)"
y="103.28194"
x="905.2099"
height="54.197887"
width="26.295755"
id="rect3344"
style="fill:#729fcf;fill-rule:evenodd;stroke:none;stroke-width:1px;stroke-linecap:butt;stroke-linejoin:miter;stroke-opacity:1" />
<text
sodipodi:linespacing="0%"
id="text3368"
y="923.25934"
x="106.1384"
style="font-style:normal;font-weight:normal;line-height:0%;font-family:sans-serif;letter-spacing:0px;word-spacing:0px;fill:#000000;fill-opacity:1;stroke:none;stroke-width:1px;stroke-linecap:butt;stroke-linejoin:miter;stroke-opacity:1"
xml:space="preserve"><tspan
style="font-size:13.14787769px;line-height:1.25"
y="923.25934"
x="106.1384"
id="tspan3370"
sodipodi:role="line">Gather</tspan></text>
</g>
<g
id="g3647">
<g
id="g3644">
<rect
style="fill:#ad7fa8;fill-rule:evenodd;stroke:none;stroke-width:1px;stroke-linecap:butt;stroke-linejoin:miter;stroke-opacity:1"
id="rect3346"
width="26.295755"
height="72.729973"
x="905.2099"
y="162.92607"
transform="matrix(0,1,1,0,0,0)" />
</g>
<text
sodipodi:linespacing="0%"
id="text3372"
y="923.25934"
x="165.23042"
style="font-style:normal;font-weight:normal;line-height:0%;font-family:sans-serif;letter-spacing:0px;word-spacing:0px;fill:#000000;fill-opacity:1;stroke:none;stroke-width:1px;stroke-linecap:butt;stroke-linejoin:miter;stroke-opacity:1"
xml:space="preserve"><tspan
style="font-size:13.14787769px;line-height:1.25"
y="923.25934"
x="165.23042"
id="tspan3374"
sodipodi:role="line">ExtraFiles</tspan></text>
</g>
<g
id="g3658">
<rect
transform="matrix(0,1,1,0,0,0)"
y="241.10229"
x="905.2099"
height="78.636055"
width="26.295755"
id="rect3348"
style="fill:#e9b96e;fill-rule:evenodd;stroke:none;stroke-width:1px;stroke-linecap:butt;stroke-linejoin:miter;stroke-opacity:1" />
<text
sodipodi:linespacing="0%"
id="text3376"
y="921.86945"
x="243.95874"
style="font-style:normal;font-weight:normal;line-height:0%;font-family:sans-serif;letter-spacing:0px;word-spacing:0px;fill:#000000;fill-opacity:1;stroke:none;stroke-width:1px;stroke-linecap:butt;stroke-linejoin:miter;stroke-opacity:1"
xml:space="preserve"><tspan
style="font-size:13.14787769px;line-height:1.25"
y="921.86945"
x="243.95874"
id="tspan3378"
sodipodi:role="line">Createrepo</tspan></text>
</g>
<g
id="g3408"
transform="translate(-150.564,114.11662)">
<rect
transform="matrix(0,1,1,0,0,0)"
y="254.60153"
x="823.54675"
height="53.653927"
width="26.295755"
id="rect3350-3"
style="fill:#729fcf;fill-rule:evenodd;stroke:none;stroke-width:1px;stroke-linecap:butt;stroke-linejoin:miter;stroke-opacity:1" />
<text
sodipodi:linespacing="0%"
id="text3380-2"
y="840.3219"
x="256.90588"
style="font-style:normal;font-weight:normal;line-height:0%;font-family:sans-serif;text-align:start;letter-spacing:0px;word-spacing:0px;writing-mode:lr-tb;text-anchor:start;fill:#ffffff;fill-opacity:1;stroke:none;stroke-width:1px;stroke-linecap:butt;stroke-linejoin:miter;stroke-opacity:1"
xml:space="preserve"><tspan
style="font-size:13.14787769px;line-height:1.25"
id="tspan3406"
sodipodi:role="line" sodipodi:role="line"
id="tspan283">OSTreeInstaller</tspan></text> x="256.90588"
y="840.3219">OSTree</tspan></text>
</g>
<g
transform="translate(-328.39105,-85.517823)"
id="g288">
<g
transform="translate(0.56706579,0)"
id="g3653">
<rect
style="fill:#fcaf3e;fill-rule:evenodd;stroke:none;stroke-width:1px;stroke-linecap:butt;stroke-linejoin:miter;stroke-opacity:1"
id="rect3428"
width="26.295755"
height="101.85102"
x="1022.637"
y="490.33765"
transform="matrix(0,1,1,0,0,0)" />
<text
sodipodi:linespacing="0%"
xml:space="preserve"
style="font-style:normal;font-variant:normal;font-weight:normal;font-stretch:normal;font-size:11.99999714px;line-height:0%;font-family:sans-serif;-inkscape-font-specification:'sans-serif, Normal';text-align:start;letter-spacing:0px;word-spacing:0px;writing-mode:lr-tb;text-anchor:start;fill:#000000;fill-opacity:1;stroke:none;stroke-width:1px;stroke-linecap:butt;stroke-linejoin:miter;stroke-opacity:1"
x="492.642"
y="1039.4121"
id="text3430"><tspan
id="tspan283"
sodipodi:role="line"
x="492.642"
y="1039.4121">OSTreeInstaller</tspan></text>
</g>
</g> </g>
</g> </g>
<g <g
id="g236" id="g236"
transform="translate(0,42.1407)"> transform="translate(-60.108974,42.1407)">
<g <g
transform="translate(88.832932,-122.61379)" transform="translate(88.832932,-122.61379)"
id="g3458"> id="g3458">
@ -343,7 +367,8 @@
y="971.54041" y="971.54041"
x="422.99252" x="422.99252"
style="font-style:normal;font-weight:normal;line-height:0%;font-family:sans-serif;letter-spacing:0px;word-spacing:0px;fill:#000000;fill-opacity:1;stroke:none;stroke-width:1px;stroke-linecap:butt;stroke-linejoin:miter;stroke-opacity:1" style="font-style:normal;font-weight:normal;line-height:0%;font-family:sans-serif;letter-spacing:0px;word-spacing:0px;fill:#000000;fill-opacity:1;stroke:none;stroke-width:1px;stroke-linecap:butt;stroke-linejoin:miter;stroke-opacity:1"
xml:space="preserve"><tspan xml:space="preserve"
sodipodi:linespacing="0%"><tspan
y="971.54041" y="971.54041"
x="422.99252" x="422.99252"
id="tspan3386" id="tspan3386"
@ -366,7 +391,8 @@
y="1006.4276" y="1006.4276"
x="422.69772" x="422.69772"
style="font-style:normal;font-weight:normal;line-height:0%;font-family:sans-serif;letter-spacing:0px;word-spacing:0px;fill:#000000;fill-opacity:1;stroke:none;stroke-width:1px;stroke-linecap:butt;stroke-linejoin:miter;stroke-opacity:1" style="font-style:normal;font-weight:normal;line-height:0%;font-family:sans-serif;letter-spacing:0px;word-spacing:0px;fill:#000000;fill-opacity:1;stroke:none;stroke-width:1px;stroke-linecap:butt;stroke-linejoin:miter;stroke-opacity:1"
xml:space="preserve"><tspan xml:space="preserve"
sodipodi:linespacing="0%"><tspan
y="1006.4276" y="1006.4276"
x="422.69772" x="422.69772"
id="tspan3390" id="tspan3390"
@ -389,7 +415,8 @@
y="1042.8416" y="1042.8416"
x="422.69772" x="422.69772"
style="font-style:normal;font-weight:normal;line-height:0%;font-family:sans-serif;letter-spacing:0px;word-spacing:0px;fill:#000000;fill-opacity:1;stroke:none;stroke-width:1px;stroke-linecap:butt;stroke-linejoin:miter;stroke-opacity:1" style="font-style:normal;font-weight:normal;line-height:0%;font-family:sans-serif;letter-spacing:0px;word-spacing:0px;fill:#000000;fill-opacity:1;stroke:none;stroke-width:1px;stroke-linecap:butt;stroke-linejoin:miter;stroke-opacity:1"
xml:space="preserve"><tspan xml:space="preserve"
sodipodi:linespacing="0%"><tspan
y="1042.8416" y="1042.8416"
x="422.69772" x="422.69772"
id="tspan3394" id="tspan3394"
@ -412,7 +439,8 @@
style="font-style:normal;font-weight:normal;line-height:0%;font-family:sans-serif;text-align:start;letter-spacing:0px;word-spacing:0px;writing-mode:lr-tb;text-anchor:start;fill:#000000;fill-opacity:1;stroke:none;stroke-width:1px;stroke-linecap:butt;stroke-linejoin:miter;stroke-opacity:1" style="font-style:normal;font-weight:normal;line-height:0%;font-family:sans-serif;text-align:start;letter-spacing:0px;word-spacing:0px;writing-mode:lr-tb;text-anchor:start;fill:#000000;fill-opacity:1;stroke:none;stroke-width:1px;stroke-linecap:butt;stroke-linejoin:miter;stroke-opacity:1"
x="423.40482" x="423.40482"
y="1079.6111" y="1079.6111"
id="text3424"><tspan id="text3424"
sodipodi:linespacing="0%"><tspan
id="tspan3434" id="tspan3434"
sodipodi:role="line" sodipodi:role="line"
x="423.40482" x="423.40482"
@ -432,10 +460,11 @@
transform="matrix(0,1,1,0,0,0)" /> transform="matrix(0,1,1,0,0,0)" />
<text <text
xml:space="preserve" xml:space="preserve"
style="font-style:normal;font-variant:normal;font-weight:normal;font-stretch:normal;font-size:11.99999714px;line-height:0%;font-family:sans-serif;-inkscape-font-specification:'sans-serif, Normal';font-variant-ligatures:normal;font-variant-caps:normal;font-variant-numeric:normal;font-feature-settings:normal;text-align:start;letter-spacing:0px;word-spacing:0px;writing-mode:lr-tb;text-anchor:start;fill:#000000;fill-opacity:1;stroke:none;stroke-width:1px;stroke-linecap:butt;stroke-linejoin:miter;stroke-opacity:1" style="font-style:normal;font-variant:normal;font-weight:normal;font-stretch:normal;font-size:11.99999714px;line-height:0%;font-family:sans-serif;-inkscape-font-specification:'sans-serif, Normal';text-align:start;letter-spacing:0px;word-spacing:0px;writing-mode:lr-tb;text-anchor:start;fill:#000000;fill-opacity:1;stroke:none;stroke-width:1px;stroke-linecap:butt;stroke-linejoin:miter;stroke-opacity:1"
x="492.642" x="492.642"
y="1065.7078" y="1065.7078"
id="text294"><tspan id="text294"
sodipodi:linespacing="0%"><tspan
y="1065.7078" y="1065.7078"
x="492.642" x="492.642"
sodipodi:role="line" sodipodi:role="line"

Side by Side Swipe Overlay

Before

Width:  |  Height:  |  Size: 18 KiB

After

Width:  |  Height:  |  Size: 19 KiB

19
pungi/phases/__init__.py
View File

@ -16,6 +16,7 @@
# phases in runtime order # phases in runtime order
from .init import InitPhase # noqa from .init import InitPhase # noqa
from .weaver import WeaverPhase # noqa
from .pkgset import PkgsetPhase # noqa from .pkgset import PkgsetPhase # noqa
from .gather import GatherPhase # noqa from .gather import GatherPhase # noqa
from .createrepo import CreaterepoPhase # noqa from .createrepo import CreaterepoPhase # noqa
@ -31,21 +32,3 @@ from .livemedia_phase import LiveMediaPhase # noqa
from .ostree import OSTreePhase # noqa from .ostree import OSTreePhase # noqa
from .ostree_installer import OstreeInstallerPhase # noqa from .ostree_installer import OstreeInstallerPhase # noqa
from .osbs import OSBSPhase # noqa from .osbs import OSBSPhase # noqa
def run_all(phases):
"""Start and stop all given phases and make them run in parallel.
This function makes sure that even if one of the phases fails and raises an
exception, all the phases will still be correctly stopped.
If multiple phases fail, a exception from one of them will be raised, but
there are no guarantees which one it will be.
"""
if not phases:
return
phases[0].start()
try:
run_all(phases[1:])
finally:
phases[0].stop()

72
pungi/phases/weaver.py Normal file
View File

@ -0,0 +1,72 @@
# -*- coding: utf-8 -*-
from kobo import shortcuts
from kobo.threads import ThreadPool, WorkerThread
class WeaverPhase(object):
"""
Special "phase" that manages other phases' run.
It needs input-running schema where particular phases are composed
sequentially and in parallel as well. A Sequential set of phases
is named "pipeline".
If any of the phases fail, we must ensure that others will stop correctly.
Otherwise the whole process will hang.
:param compose: it is needed for logging
:param phases_schema: two-dimensional array of phases. Top dimension
denotes particular pipelines. Second dimension contains phases.
"""
name = "weaver"
def __init__(self, compose, phases_schema):
self.msg = "---------- PHASE: %s ----------" % self.name.upper()
self.compose = compose
self.finished = False
self.pool = ThreadPool(logger=self.compose._logger)
if not phases_schema:
msg = "No running schema was set for WeaverPhase"
self.pool.log_error(msg)
raise ValueError(msg)
self._phases_schema = phases_schema
def start(self):
if self.finished:
msg = "Phase '%s' has already finished and can not be started twice" % self.name
self.pool.log_error(msg)
raise RuntimeError(msg)
self.compose.log_info("[BEGIN] %s" % self.msg)
self.run()
def run(self):
for pipeline in shortcuts.force_list(self._phases_schema):
self.pool.add(PipelineThread(self.pool))
self.pool.queue_put(shortcuts.force_list(pipeline))
self.pool.start()
def stop(self):
if self.finished:
return
if hasattr(self, "pool"):
self.pool.stop()
self.finished = True
self.compose.log_info("[DONE ] %s" % self.msg)
class PipelineThread(WorkerThread):
"""
Launches phases in pipeline sequentially
"""
def process(self, item, num):
pipeline = shortcuts.force_list(item)
phases_names = ", ".join(phase.name for phase in pipeline)
msg = "Running pipeline (%d/%d). Phases: %s" % (num, self.pool.queue_total, phases_names)
self.pool.log_info("[BEGIN] %s" % (msg,))
for phase in pipeline:
phase.start()
phase.stop()
self.pool.log_info("[DONE ] %s" % (msg,))

131
tests/test_phase_base.py
View File

@ -7,11 +7,13 @@ try:
except ImportError: except ImportError:
import unittest import unittest
import os import os
import random
import sys import sys
import time
sys.path.insert(0, os.path.join(os.path.dirname(__file__), "..")) sys.path.insert(0, os.path.join(os.path.dirname(__file__), ".."))
from pungi.phases import run_all, base from pungi.phases import base, weaver
from tests.helpers import DummyCompose, PungiTestCase, boom from tests.helpers import DummyCompose, PungiTestCase, boom
@ -72,49 +74,124 @@ class ImageConfigMixinTestCase(PungiTestCase):
self.assertEqual(resolve_git_url.num, 3, 'Resolver was not called three times') self.assertEqual(resolve_git_url.num, 3, 'Resolver was not called three times')
class TestRunAll(unittest.TestCase): class TestWeaver(unittest.TestCase):
def setUp(self):
# prepare 6 phase mock-objects.
for test_phase_number in range(1, 7):
# This is equvalent to:
# self.pX = mock.Mock()
# self.pX.name = "phase X"
# self.pX.start = default_method
test_phase_name = "p" + repr(test_phase_number)
tmp = mock.Mock()
tmp.name = "phase %d" % test_phase_number
tmp.start.side_effect = self.method_regular
setattr(self, test_phase_name, tmp)
self.compose = DummyCompose(None, {})
def method_regular(self):
"""
It only have to cause some delay (tens of miliseconds).
Delay is needed for threads that has enough time to start.
"""
multiplier = random.sample(range(1, 10), 1)
time.sleep(multiplier[0] * 0.01)
def method_with_exception(self):
self.method_regular() # make some delay
boom() # throw exception
def assertFinalized(self, p): def assertFinalized(self, p):
self.assertEqual(p.mock_calls, [mock.call.start(), mock.call.stop()]) self.assertEqual(p.mock_calls, [mock.call.start(), mock.call.stop()])
def test_calls_stop(self): def assertInterrupted(self, p):
p1 = mock.Mock() self.assertEqual(p.mock_calls, [mock.call.start()])
p2 = mock.Mock()
run_all([p1, p2]) def assertMissed(self, p):
self.assertEqual(p.mock_calls, [])
self.assertFinalized(p1) def test_parallel(self):
self.assertFinalized(p2) phases_schema = (self.p1, self.p2)
weaver_phase = weaver.WeaverPhase(self.compose, phases_schema)
weaver_phase.start()
weaver_phase.stop()
def test_calls_stop_on_failure(self): self.assertFinalized(self.p1)
p1 = mock.Mock() self.assertFinalized(self.p2)
p2 = mock.Mock()
p3 = mock.Mock()
p2.stop.side_effect = boom def test_pipeline(self):
phases_schema = ((self.p1, self.p2),)
weaver_phase = weaver.WeaverPhase(self.compose, phases_schema)
weaver_phase.start()
weaver_phase.stop()
self.assertFinalized(self.p1)
self.assertFinalized(self.p2)
def test_stop_on_failure(self):
self.p2.start.side_effect = self.method_with_exception
phases_schema = ((self.p1, self.p2, self.p3),) # one pipeline
weaver_phase = weaver.WeaverPhase(self.compose, phases_schema)
with self.assertRaises(Exception) as ctx: with self.assertRaises(Exception) as ctx:
run_all([p1, p2, p3]) weaver_phase.start()
weaver_phase.stop()
self.assertEqual('BOOM', str(ctx.exception)) self.assertEqual('BOOM', str(ctx.exception))
self.assertFinalized(p1) self.assertFinalized(self.p1)
self.assertFinalized(p2) self.assertInterrupted(self.p2)
self.assertFinalized(p3) self.assertMissed(self.p3)
def test_parallel_stop_on_failure(self):
self.p2.start.side_effect = self.method_with_exception
phases_schema = (self.p1, self.p2, self.p3) # one pipeline
weaver_phase = weaver.WeaverPhase(self.compose, phases_schema)
with self.assertRaises(Exception) as ctx:
weaver_phase.start()
weaver_phase.stop()
self.assertEqual('BOOM', str(ctx.exception))
self.assertFinalized(self.p1)
self.assertInterrupted(self.p2)
self.assertFinalized(self.p3)
def test_multiple_fail(self): def test_multiple_fail(self):
p1 = mock.Mock(name='p1') self.p2.start.side_effect = self.method_with_exception
p2 = mock.Mock(name='p2') self.p3.start.side_effect = self.method_with_exception
p3 = mock.Mock(name='p3')
p2.stop.side_effect = boom
p3.stop.side_effect = boom
phases_schema = ((self.p1, self.p2, self.p3),) # one pipeline
weaver_phase = weaver.WeaverPhase(self.compose, phases_schema)
with self.assertRaises(Exception) as ctx: with self.assertRaises(Exception) as ctx:
run_all([p1, p2, p3]) weaver_phase.start()
weaver_phase.stop()
self.assertEqual('BOOM', str(ctx.exception)) self.assertEqual('BOOM', str(ctx.exception))
self.assertFinalized(p1) self.assertFinalized(self.p1)
self.assertFinalized(p2) self.assertInterrupted(self.p2)
self.assertFinalized(p3) self.assertMissed(self.p3)
def test_multi_pipeline(self):
self.p2.start.side_effect = self.method_with_exception
phases_schema = (
self.p1,
(self.p2, self.p3, self.p4),
(self.p5, self.p6),
)
weaver_phase = weaver.WeaverPhase(self.compose, phases_schema)
with self.assertRaises(Exception) as ctx:
weaver_phase.start()
weaver_phase.stop()
self.assertEqual('BOOM', str(ctx.exception))
self.assertFinalized(self.p1)
self.assertInterrupted(self.p2)
self.assertMissed(self.p3)
self.assertMissed(self.p4)
self.assertFinalized(self.p5)
self.assertFinalized(self.p6)
if __name__ == "__main__": if __name__ == "__main__":