almalinux/pungi
14
1
Fork 0
Code Issues Pull Requests Releases Activity

docs: Add overview of what each phase does

Browse Source 
Fixes: #520
Signed-off-by: Lubomír Sedlář <lsedlar@redhat.com>
This commit is contained in:
Lubomír Sedlář 2015-11-19 13:33:33 +01:00
parent bef573d222
commit 9ab3dc89b2
3 changed files with 594 additions and 0 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

442
doc/_static/phases.svg vendored Normal file
View File

@ -0,0 +1,442 @@
<?xml version="1.0" encoding="UTF-8" standalone="no"?>
<!-- Created with Inkscape (http://www.inkscape.org/) -->
<svg
xmlns:dc="http://purl.org/dc/elements/1.1/"
xmlns:cc="http://creativecommons.org/ns#"
xmlns:rdf="http://www.w3.org/1999/02/22-rdf-syntax-ns#"
xmlns:svg="http://www.w3.org/2000/svg"
xmlns="http://www.w3.org/2000/svg"
xmlns:sodipodi="http://sodipodi.sourceforge.net/DTD/sodipodi-0.dtd"
xmlns:inkscape="http://www.inkscape.org/namespaces/inkscape"
width="821.33331"
height="264.33334"
viewBox="0 0 821.33334 264.33335"
id="svg2"
version="1.1"
inkscape:version="0.92+devel unknown"
sodipodi:docname="phases.svg"
inkscape:export-filename="/home/lsedlar/repos/pungi/doc/_static/phases.png"
inkscape:export-xdpi="90"
inkscape:export-ydpi="90">
<sodipodi:namedview
id="base"
pagecolor="#ffffff"
bordercolor="#666666"
borderopacity="1.0"
inkscape:pageopacity="1"
inkscape:pageshadow="2"
inkscape:zoom="1.6532468"
inkscape:cx="487.41703"
inkscape:cy="90.21684"
inkscape:document-units="px"
inkscape:current-layer="layer1"
showgrid="false"
inkscape:window-width="1920"
inkscape:window-height="1016"
inkscape:window-x="1920"
inkscape:window-y="27"
inkscape:window-maximized="1"
units="px"
inkscape:document-rotation="0"
showguides="true"
inkscape:guide-bbox="true" />
<defs
id="defs4">
<marker
inkscape:isstock="true"
style="overflow:visible"
id="Arrow1Lend"
refX="0"
refY="0"
orient="auto"
inkscape:stockid="Arrow1Lend">
<path
inkscape:connector-curvature="0"
transform="matrix(-0.8,0,0,-0.8,-10,0)"
style="fill:#000000;fill-opacity:1;fill-rule:evenodd;stroke:#000000;stroke-width:1.00000003pt;stroke-opacity:1"
d="M 0,0 5,-5 -12.5,0 5,5 Z"
id="path4451" />
</marker>
</defs>
<metadata
id="metadata7">
<rdf:RDF>
<cc:Work
rdf:about="">
<dc:format>image/svg+xml</dc:format>
<dc:type
rdf:resource="http://purl.org/dc/dcmitype/StillImage" />
<dc:title></dc:title>
</cc:Work>
</rdf:RDF>
</metadata>
<g
transform="matrix(1.066667,0,0,1.066667,0,-858.18642)"
id="layer1"
inkscape:groupmode="layer"
inkscape:label="Vrstva 1">
<g
transform="translate(-1.131703,-122.61379)"
id="g3411">
<rect
style="fill:#8ae234;fill-rule:evenodd;stroke:none;stroke-width:1px;stroke-linecap:butt;stroke-linejoin:miter;stroke-opacity:1"
id="rect3340"
width="26.295755"
height="49.214859"
x="953.49097"
y="49.250374"
transform="matrix(0,1,1,0,0,0)" />
<text
xml:space="preserve"
style="font-style:normal;font-weight:normal;line-height:0%;font-family:sans-serif;letter-spacing:0px;word-spacing:0px;fill:#000000;fill-opacity:1;stroke:none;stroke-width:1px;stroke-linecap:butt;stroke-linejoin:miter;stroke-opacity:1"
x="51.554729"
y="970.26605"
id="text3360"><tspan
sodipodi:role="line"
id="tspan3362"
x="51.554729"
y="970.26605"
style="font-size:13.14787769px;line-height:1.25">Pkgset</tspan></text>
</g>
<g
transform="translate(-4.582059,-122.61379)"
id="g3416">
<rect
style="fill:#fcaf3e;fill-rule:evenodd;stroke:none;stroke-width:1px;stroke-linecap:butt;stroke-linejoin:miter;stroke-opacity:1"
id="rect3342"
width="26.295755"
height="231.47725"
x="953.49097"
y="108.04571"
transform="matrix(0,1,1,0,0,0)" />
<text
xml:space="preserve"
style="font-style:normal;font-weight:normal;line-height:0%;font-family:sans-serif;letter-spacing:0px;word-spacing:0px;fill:#000000;fill-opacity:1;stroke:none;stroke-width:1px;stroke-linecap:butt;stroke-linejoin:miter;stroke-opacity:1"
x="110.35005"
y="971.54041"
id="text3364"><tspan
sodipodi:role="line"
id="tspan3366"
x="110.35005"
y="971.54041"
style="font-size:13.14787769px;line-height:1.25">Buildinstall</tspan></text>
<rect
style="fill:#729fcf;fill-rule:evenodd;stroke:none;stroke-width:1px;stroke-linecap:butt;stroke-linejoin:miter;stroke-opacity:1"
id="rect3344"
width="26.295755"
height="54.197887"
x="989.65247"
y="112.96759"
transform="matrix(0,1,1,0,0,0)" />
<text
xml:space="preserve"
style="font-style:normal;font-weight:normal;line-height:0%;font-family:sans-serif;letter-spacing:0px;word-spacing:0px;fill:#000000;fill-opacity:1;stroke:none;stroke-width:1px;stroke-linecap:butt;stroke-linejoin:miter;stroke-opacity:1"
x="115.82405"
y="1007.7019"
id="text3368"><tspan
sodipodi:role="line"
id="tspan3370"
x="115.82405"
y="1007.7019"
style="font-size:13.14787769px;line-height:1.25">Gather</tspan></text>
<rect
style="fill:#ad7fa8;fill-rule:evenodd;stroke:none;stroke-width:1px;stroke-linecap:butt;stroke-linejoin:miter;stroke-opacity:1"
id="rect3346"
width="26.295755"
height="72.729973"
x="989.65247"
y="172.61172"
transform="matrix(0,1,1,0,0,0)" />
<text
xml:space="preserve"
style="font-style:normal;font-weight:normal;line-height:0%;font-family:sans-serif;letter-spacing:0px;word-spacing:0px;fill:#000000;fill-opacity:1;stroke:none;stroke-width:1px;stroke-linecap:butt;stroke-linejoin:miter;stroke-opacity:1"
x="174.91608"
y="1007.7019"
id="text3372"><tspan
sodipodi:role="line"
id="tspan3374"
x="174.91608"
y="1007.7019"
style="font-size:13.14787769px;line-height:1.25">ExtraFiles</tspan></text>
<rect
style="fill:#e9b96e;fill-rule:evenodd;stroke:none;stroke-width:1px;stroke-linecap:butt;stroke-linejoin:miter;stroke-opacity:1"
id="rect3348"
width="26.295755"
height="78.636055"
x="989.65247"
y="250.78795"
transform="matrix(0,1,1,0,0,0)" />
<text
xml:space="preserve"
style="font-style:normal;font-weight:normal;line-height:0%;font-family:sans-serif;letter-spacing:0px;word-spacing:0px;fill:#000000;fill-opacity:1;stroke:none;stroke-width:1px;stroke-linecap:butt;stroke-linejoin:miter;stroke-opacity:1"
x="253.64439"
y="1006.312"
id="text3376"><tspan
sodipodi:role="line"
id="tspan3378"
x="253.64439"
y="1006.312"
style="font-size:13.14787769px;line-height:1.25">Createrepo</tspan></text>
</g>
<g
transform="translate(51.751666,-122.61373)"
id="g3430">
<rect
style="fill:#a40000;fill-rule:evenodd;stroke:none;stroke-width:1px;stroke-linecap:butt;stroke-linejoin:miter;stroke-opacity:1"
id="rect3350"
width="26.295755"
height="83.352409"
x="953.49091"
y="349.10342"
transform="matrix(0,1,1,0,0,0)" />
<text
xml:space="preserve"
style="font-style:normal;font-weight:normal;line-height:0%;font-family:sans-serif;letter-spacing:0px;word-spacing:0px;fill:#ffffff;fill-opacity:1;stroke:none;stroke-width:1px;stroke-linecap:butt;stroke-linejoin:miter;stroke-opacity:1"
x="351.40778"
y="970.26605"
id="text3380"><tspan
sodipodi:role="line"
id="tspan3382"
x="351.40778"
y="970.26605"
style="font-size:13.14787769px;line-height:1.25">Productimg</tspan></text>
</g>
<g
transform="translate(45.632403,-122.61379)"
id="g3446">
<rect
y="554.10059"
x="953.49097"
height="115.80065"
width="26.295755"
id="rect3338-1"
style="fill:#e9b96e;fill-rule:evenodd;stroke:none;stroke-width:1px;stroke-linecap:butt;stroke-linejoin:miter;stroke-opacity:1"
transform="matrix(0,1,1,0,0,0)" />
<text
xml:space="preserve"
style="font-style:normal;font-variant:normal;font-weight:normal;font-stretch:normal;line-height:0%;font-family:sans-serif;-inkscape-font-specification:'sans-serif, Normal';text-align:start;letter-spacing:0px;word-spacing:0px;writing-mode:lr-tb;text-anchor:start;fill:#000000;fill-opacity:1;stroke:none;stroke-width:1px;stroke-linecap:butt;stroke-linejoin:miter;stroke-opacity:1"
x="556.95709"
y="971.54041"
id="text3384-0"><tspan
y="971.54041"
x="556.95709"
sodipodi:role="line"
id="tspan3391"
style="font-size:13.14749908px;line-height:1.25">ImageChecksum</tspan></text>
</g>
<g
transform="translate(167.68135,-122.61379)"
id="g3398">
<rect
y="553.98242"
x="953.49097"
height="46.01757"
width="26.295755"
id="rect3400"
style="fill:#3465a4;fill-rule:evenodd;stroke:none;stroke-width:1px;stroke-linecap:butt;stroke-linejoin:miter;stroke-opacity:1"
transform="matrix(0,1,1,0,0,0)" />
<text
xml:space="preserve"
style="font-style:normal;font-weight:normal;line-height:0%;font-family:sans-serif;letter-spacing:0px;word-spacing:0px;fill:#ffffff;fill-opacity:1;stroke:none;stroke-width:1px;stroke-linecap:butt;stroke-linejoin:miter;stroke-opacity:1"
x="557.61566"
y="971.33813"
id="text3396"><tspan
sodipodi:role="line"
id="tspan3398"
x="557.61566"
y="971.33813"
style="font-size:13.14787769px;line-height:1.25">Test</tspan></text>
</g>
<g
transform="translate(2.318656,-122.61379)"
id="g3406">
<rect
style="fill:#fce94f;fill-rule:evenodd;stroke:none;stroke-width:1px;stroke-linecap:butt;stroke-linejoin:miter;stroke-opacity:1"
id="rect3336"
width="26.295755"
height="39.669899"
x="953.49097"
y="-2.7716319e-06"
transform="matrix(0,1,1,0,0,0)" />
<text
xml:space="preserve"
style="font-style:normal;font-weight:normal;line-height:0%;font-family:sans-serif;letter-spacing:0px;word-spacing:0px;fill:#000000;fill-opacity:1;stroke:none;stroke-width:1px;stroke-linecap:butt;stroke-linejoin:miter;stroke-opacity:1"
x="1.1060941"
y="971.63348"
id="text3356"><tspan
sodipodi:role="line"
id="tspan3358"
x="1.1060941"
y="971.63348"
style="font-size:13.14787769px;line-height:1.25">Init</tspan></text>
</g>
<path
inkscape:connector-curvature="0"
id="path3642"
d="M 100.90864,817.7484 H 712.42335"
style="fill:none;fill-rule:evenodd;stroke:#000000;stroke-width:1.23489845px;stroke-linecap:butt;stroke-linejoin:miter;stroke-opacity:1;marker-end:url(#Arrow1Lend)" />
<g
transform="translate(86.469501,7.3304162)"
id="g3408">
<rect
style="fill:#729fcf;fill-rule:evenodd;stroke:none;stroke-width:1px;stroke-linecap:butt;stroke-linejoin:miter;stroke-opacity:1"
id="rect3350-3"
width="26.295755"
height="53.653927"
x="823.54675"
y="254.60153"
transform="matrix(0,1,1,0,0,0)" />
<text
xml:space="preserve"
style="font-style:normal;font-weight:normal;line-height:0%;font-family:sans-serif;text-align:start;letter-spacing:0px;word-spacing:0px;writing-mode:lr-tb;text-anchor:start;fill:#ffffff;fill-opacity:1;stroke:none;stroke-width:1px;stroke-linecap:butt;stroke-linejoin:miter;stroke-opacity:1"
x="256.90588"
y="840.3219"
id="text3380-2"><tspan
y="840.3219"
x="256.90588"
sodipodi:role="line"
id="tspan3406"
style="font-size:13.14787769px;line-height:1.25">OSTree</tspan></text>
</g>
<g
id="g336">
<g
transform="translate(70.651385,-122.61379)"
id="g3458">
<rect
transform="matrix(0,1,1,0,0,0)"
style="fill:#edd400;fill-rule:evenodd;stroke:none;stroke-width:1px;stroke-linecap:butt;stroke-linejoin:miter;stroke-opacity:1"
id="rect3338"
width="26.295755"
height="102.36562"
x="953.49097"
y="420.13605" />
<text
id="text3384"
y="971.54041"
x="422.99252"
style="font-style:normal;font-weight:normal;line-height:0%;font-family:sans-serif;letter-spacing:0px;word-spacing:0px;fill:#000000;fill-opacity:1;stroke:none;stroke-width:1px;stroke-linecap:butt;stroke-linejoin:miter;stroke-opacity:1"
xml:space="preserve"><tspan
y="971.54041"
x="422.99252"
id="tspan3386"
sodipodi:role="line"
style="font-size:13.14787769px;line-height:1.25">Createiso</tspan></text>
</g>
<g
id="g3453"
transform="translate(70.394938,-121.89312)">
<rect
transform="matrix(0,1,1,0,0,0)"
y="420.39337"
x="989.65247"
height="101.85102"
width="26.295755"
id="rect3352"
style="fill:#73d216;fill-rule:evenodd;stroke:none;stroke-width:1px;stroke-linecap:butt;stroke-linejoin:miter;stroke-opacity:1" />
<text
id="text3388"
y="1006.4276"
x="422.69772"
style="font-style:normal;font-weight:normal;line-height:0%;font-family:sans-serif;letter-spacing:0px;word-spacing:0px;fill:#000000;fill-opacity:1;stroke:none;stroke-width:1px;stroke-linecap:butt;stroke-linejoin:miter;stroke-opacity:1"
xml:space="preserve"><tspan
y="1006.4276"
x="422.69772"
id="tspan3390"
sodipodi:role="line"
style="font-size:13.14787769px;line-height:1.25">LiveImages</tspan></text>
</g>
<g
id="g3448"
transform="translate(70.394938,-121.42489)">
<rect
transform="matrix(0,1,1,0,0,0)"
y="420.39337"
x="1026.0664"
height="101.85102"
width="26.295755"
id="rect3354"
style="fill:#f57900;fill-rule:evenodd;stroke:none;stroke-width:1px;stroke-linecap:butt;stroke-linejoin:miter;stroke-opacity:1" />
<text
id="text3392"
y="1042.8416"
x="422.69772"
style="font-style:normal;font-weight:normal;line-height:0%;font-family:sans-serif;letter-spacing:0px;word-spacing:0px;fill:#000000;fill-opacity:1;stroke:none;stroke-width:1px;stroke-linecap:butt;stroke-linejoin:miter;stroke-opacity:1"
xml:space="preserve"><tspan
y="1042.8416"
x="422.69772"
id="tspan3394"
sodipodi:role="line"
style="font-size:13.14787769px;line-height:1.25">ImageBuild</tspan></text>
</g>
<g
id="g3443"
transform="translate(69.687846,-121.31225)">
<rect
style="fill:#edd400;fill-rule:evenodd;stroke:none;stroke-width:1px;stroke-linecap:butt;stroke-linejoin:miter;stroke-opacity:1"
id="rect3422"
width="26.295755"
height="101.85102"
x="1062.8359"
y="421.10046"
transform="matrix(0,1,1,0,0,0)" />
<text
xml:space="preserve"
style="font-style:normal;font-weight:normal;line-height:0%;font-family:sans-serif;text-align:start;letter-spacing:0px;word-spacing:0px;writing-mode:lr-tb;text-anchor:start;fill:#000000;fill-opacity:1;stroke:none;stroke-width:1px;stroke-linecap:butt;stroke-linejoin:miter;stroke-opacity:1"
x="423.40482"
y="1079.6111"
id="text3424"><tspan
id="tspan3434"
sodipodi:role="line"
x="423.40482"
y="1079.6111"
style="font-size:13.14787769px;line-height:1.25">LiveMedia</tspan></text>
</g>
<g
transform="translate(0.45066315,-44.231132)"
id="g288">
<rect
transform="matrix(0,1,1,0,0,0)"
y="490.33765"
x="1022.637"
height="101.85102"
width="26.295755"
id="rect3428"
style="fill:#fcaf3e;fill-rule:evenodd;stroke:none;stroke-width:1px;stroke-linecap:butt;stroke-linejoin:miter;stroke-opacity:1" />
<text
id="text3430"
y="1039.4121"
x="492.642"
style="font-style:normal;font-variant:normal;font-weight:normal;font-stretch:normal;font-size:11.99999714px;line-height:0%;font-family:sans-serif;-inkscape-font-specification:'sans-serif, Normal';font-variant-ligatures:normal;font-variant-caps:normal;font-variant-numeric:normal;font-feature-settings:normal;text-align:start;letter-spacing:0px;word-spacing:0px;writing-mode:lr-tb;text-anchor:start;fill:#000000;fill-opacity:1;stroke:none;stroke-width:1px;stroke-linecap:butt;stroke-linejoin:miter;stroke-opacity:1"
xml:space="preserve"><tspan
y="1039.4121"
x="492.642"
sodipodi:role="line"
id="tspan283">OSTreeInstaller</tspan></text>
</g>
<g
transform="translate(0.45066315,-35.507965)"
id="g306">
<rect
style="fill:#c17d11;fill-rule:evenodd;stroke:none;stroke-width:1px;stroke-linecap:butt;stroke-linejoin:miter;stroke-opacity:1"
id="rect290"
width="26.295755"
height="101.85102"
x="1048.9327"
y="490.33765"
transform="matrix(0,1,1,0,0,0)" />
<text
xml:space="preserve"
style="font-style:normal;font-variant:normal;font-weight:normal;font-stretch:normal;font-size:11.99999714px;line-height:0%;font-family:sans-serif;-inkscape-font-specification:'sans-serif, Normal';font-variant-ligatures:normal;font-variant-caps:normal;font-variant-numeric:normal;font-feature-settings:normal;text-align:start;letter-spacing:0px;word-spacing:0px;writing-mode:lr-tb;text-anchor:start;fill:#000000;fill-opacity:1;stroke:none;stroke-width:1px;stroke-linecap:butt;stroke-linejoin:miter;stroke-opacity:1"
x="492.642"
y="1065.7078"
id="text294"><tspan
y="1065.7078"
x="492.642"
sodipodi:role="line"
id="tspan301">OSBS</tspan></text>
</g>
</g>
</g>
</svg>
Side by Side

After

Width:  |  Height:  |  Size: 18 KiB

1
doc/index.rst
View File

@ -16,3 +16,4 @@ Contents:
testing testing
configuration configuration
messaging messaging
phases

151
doc/phases.rst Normal file
View File

@ -0,0 +1,151 @@
.. _phases:
Phases
======
Each invocation of ``punji-koji`` consists of a set of phases.
.. image:: _static/phases.svg
:alt: phase diagram
Most of the phases run sequentially (left-to-right in the diagram), but there
are use cases where multiple phases run in parallel. This happens for phases
whose main point is to wait for a Koji task to finish.
Init
----
The first phase to ever run. Can not be skipped. It prepares the comps files
for variants (by filtering out groups and packages that should not be there).
Pkgset
------
This phase loads a set of packages that should be composed. It has two separate
results: it prepares repos with packages in ``work/`` directory (one per arch)
for further processing, and it returns a data structure with mapping of
packages to architectures.
Buildinstall
------------
Spawns a bunch of threads, each of which runs either ``lorax`` or
``buildinstall`` command (the latter coming from ``anaconda`` package). The
commands create ``boot.iso`` and other boot configuration files. The image is
finally linked into the ``compose/`` directory as netinstall media.
The created images are also needed for creating live media or other images in
later phases.
With ``lorax`` this phase runs one task per variant.arch combination. For
``buildinstall`` command there is only one task per architecture and
``product.img`` should be used to customize the results.
Gather
------
This phase uses data collected by ``pkgset`` phase and figures out what
packages should be in each variant. The basic mapping can come from comps file,
a JSON mapping or ``additional_packages`` config option. This inputs can then
be enriched by adding all dependencies.
Once the mapping is finalized, the packages are linked to appropriate places
and the ``rpms.json`` manifest is created.
ExtraFiles
----------
This phase collects extra files from the configuration and copies them to the
compose directory. The files are described by a JSON file in the compose
subtree where the files are copied. This metadata is meant to be distributed
with the data (on ISO images).
Createrepo
----------
This phase creates RPM repositories for each variant.arch tree. It is actually
reading the ``rpms.json`` manifest to figure out which packages should be
included.
OSTree
------
Updates an ostree repository with a new commit with packages from the compose.
The repository lives outside of the compose and is updated immediately. If the
compose fails in a later stage, the commit will not be reverted.
Implementation wise, this phase runs ``rpm-ostree`` command in Koji runroot (to
allow running on different arches).
Productimg
----------
Creates ``product.img`` files for customizing the bootable media created in
``buildinstall`` phase.
Createiso
---------
Generates ISO files and accumulates enough metadata to be able to create
``image.json`` manifest. The file is however not created in this phase, instead
it is dumped in the ``pungi-koji`` script itself.
The files include a repository with all RPMs from the variant. There will be
multiple images if the packages do not fit on a single image.
The image will be bootable if ``buildinstall`` phase is enabled and the
packages fit on a single image.
There can also be images with source repositories. These are never bootable.
LiveImages, LiveMedia
---------------------
Creates media in Koji with ``koji spin-livecd``, ``koji spin-appliance`` or
``koji spin-livemedia`` command. When the media are finished, the images are
copied into the ``compose/`` directory and metadata for images is updated.
ImageBuild
----------
This phase wraps up ``koji image-build``. It also updates the metadata
ultimately responsible for ``images.json`` manifest.
OSBS
----
This phase builds docker base images in `OSBS
<http://osbs.readthedocs.io/en/latest/index.html>`_.
The finished images are available in registry provided by OSBS, but not
downloaded directly into the compose. The is metadata about the created image
in ``compose/metadata/osbs.json``.
OSTreeInstaller
---------------
Creates bootable media that carry an ostree repository as a payload. These
images are created by running ``lorax`` with special templates. Again it runs
in Koji runroot.
ImageChecksum
-------------
Responsible for generating checksums for the images. The checksums are stored
in image manifest as well as files on disk. The list of images to be processed
is obtained from the image manifest. This way all images will get the same
checksums irrespective of the phase that created them.
Test
----
This phase is supposed to run some sanity checks on the finished compose.
The first test is to run ``repoclosure`` on each repository. However, even if
it fails, the compose will still be considered a success. The actual error has
to be looked up in the compose logs directory.
The other test is to check all images listed the metadata and verify that they
look sane. For ISO files headers are checked to verify the format is correct,
and for bootable media a check is run to verify they have properties that allow
booting.