- Changelog date order

- Typo

1715.patch

@@ -0,0 +1,41 @@
+From 432b0bce0401c4bbcd1a958a89305c475a794f26 Mon Sep 17 00:00:00 2001
+From: Adam Williamson <awilliam@redhat.com>
+Date: Jan 19 2024 07:25:09 +0000
+Subject: checks: don't require "repo" in the "ostree" schema
+
+
+Per @siosm in https://pagure.io/pungi-fedora/pull-request/1227
+this option "is deprecated and not needed anymore", so Pungi
+should not be requiring it.
+
+Merges: https://pagure.io/pungi/pull-request/1714
+Signed-off-by: Adam Williamson <awilliam@redhat.com>
+
+---
+
+diff --git a/pungi/checks.py b/pungi/checks.py
+index a340f93..db8b297 100644
+--- a/pungi/checks.py
++++ b/pungi/checks.py
+@@ -1066,7 +1066,6 @@ def make_schema():
+                                     "required": [
+                                         "treefile",
+                                         "config_url",
+-                                        "repo",
+                                         "ostree_repo",
+                                     ],
+                                     "additionalProperties": False,
+diff --git a/pungi/phases/ostree.py b/pungi/phases/ostree.py
+index 90578ae..2649cdb 100644
+--- a/pungi/phases/ostree.py
++++ b/pungi/phases/ostree.py
+@@ -85,7 +85,7 @@ class OSTreeThread(WorkerThread):
+         comps_repo = compose.paths.work.comps_repo(
+             "$basearch", variant=variant, create_dir=False
+         )
+-        repos = shortcuts.force_list(config["repo"]) + self.repos
++        repos = shortcuts.force_list(config.get("repo", [])) + self.repos
+         if compose.has_comps:
+             repos.append(translate_path(compose, comps_repo))
+         repos = get_repo_dicts(repos, logger=self.pool)
+

TODO

@@ -47,7 +47,6 @@ Split Pungi into smaller well-defined tools
 
 * create install images
   * lorax
-  * buildinstall
 
 * create isos
   * isos

doc/_static/phases.svg

@@ -1,22 +1,22 @@
 <?xml version="1.0" encoding="UTF-8" standalone="no"?>
 <svg
-   xmlns:dc="http://purl.org/dc/elements/1.1/"
-   xmlns:cc="http://creativecommons.org/ns#"
-   xmlns:rdf="http://www.w3.org/1999/02/22-rdf-syntax-ns#"
-   xmlns:svg="http://www.w3.org/2000/svg"
-   xmlns="http://www.w3.org/2000/svg"
-   xmlns:sodipodi="http://sodipodi.sourceforge.net/DTD/sodipodi-0.dtd"
-   xmlns:inkscape="http://www.inkscape.org/namespaces/inkscape"
    width="610.46454"
-   height="301.1662"
-   viewBox="0 0 610.46457 301.1662"
+   height="327.16599"
+   viewBox="0 0 610.46457 327.16599"
    id="svg2"
    version="1.1"
-   inkscape:version="1.0.2 (e86c870879, 2021-01-15)"
+   inkscape:version="1.3.2 (091e20e, 2023-11-25)"
    sodipodi:docname="phases.svg"
    inkscape:export-filename="/home/lsedlar/repos/pungi/doc/_static/phases.png"
    inkscape:export-xdpi="90"
-   inkscape:export-ydpi="90">
+   inkscape:export-ydpi="90"
+   xmlns:inkscape="http://www.inkscape.org/namespaces/inkscape"
+   xmlns:sodipodi="http://sodipodi.sourceforge.net/DTD/sodipodi-0.dtd"
+   xmlns="http://www.w3.org/2000/svg"
+   xmlns:svg="http://www.w3.org/2000/svg"
+   xmlns:rdf="http://www.w3.org/1999/02/22-rdf-syntax-ns#"
+   xmlns:cc="http://creativecommons.org/ns#"
+   xmlns:dc="http://purl.org/dc/elements/1.1/">
   <sodipodi:namedview
      id="base"
      pagecolor="#ffffff"
@@ -25,15 +25,15 @@
      inkscape:pageopacity="1"
      inkscape:pageshadow="2"
      inkscape:zoom="1.5"
-     inkscape:cx="9.4746397"
-     inkscape:cy="58.833855"
+     inkscape:cx="268"
+     inkscape:cy="260.66667"
      inkscape:document-units="px"
      inkscape:current-layer="layer1"
      showgrid="false"
-     inkscape:window-width="2560"
-     inkscape:window-height="1376"
+     inkscape:window-width="1920"
+     inkscape:window-height="1027"
      inkscape:window-x="0"
-     inkscape:window-y="0"
+     inkscape:window-y="25"
      inkscape:window-maximized="1"
      units="px"
      inkscape:document-rotation="0"
@@ -43,7 +43,10 @@
      fit-margin-left="7.4"
      fit-margin-right="7.4"
      fit-margin-bottom="7.4"
-     lock-margins="true" />
+     lock-margins="true"
+     inkscape:showpageshadow="2"
+     inkscape:pagecheckerboard="0"
+     inkscape:deskcolor="#d1d1d1" />
   <defs
      id="defs4">
     <marker
@@ -70,7 +73,6 @@
         <dc:format>image/svg+xml</dc:format>
         <dc:type
            rdf:resource="http://purl.org/dc/dcmitype/StillImage" />
-        <dc:title />
       </cc:Work>
     </rdf:RDF>
   </metadata>
@@ -103,7 +105,7 @@
            style="font-size:13.1479px;line-height:1.25">Pkgset</tspan></text>
     </g>
     <g
-       transform="translate(58.253953,-80.817124)"
+       transform="translate(56.378954,-80.817124)"
        id="g3398">
       <rect
          y="553.98242"
@@ -301,25 +303,29 @@
         </g>
       </g>
     </g>
-    <rect
-       transform="matrix(0,1,1,0,0,0)"
-       style="fill:#e9b96e;fill-rule:evenodd;stroke:none;stroke-width:1.85901px;stroke-linecap:butt;stroke-linejoin:miter;stroke-opacity:1"
-       id="rect3338-1"
-       width="90.874992"
-       height="115.80065"
-       x="872.67383"
-       y="486.55563" />
-    <text
-       id="text3384-0"
-       y="921.73846"
-       x="489.56451"
-       style="font-style:normal;font-variant:normal;font-weight:normal;font-stretch:normal;line-height:0%;font-family:sans-serif;-inkscape-font-specification:'sans-serif, Normal';text-align:start;letter-spacing:0px;word-spacing:0px;writing-mode:lr-tb;text-anchor:start;fill:#000000;fill-opacity:1;stroke:none;stroke-width:1px;stroke-linecap:butt;stroke-linejoin:miter;stroke-opacity:1"
-       xml:space="preserve"><tspan
-         style="font-size:13.1475px;line-height:1.25"
-         id="tspan3391"
-         sodipodi:role="line"
+    <g
+       id="g2"
+       transform="translate(-1.4062678e-8,9.3749966)">
+      <rect
+         transform="matrix(0,1,1,0,0,0)"
+         style="fill:#e9b96e;fill-rule:evenodd;stroke:none;stroke-width:1.85901px;stroke-linecap:butt;stroke-linejoin:miter;stroke-opacity:1"
+         id="rect3338-1"
+         width="103.12497"
+         height="115.80065"
+         x="863.29883"
+         y="486.55563" />
+      <text
+         id="text3384-0"
+         y="921.73846"
          x="489.56451"
-         y="921.73846">ImageChecksum</tspan></text>
+         style="font-style:normal;font-variant:normal;font-weight:normal;font-stretch:normal;line-height:0%;font-family:sans-serif;-inkscape-font-specification:'sans-serif, Normal';text-align:start;letter-spacing:0px;word-spacing:0px;writing-mode:lr-tb;text-anchor:start;fill:#000000;fill-opacity:1;stroke:none;stroke-width:1px;stroke-linecap:butt;stroke-linejoin:miter;stroke-opacity:1"
+         xml:space="preserve"><tspan
+           style="font-size:13.1475px;line-height:1.25"
+           id="tspan3391"
+           sodipodi:role="line"
+           x="489.56451"
+           y="921.73846">ImageChecksum</tspan></text>
+    </g>
     <g
        transform="translate(-42.209584,-80.817124)"
        id="g3458">
@@ -417,16 +423,16 @@
        id="rect290"
        width="26.295755"
        height="224.35098"
-       x="1063.5973"
+       x="1091.7223"
        y="378.43698"
        transform="matrix(0,1,1,0,0,0)" />
     <text
        xml:space="preserve"
        style="font-style:normal;font-variant:normal;font-weight:normal;font-stretch:normal;line-height:0%;font-family:sans-serif;-inkscape-font-specification:'sans-serif, Normal';text-align:start;letter-spacing:0px;word-spacing:0px;writing-mode:lr-tb;text-anchor:start;fill:#000000;fill-opacity:1;stroke:none;stroke-width:1px;stroke-linecap:butt;stroke-linejoin:miter;stroke-opacity:1"
        x="380.74133"
-       y="1080.3723"
+       y="1106.6223"
        id="text294"><tspan
-         y="1080.3723"
+         y="1106.6223"
          x="380.74133"
          sodipodi:role="line"
          id="tspan301"
@@ -454,32 +460,9 @@
            y="1069.0087"
            id="tspan3812">ExtraIsos</tspan></text>
     </g>
-    <g
-       id="g1031"
-       transform="translate(-40.740337,29.23522)">
-      <rect
-         transform="matrix(0,1,1,0,0,0)"
-         style="fill:#5ed4ec;fill-opacity:1;fill-rule:evenodd;stroke:none;stroke-width:1px;stroke-linecap:butt;stroke-linejoin:miter;stroke-opacity:1"
-         id="rect206"
-         width="26.295755"
-         height="102.36562"
-         x="1066.8611"
-         y="418.66275" />
-      <text
-         id="text210"
-         y="1084.9105"
-         x="421.51923"
-         style="font-style:normal;font-weight:normal;line-height:0%;font-family:sans-serif;letter-spacing:0px;word-spacing:0px;fill:#000000;fill-opacity:1;stroke:none;stroke-width:1px;stroke-linecap:butt;stroke-linejoin:miter;stroke-opacity:1"
-         xml:space="preserve"><tspan
-           y="1084.9105"
-           x="421.51923"
-           id="tspan208"
-           sodipodi:role="line"
-           style="font-size:13.1479px;line-height:1.25">Repoclosure</tspan></text>
-    </g>
     <rect
        y="377.92242"
-       x="1096.0963"
+       x="1122.3463"
        height="224.24059"
        width="26.295755"
        id="rect87"
@@ -489,17 +472,18 @@
        xml:space="preserve"
        style="font-style:normal;font-weight:normal;line-height:0%;font-family:sans-serif;letter-spacing:0px;word-spacing:0px;fill:#000000;fill-opacity:1;stroke:none;stroke-width:1px;stroke-linecap:butt;stroke-linejoin:miter;stroke-opacity:1"
        x="380.7789"
-       y="1114.1458"
+       y="1140.3958"
        id="text91"><tspan
          style="font-size:13.1479px;line-height:1.25"
          sodipodi:role="line"
          id="tspan89"
          x="380.7789"
-         y="1114.1458">Repoclosure</tspan></text>
+         y="1140.3958">Repoclosure</tspan></text>
     <g
-       id="g206">
+       id="g206"
+       transform="translate(0,-1.8749994)">
       <rect
-         style="fill:#fcaf3e;fill-rule:evenodd;stroke:none;stroke-width:1.00033px;stroke-linecap:butt;stroke-linejoin:miter;stroke-opacity:1"
+         style="fill:#fcd9a4;fill-opacity:1;fill-rule:evenodd;stroke:none;stroke-width:1.00033px;stroke-linecap:butt;stroke-linejoin:miter;stroke-opacity:1"
          id="rect290-6"
          width="26.295755"
          height="101.91849"
@@ -516,26 +500,58 @@
            x="380.23166"
            sodipodi:role="line"
            id="tspan301-5"
+           style="font-size:12px;line-height:0">KiwiBuild</tspan></text>
+    </g>
+    <g
+       id="g3">
+      <g
+         id="g1">
+        <g
+           id="g4">
+          <rect
+             transform="matrix(0,1,1,0,0,0)"
+             style="fill:#729fcf;fill-rule:evenodd;stroke:none;stroke-width:1.83502px;stroke-linecap:butt;stroke-linejoin:miter;stroke-opacity:1"
+             id="rect3338-1-3"
+             width="103.12497"
+             height="115.80065"
+             x="983.44263"
+             y="486.55563" />
+          <text
+             id="text3384-0-6"
+             y="1038.8422"
+             x="489.56451"
+             style="font-style:normal;font-variant:normal;font-weight:normal;font-stretch:normal;line-height:0%;font-family:sans-serif;-inkscape-font-specification:'sans-serif, Normal';text-align:start;letter-spacing:0px;word-spacing:0px;writing-mode:lr-tb;text-anchor:start;fill:#000000;fill-opacity:1;stroke:none;stroke-width:1px;stroke-linecap:butt;stroke-linejoin:miter;stroke-opacity:1"
+             xml:space="preserve"><tspan
+               style="font-size:13.1475px;line-height:1.25"
+               id="tspan3391-7"
+               sodipodi:role="line"
+               x="489.56451"
+               y="1038.8422">ImageContainer</tspan></text>
+        </g>
+      </g>
+    </g>
+    <g
+       id="g206-1"
+       transform="translate(-0.04628921,28.701853)">
+      <rect
+         style="fill:#fcaf3e;fill-rule:evenodd;stroke:none;stroke-width:1.00033px;stroke-linecap:butt;stroke-linejoin:miter;stroke-opacity:1"
+         id="rect290-6-7"
+         width="26.295755"
+         height="101.91849"
+         x="1032.3469"
+         y="377.92731"
+         transform="matrix(0,1,1,0,0,0)" />
+      <text
+         xml:space="preserve"
+         style="font-style:normal;font-variant:normal;font-weight:normal;font-stretch:normal;line-height:0%;font-family:sans-serif;-inkscape-font-specification:'sans-serif, Normal';text-align:start;letter-spacing:0px;word-spacing:0px;writing-mode:lr-tb;text-anchor:start;fill:#000000;fill-opacity:1;stroke:none;stroke-width:1px;stroke-linecap:butt;stroke-linejoin:miter;stroke-opacity:1"
+         x="380.23166"
+         y="1049.1219"
+         id="text294-7-5"><tspan
+           y="1049.1219"
+           x="380.23166"
+           sodipodi:role="line"
+           id="tspan301-5-5"
            style="font-size:12px;line-height:0">OSBuild</tspan></text>
     </g>
-    <rect
-       transform="matrix(0,1,1,0,0,0)"
-       style="fill:#729fcf;fill-rule:evenodd;stroke:none;stroke-width:1.83502px;stroke-linecap:butt;stroke-linejoin:miter;stroke-opacity:1"
-       id="rect3338-1-3"
-       width="88.544876"
-       height="115.80065"
-       x="970.31763"
-       y="486.55563" />
-    <text
-       id="text3384-0-6"
-       y="1018.2172"
-       x="489.56451"
-       style="font-style:normal;font-variant:normal;font-weight:normal;font-stretch:normal;line-height:0%;font-family:sans-serif;-inkscape-font-specification:'sans-serif, Normal';text-align:start;letter-spacing:0px;word-spacing:0px;writing-mode:lr-tb;text-anchor:start;fill:#000000;fill-opacity:1;stroke:none;stroke-width:1px;stroke-linecap:butt;stroke-linejoin:miter;stroke-opacity:1"
-       xml:space="preserve"><tspan
-         style="font-size:13.1475px;line-height:1.25"
-         id="tspan3391-7"
-         sodipodi:role="line"
-         x="489.56451"
-         y="1018.2172">ImageContainer</tspan></text>
   </g>
 </svg>

doc/conf.py

@@ -18,12 +18,12 @@ import os
 # If extensions (or modules to document with autodoc) are in another directory,
 # add these directories to sys.path here. If the directory is relative to the
 # documentation root, use os.path.abspath to make it absolute, like shown here.
-#sys.path.insert(0, os.path.abspath('.'))
+# sys.path.insert(0, os.path.abspath('.'))
 
 # -- General configuration ------------------------------------------------
 
 # If your documentation needs a minimal Sphinx version, state it here.
-#needs_sphinx = '1.0'
+# needs_sphinx = '1.0'
 
 # Add any Sphinx extension module names here, as strings. They can be
 # extensions coming with Sphinx (named 'sphinx.ext.*') or your custom
@@ -31,207 +31,201 @@ import os
 extensions = []
 
 # Add any paths that contain templates here, relative to this directory.
-templates_path = ['_templates']
+templates_path = ["_templates"]
 
 # The suffix of source filenames.
-source_suffix = '.rst'
+source_suffix = ".rst"
 
 # The encoding of source files.
-#source_encoding = 'utf-8-sig'
+# source_encoding = 'utf-8-sig'
 
 # The master toctree document.
-master_doc = 'index'
+master_doc = "index"
 
 # General information about the project.
-project = u'Pungi'
-copyright = u'2016, Red Hat, Inc.'
+project = "Pungi"
+copyright = "2016, Red Hat, Inc."
 
 # The version info for the project you're documenting, acts as replacement for
 # |version| and |release|, also used in various other places throughout the
 # built documents.
 #
 # The short X.Y version.
-version = '4.4'
+version = "4.7"
 # The full version, including alpha/beta/rc tags.
-release = '4.4.0'
+release = "4.7.0"
 
 # The language for content autogenerated by Sphinx. Refer to documentation
 # for a list of supported languages.
-#language = None
+# language = None
 
 # There are two options for replacing |today|: either, you set today to some
 # non-false value, then it is used:
-#today = ''
+# today = ''
 # Else, today_fmt is used as the format for a strftime call.
-#today_fmt = '%B %d, %Y'
+# today_fmt = '%B %d, %Y'
 
 # List of patterns, relative to source directory, that match files and
 # directories to ignore when looking for source files.
-exclude_patterns = ['_build']
+exclude_patterns = ["_build"]
 
 # The reST default role (used for this markup: `text`) to use for all
 # documents.
-#default_role = None
+# default_role = None
 
 # If true, '()' will be appended to :func: etc. cross-reference text.
-#add_function_parentheses = True
+# add_function_parentheses = True
 
 # If true, the current module name will be prepended to all description
 # unit titles (such as .. function::).
-#add_module_names = True
+# add_module_names = True
 
 # If true, sectionauthor and moduleauthor directives will be shown in the
 # output. They are ignored by default.
-#show_authors = False
+# show_authors = False
 
 # The name of the Pygments (syntax highlighting) style to use.
-pygments_style = 'sphinx'
+pygments_style = "sphinx"
 
 # A list of ignored prefixes for module index sorting.
-#modindex_common_prefix = []
+# modindex_common_prefix = []
 
 # If true, keep warnings as "system message" paragraphs in the built documents.
-#keep_warnings = False
+# keep_warnings = False
 
 
 # -- Options for HTML output ----------------------------------------------
 
 # The theme to use for HTML and HTML Help pages.  See the documentation for
 # a list of builtin themes.
-html_theme = 'default'
+html_theme = "default"
 
 # Theme options are theme-specific and customize the look and feel of a theme
 # further.  For a list of options available for each theme, see the
 # documentation.
-#html_theme_options = {}
+# html_theme_options = {}
 
 # Add any paths that contain custom themes here, relative to this directory.
-#html_theme_path = []
+# html_theme_path = []
 
 # The name for this set of Sphinx documents.  If None, it defaults to
 # "<project> v<release> documentation".
-#html_title = None
+# html_title = None
 
 # A shorter title for the navigation bar.  Default is the same as html_title.
-#html_short_title = None
+# html_short_title = None
 
 # The name of an image file (relative to this directory) to place at the top
 # of the sidebar.
-#html_logo = None
+# html_logo = None
 
 # The name of an image file (within the static path) to use as favicon of the
 # docs.  This file should be a Windows icon file (.ico) being 16x16 or 32x32
 # pixels large.
-#html_favicon = None
+# html_favicon = None
 
 # Add any paths that contain custom static files (such as style sheets) here,
 # relative to this directory. They are copied after the builtin static files,
 # so a file named "default.css" will overwrite the builtin "default.css".
-html_static_path = ['_static']
+html_static_path = ["_static"]
 
 # Add any extra paths that contain custom files (such as robots.txt or
 # .htaccess) here, relative to this directory. These files are copied
 # directly to the root of the documentation.
-#html_extra_path = []
+# html_extra_path = []
 
 # If not '', a 'Last updated on:' timestamp is inserted at every page bottom,
 # using the given strftime format.
-#html_last_updated_fmt = '%b %d, %Y'
+# html_last_updated_fmt = '%b %d, %Y'
 
 # If true, SmartyPants will be used to convert quotes and dashes to
 # typographically correct entities.
-#html_use_smartypants = True
+# html_use_smartypants = True
 
 # Custom sidebar templates, maps document names to template names.
-#html_sidebars = {}
+# html_sidebars = {}
 
 # Additional templates that should be rendered to pages, maps page names to
 # template names.
-#html_additional_pages = {}
+# html_additional_pages = {}
 
 # If false, no module index is generated.
-#html_domain_indices = True
+# html_domain_indices = True
 
 # If false, no index is generated.
-#html_use_index = True
+# html_use_index = True
 
 # If true, the index is split into individual pages for each letter.
-#html_split_index = False
+# html_split_index = False
 
 # If true, links to the reST sources are added to the pages.
-#html_show_sourcelink = True
+# html_show_sourcelink = True
 
 # If true, "Created using Sphinx" is shown in the HTML footer. Default is True.
-#html_show_sphinx = True
+# html_show_sphinx = True
 
 # If true, "(C) Copyright ..." is shown in the HTML footer. Default is True.
-#html_show_copyright = True
+# html_show_copyright = True
 
 # If true, an OpenSearch description file will be output, and all pages will
 # contain a <link> tag referring to it.  The value of this option must be the
 # base URL from which the finished HTML is served.
-#html_use_opensearch = ''
+# html_use_opensearch = ''
 
 # This is the file name suffix for HTML files (e.g. ".xhtml").
-#html_file_suffix = None
+# html_file_suffix = None
 
 # Output file base name for HTML help builder.
-htmlhelp_basename = 'Pungidoc'
+htmlhelp_basename = "Pungidoc"
 
 
 # -- Options for LaTeX output ---------------------------------------------
 
 latex_elements = {
-# The paper size ('letterpaper' or 'a4paper').
-#'papersize': 'letterpaper',
-
-# The font size ('10pt', '11pt' or '12pt').
-#'pointsize': '10pt',
-
-# Additional stuff for the LaTeX preamble.
-#'preamble': '',
+    # The paper size ('letterpaper' or 'a4paper').
+    #'papersize': 'letterpaper',
+    # The font size ('10pt', '11pt' or '12pt').
+    #'pointsize': '10pt',
+    # Additional stuff for the LaTeX preamble.
+    #'preamble': '',
 }
 
 # Grouping the document tree into LaTeX files. List of tuples
 # (source start file, target name, title,
 #  author, documentclass [howto, manual, or own class]).
 latex_documents = [
-  ('index', 'Pungi.tex', u'Pungi Documentation',
-   u'Daniel Mach', 'manual'),
+    ("index", "Pungi.tex", "Pungi Documentation", "Daniel Mach", "manual"),
 ]
 
 # The name of an image file (relative to this directory) to place at the top of
 # the title page.
-#latex_logo = None
+# latex_logo = None
 
 # For "manual" documents, if this is true, then toplevel headings are parts,
 # not chapters.
-#latex_use_parts = False
+# latex_use_parts = False
 
 # If true, show page references after internal links.
-#latex_show_pagerefs = False
+# latex_show_pagerefs = False
 
 # If true, show URL addresses after external links.
-#latex_show_urls = False
+# latex_show_urls = False
 
 # Documents to append as an appendix to all manuals.
-#latex_appendices = []
+# latex_appendices = []
 
 # If false, no module index is generated.
-#latex_domain_indices = True
+# latex_domain_indices = True
 
 
 # -- Options for manual page output ---------------------------------------
 
 # One entry per manual page. List of tuples
 # (source start file, name, description, authors, manual section).
-man_pages = [
-    ('index', 'pungi', u'Pungi Documentation',
-     [u'Daniel Mach'], 1)
-]
+man_pages = [("index", "pungi", "Pungi Documentation", ["Daniel Mach"], 1)]
 
 # If true, show URL addresses after external links.
-#man_show_urls = False
+# man_show_urls = False
 
 
 # -- Options for Texinfo output -------------------------------------------
@@ -240,19 +234,25 @@ man_pages = [
 # (source start file, target name, title, author,
 #  dir menu entry, description, category)
 texinfo_documents = [
-  ('index', 'Pungi', u'Pungi Documentation',
-   u'Daniel Mach', 'Pungi', 'One line description of project.',
-   'Miscellaneous'),
+    (
+        "index",
+        "Pungi",
+        "Pungi Documentation",
+        "Daniel Mach",
+        "Pungi",
+        "One line description of project.",
+        "Miscellaneous",
+    ),
 ]
 
 # Documents to append as an appendix to all manuals.
-#texinfo_appendices = []
+# texinfo_appendices = []
 
 # If false, no module index is generated.
-#texinfo_domain_indices = True
+# texinfo_domain_indices = True
 
 # How to display URL addresses: 'footnote', 'no', or 'inline'.
-#texinfo_show_urls = 'footnote'
+# texinfo_show_urls = 'footnote'
 
 # If true, do not generate a @detailmenu in the "Top" node's menu.
-#texinfo_no_detailmenu = False
+# texinfo_no_detailmenu = False

doc/configuration.rst

@@ -194,6 +194,17 @@ Options
     Tracking Service Kerberos authentication. If not defined, the default
     Kerberos principal is used.
 
+**cts_oidc_token_url**
+    (*str*) -- URL to the OIDC token endpoint.
+    For example ``https://oidc.example.com/openid-connect/token``.
+    This option can be overridden by the environment variable ``CTS_OIDC_TOKEN_URL``.
+
+**cts_oidc_client_id*
+    (*str*) -- OIDC client ID.
+    This option can be overridden by the environment variable ``CTS_OIDC_CLIENT_ID``.
+    Note that environment variable ``CTS_OIDC_CLIENT_SECRET`` must be configured with
+    corresponding client secret to authenticate to CTS via OIDC.
+
 **compose_type**
     (*str*) -- Allows to set default compose type. Type set via a command-line
     option overwrites this.
@@ -281,8 +292,8 @@ There a couple common format specifiers available for both the options:
     format string. The pattern should not overlap, otherwise it is undefined
     which one will be used.
 
-    This format will be used for all phases generating images. Currently that
-    means ``createiso``, ``live_images`` and ``buildinstall``.
+    This format will be used for some phases generating images. Currently that
+    means ``createiso``, ``buildinstall`` and ``ostree_installer``.
 
     Available extra keys are:
      * ``disc_num``
@@ -312,7 +323,6 @@ There a couple common format specifiers available for both the options:
 
     Available keys are:
      * ``boot`` -- for ``boot.iso`` images created in  *buildinstall* phase
-     * ``live`` -- for images created by *live_images* phase
      * ``dvd`` -- for images created by *createiso* phase
      * ``ostree`` -- for ostree installer images
 
@@ -340,48 +350,10 @@ Example
 
     disc_types = {
         'boot': 'netinst',
-        'live': 'Live',
         'dvd': 'DVD',
     }
 
 
-Signing
-=======
-
-If you want to sign deliverables generated during pungi run like RPM wrapped
-images. You must provide few configuration options:
-
-**signing_command** [optional]
-    (*str*) -- Command that will be run with a koji build as a single
-    argument. This command must not require any user interaction.
-    If you need to pass a password for a signing key to the command,
-    do this via command line option of the command and use string
-    formatting syntax ``%(signing_key_password)s``.
-    (See **signing_key_password_file**).
-
-**signing_key_id** [optional]
-    (*str*) -- ID of the key that will be used for the signing.
-    This ID will be used when crafting koji paths to signed files
-    (``kojipkgs.fedoraproject.org/packages/NAME/VER/REL/data/signed/KEYID/..``).
-
-**signing_key_password_file** [optional]
-    (*str*) -- Path to a file with password that will be formatted
-    into **signing_command** string via ``%(signing_key_password)s``
-    string format syntax (if used).
-    Because pungi config is usually stored in git and is part of compose
-    logs we don't want password to be included directly in the config.
-    Note: If ``-`` string is used instead of a filename, then you will be asked
-    for the password interactivelly right after pungi starts.
-
-Example
--------
-::
-
-        signing_command = '~/git/releng/scripts/sigulsign_unsigned.py -vv --password=%(signing_key_password)s fedora-24'
-        signing_key_id = '81b46521'
-        signing_key_password_file = '~/password_for_fedora-24_key'
-
-
 .. _git-urls:
 
 Git URLs
@@ -631,7 +603,7 @@ Options
 -------
 
 **buildinstall_method**
-    (*str*) -- "lorax" (f16+, rhel7+) or "buildinstall" (older releases)
+    (*str*) -- "lorax" (f16+, rhel7+)
 **lorax_options**
     (*list*) -- special options passed on to *lorax*.
 
@@ -930,6 +902,10 @@ Options
     comps file can not be found in the package set. When disabled (the
     default), such cases are still reported as warnings in the log.
 
+    With ``dnf`` gather backend, this option will abort the compose on any
+    missing package no matter if it's listed in comps, ``additional_packages``
+    or prepopulate file.
+
 **gather_source_mapping**
     (*str*) -- JSON mapping with initial packages for the compose. The value
     should be a path to JSON file with following mapping: ``{variant: {arch:
@@ -1353,8 +1329,8 @@ All non-``RC`` milestones from label get appended to the version. For release
 either label is used or date, type and respin.
 
 
-Common options for Live Images, Live Media and Image Build
-==========================================================
+Common options for Live Media and Image Build
+=============================================
 
 All images can have ``ksurl``, ``version``, ``release`` and ``target``
 specified. Since this can create a lot of duplication, there are global options
@@ -1370,14 +1346,12 @@ The kickstart URL is configured by these options.
  * ``global_ksurl`` -- global fallback setting
  * ``live_media_ksurl``
  * ``image_build_ksurl``
- * ``live_images_ksurl``
 
 Target is specified by these settings.
 
  * ``global_target`` -- global fallback setting
  * ``live_media_target``
  * ``image_build_target``
- * ``live_images_target``
  * ``osbuild_target``
 
 Version is specified by these options. If no version is set, a default value
@@ -1386,7 +1360,6 @@ will be provided according to :ref:`automatic versioning <auto-version>`.
  * ``global_version`` -- global fallback setting
  * ``live_media_version``
  * ``image_build_version``
- * ``live_images_version``
  * ``osbuild_version``
 
 Release is specified by these options. If set to a magic value to
@@ -1396,44 +1369,14 @@ to :ref:`automatic versioning <auto-version>`.
  * ``global_release`` -- global fallback setting
  * ``live_media_release``
  * ``image_build_release``
- * ``live_images_release``
  * ``osbuild_release``
 
-Each configuration block can also optionally specify a ``failable`` key. For
-live images it should have a boolean value. For live media and image build it
+Each configuration block can also optionally specify a ``failable`` key. It
 should be a list of strings containing architectures that are optional. If any
 deliverable fails on an optional architecture, it will not abort the whole
 compose. If the list contains only ``"*"``, all arches will be substituted.
 
 
-Live Images Settings
-====================
-
-**live_images**
-    (*list*) -- Configuration for the particular image. The elements of the
-    list should be tuples ``(variant_uid_regex, {arch|*: config})``. The config
-    should be a dict with these keys:
-
-      * ``kickstart`` (*str*)
-      * ``ksurl`` (*str*) [optional] -- where to get the kickstart from
-      * ``name`` (*str*)
-      * ``version`` (*str*)
-      * ``target`` (*str*)
-      * ``repo`` (*str|[str]*) -- repos specified by URL or variant UID
-      * ``specfile`` (*str*) -- for images wrapped in RPM
-      * ``scratch`` (*bool*) -- only RPM-wrapped images can use scratch builds,
-        but by default this is turned off
-      * ``type`` (*str*) -- what kind of task to start in Koji. Defaults to
-        ``live`` meaning ``koji spin-livecd`` will be used. Alternative option
-        is ``appliance`` corresponding to ``koji spin-appliance``.
-      * ``sign`` (*bool*) -- only RPM-wrapped images can be signed
-
-**live_images_no_rename**
-    (*bool*) -- When set to ``True``, filenames generated by Koji will be used.
-    When ``False``, filenames will be generated based on ``image_name_format``
-    configuration option.
-
-
 Live Media Settings
 ===================
 
@@ -1589,6 +1532,61 @@ Example
     }
 
 
+KiwiBuild Settings
+==================
+
+**kiwibuild**
+    (*dict*) -- configuration for building images using kiwi by a Koji plugin.
+    Pungi will trigger a Koji task delegating to kiwi, which will build the image,
+    import it to Koji via content generators.
+
+    Format: ``{variant_uid_regex: [{...}]}``.
+
+    Required keys in the configuration dict:
+
+    * ``kiwi_profile`` -- (*str*) select profile from description file.
+
+    Description scm, description path and target have to be provided too, but
+    instead of specifying them for each image separately, you can use the
+    ``kiwibuild_*`` options or ``global_target``.
+
+    Optional keys:
+
+    * ``description_scm`` -- (*str*) scm URL of description kiwi description.
+    * ``description_path`` -- (*str*) path to kiwi description inside the scm
+      repo.
+    * ``repos`` -- additional repos used to install RPMs in the image. The
+      compose repository for the enclosing variant is added automatically.
+      Either variant name or a URL is supported.
+    * ``target`` -- (*str*) which build target to use for the task. If not
+      provided, then either ``kiwibuild_target`` or ``global_target`` is
+      needed.
+    * ``release`` -- (*str*) release of the output image.
+    * ``arches`` -- (*[str]*) List of architectures to build for. If not
+      provided, all variant architectures will be built.
+    * ``failable`` -- (*[str]*) List of architectures for which this
+      deliverable is not release blocking.
+    * ``type`` -- (*str*) override default type from the bundle with this value.
+    * ``type_attr`` -- (*[str]*) override default attributes for the build type
+      from description.
+    * ``bundle_name_format`` -- (*str*) override default bundle format name.
+
+**kiwibuild_description_scm**
+    (*str*) -- URL for scm containing the description files
+
+**kiwibuild_description_path**
+    (*str*) -- path to a description file within the description scm
+
+**kiwibuild_type**
+    (*str*) -- override default type from the bundle with this value.
+
+**kiwibuild_type_attr**
+    (*[str]*) -- override default attributes for the build type from description.
+
+**kiwibuild_bundle_name_format**
+    (*str*) -- override default bundle format name.
+
+
 OSBuild Composer for building images
 ====================================
 
@@ -1637,11 +1635,17 @@ OSBuild Composer for building images
     * ``arches`` -- list of architectures for which to build the image. By
       default, the variant arches are used. This option can only restrict it,
       not add a new one.
+    * ``manifest_type`` -- the image type that is put into the manifest by
+      pungi. If not supplied then it is autodetected from the Koji output.
     * ``ostree_url`` -- URL of the repository that's used to fetch the parent
       commit from.
     * ``ostree_ref`` -- name of the ostree branch
     * ``ostree_parent`` -- commit hash or a a branch-like reference to the
       parent commit.
+    * ``customizations`` -- a dictionary with customizations to use for the
+      image build. For the list of supported customizations, see the **hosted**
+      variants in the `Image Builder documentation
+      <https://osbuild.org/docs/user-guide/blueprint-reference#installation-device>`.
     * ``upload_options`` -- a dictionary with upload options specific to the
       target cloud environment. If provided, the image will be uploaded to the
       cloud environment, in addition to the Koji server. One can't combine
@@ -1749,16 +1753,16 @@ another directory. Any new packages in the compose will be added to the
 repository with a new commit.
 
 **ostree**
-    (*dict*) -- a mapping of configuration for each. The format should be
-    ``{variant_uid_regex: config_dict}``. It is possible to use a list of
+    (*dict*) -- a mapping of configuration for each variant. The format should
+    be ``{variant_uid_regex: config_dict}``. It is possible to use a list of
     configuration dicts as well.
 
     The configuration dict for each variant arch pair must have these keys:
 
     * ``treefile`` -- (*str*) Filename of configuration for ``rpm-ostree``.
     * ``config_url`` -- (*str*) URL for Git repository with the ``treefile``.
-    * ``repo`` -- (*str|dict|[str|dict]*) repos specified by URL or variant UID
-      or a dict of repo options, ``baseurl`` is required in the dict.
+    * ``repo`` -- (*str|dict|[str|dict]*) repos specified by URL or a dict of
+      repo options, ``baseurl`` is required in the dict.
     * ``ostree_repo`` -- (*str*) Where to put the ostree repository
 
     These keys are optional:
@@ -1789,6 +1793,8 @@ repository with a new commit.
     * ``tag_ref`` -- (*bool*, default ``True``) If set to ``False``, a git
       reference will not be created.
     * ``ostree_ref`` -- (*str*) To override value ``ref`` from ``treefile``.
+    * ``runroot_packages`` -- (*list*) A list of additional package names to be
+      installed in the runroot environment in Koji.
 
 Example config
 --------------
@@ -1798,13 +1804,11 @@ Example config
         "^Atomic$": {
             "treefile": "fedora-atomic-docker-host.json",
             "config_url": "https://git.fedorahosted.org/git/fedora-atomic.git",
+            "keep_original_sources": True,
             "repo": [
-                "Server",
                 "http://example.com/repo/x86_64/os",
-                {"baseurl": "Everything"},
                 {"baseurl": "http://example.com/linux/repo", "exclude": "systemd-container"},
             ],
-            "keep_original_sources": True,
             "ostree_repo": "/mnt/koji/compose/atomic/Rawhide/",
             "update_summary": True,
             # Automatically generate a reasonable version
@@ -1820,6 +1824,79 @@ Example config
     has the pungi_ostree plugin installed.
 
 
+OSTree Native Container Settings
+================================
+
+The ``ostree_container`` phase of *Pungi* can create an ostree native container
+image as an OCI archive. This is done by running ``rpm-ostree compose image``
+in a Koji runroot environment.
+
+While rpm-ostree can use information from previously built images to improve
+the split in container layers, we can not use that functionnality until
+https://github.com/containers/skopeo/pull/2114 is resolved. Each invocation
+will thus create a new OCI archive image *from scratch*.
+
+**ostree_container**
+    (*dict*) -- a mapping of configuration for each variant. The format should
+    be ``{variant_uid_regex: config_dict}``. It is possible to use a list of
+    configuration dicts as well.
+
+    The configuration dict for each variant arch pair must have these keys:
+
+    * ``treefile`` -- (*str*) Filename of configuration for ``rpm-ostree``.
+    * ``config_url`` -- (*str*) URL for Git repository with the ``treefile``.
+
+    These keys are optional:
+
+    * ``repo`` -- (*str|dict|[str|dict]*) repos specified by URL or a dict of
+      repo options, ``baseurl`` is required in the dict.
+    * ``keep_original_sources`` -- (*bool*) Keep the existing source repos in
+      the tree config file. If not enabled, all the original source repos will
+      be removed from the tree config file.
+    * ``config_branch`` -- (*str*) Git branch of the repo to use. Defaults to
+      ``main``.
+    * ``arches`` -- (*[str]*) List of architectures for which to generate
+      ostree native container images. There will be one task per architecture.
+      By default all architectures in the variant are used.
+    * ``failable`` -- (*[str]*) List of architectures for which this
+      deliverable is not release blocking.
+    * ``version`` -- (*str*) Version string to be added to the OCI archive name.
+      If this option is set to ``!OSTREE_VERSION_FROM_LABEL_DATE_TYPE_RESPIN``,
+      a value will be generated automatically as ``$VERSION.$RELEASE``.
+      If this option is set to ``!VERSION_FROM_VERSION_DATE_RESPIN``,
+      a value will be generated automatically as ``$VERSION.$DATE.$RESPIN``.
+      :ref:`See how those values are created <auto-version>`.
+    * ``tag_ref`` -- (*bool*, default ``True``) If set to ``False``, a git
+      reference will not be created.
+    * ``runroot_packages`` -- (*list*) A list of additional package names to be
+      installed in the runroot environment in Koji.
+
+Example config
+--------------
+::
+
+    ostree_container = {
+        "^Sagano$": {
+            "treefile": "fedora-tier-0-38.yaml",
+            "config_url": "https://gitlab.com/CentOS/cloud/sagano.git",
+            "config_branch": "main",
+            "repo": [
+                "http://example.com/repo/x86_64/os",
+                {"baseurl": "http://example.com/linux/repo", "exclude": "systemd-container"},
+            ],
+            # Automatically generate a reasonable version
+            "version": "!OSTREE_VERSION_FROM_LABEL_DATE_TYPE_RESPIN",
+            # Only run this for x86_64 even if Sagano has more arches
+            "arches": ["x86_64"],
+        }
+    }
+
+**ostree_container_use_koji_plugin** = False
+    (*bool*) -- When set to ``True``, the Koji pungi_ostree task will be
+    used to execute rpm-ostree instead of runroot. Use only if the Koji instance
+    has the pungi_ostree plugin installed.
+
+
 Ostree Installer Settings
 =========================
 
@@ -2170,9 +2247,9 @@ Miscellaneous Settings
     format string accepting ``%(variant_name)s`` and ``%(arch)s`` placeholders.
 
 **symlink_isos_to**
-    (*str*) -- If set, the ISO files from ``buildinstall``, ``createiso`` and
-    ``live_images`` phases will be put into this destination, and a symlink
-    pointing to this location will be created in actual compose directory.
+    (*str*) -- If set, the ISO files from ``buildinstall`` and ``createiso``
+    phases will be put into this destination, and a symlink pointing to this
+    location will be created in actual compose directory.
 
 **dogpile_cache_backend**
     (*str*) -- If set, Pungi will use the configured Dogpile cache backend to

doc/examples.rst

@@ -294,30 +294,6 @@ This is a shortened configuration for Fedora Radhide compose as of 2019-10-14.
         })
     ]
 
-    live_target = 'f32'
-    live_images_no_rename = True
-    live_images = [
-        ('^Workstation$', {
-            'armhfp': {
-                'kickstart': 'fedora-arm-workstation.ks',
-                'name': 'Fedora-Workstation-armhfp',
-                # Again workstation takes packages from Everything.
-                'repo': 'Everything',
-                'type': 'appliance',
-                'failable': True,
-            }
-        }),
-        ('^Server$', {
-            # But Server has its own repo.
-            'armhfp': {
-                'kickstart': 'fedora-arm-server.ks',
-                'name': 'Fedora-Server-armhfp',
-                'type': 'appliance',
-                'failable': True,
-            }
-        }),
-    ]
-
     ostree = {
         "^Silverblue$": {
             "version": "!OSTREE_VERSION_FROM_LABEL_DATE_TYPE_RESPIN",
@@ -343,6 +319,20 @@ This is a shortened configuration for Fedora Radhide compose as of 2019-10-14.
         }
     }
 
+    ostree_container = {
+        "^Sagano$": {
+            "treefile": "fedora-tier-0-38.yaml",
+            "config_url": "https://gitlab.com/CentOS/cloud/sagano.git",
+            "config_branch": "main",
+            # Consume packages from Everything
+            "repo": "Everything",
+            # Automatically generate a reasonable version
+            "version": "!OSTREE_VERSION_FROM_LABEL_DATE_TYPE_RESPIN",
+            # Only run this for x86_64 even if Sagano has more arches
+            "arches": ["x86_64"],
+        }
+    }
+
     ostree_installer = [
         ("^Silverblue$", {
             "x86_64": {

doc/index.rst

@@ -19,6 +19,7 @@ Contents:
     scm_support
     messaging
     gathering
+    koji
     comps
     contributing
     testing

doc/koji.rst

@@ -0,0 +1,105 @@
+======================
+Getting data from koji
+======================
+
+When Pungi is configured to get packages from a Koji tag, it somehow needs to
+access the actual RPM files.
+
+Historically, this required the storage used by Koji to be directly available
+on the host where Pungi was running. This was usually achieved by using NFS for
+the Koji volume, and mounting it on the compose host.
+
+The compose could be created directly on the same volume. In such case the
+packages would be hardlinked, significantly reducing space consumption.
+
+The compose could also be created on a different storage, in which case the
+packages would either need to be copied over or symlinked. Using symlinks
+requires that anything that accesses the compose (e.g. a download server) would
+also need to mount the Koji volume in the same location.
+
+There is also a risk with symlinks that the package in Koji can change (due to
+being resigned for example), which would invalidate composes linking to it.
+
+
+Using Koji without direct mount
+===============================
+
+It is possible now to run a compose from a Koji tag without direct access to
+Koji storage.
+
+Pungi can download the packages over HTTP protocol, store them in a local
+cache, and consume them from there.
+
+The local cache has similar structure to what is on the Koji volume.
+
+When Pungi needs some package, it has a path on Koji volume. It will replace
+the ``topdir`` with the cache location. If such file exists, it will be used.
+If it doesn't exist, it will be downloaded from Koji (by replacing the
+``topdir`` with ``topurl``).
+
+::
+
+    Koji path                            /mnt/koji/packages/foo/1/1.fc38/data/signed/abcdef/noarch/foo-1-1.fc38.noarch.rpm
+    Koji URL    https://kojipkgs.fedoraproject.org/packages/foo/1/1.fc38/data/signed/abcdef/noarch/foo-1-1.fc38.noarch.rpm
+    Local path                  /mnt/compose/cache/packages/foo/1/1.fc38/data/signed/abcdef/noarch/foo-1-1.fc38.noarch.rpm
+
+The packages can be hardlinked from this cache directory.
+
+
+Cleanup
+-------
+
+While the approach above allows each RPM to be downloaded only once, it will
+eventually result in the Koji volume being mirrored locally. Most of the
+packages will however no longer be needed.
+
+There is a script ``pungi-cache-cleanup`` that can help with that. It can find
+and remove files from the cache that are no longer needed.
+
+A file is no longer needed if it has a single link (meaning it is only in the
+cache, not in any compose), and it has mtime older than a given threshold.
+
+It doesn't make sense to delete files that are hardlinked in an existing
+compose as it would not save any space anyway.
+
+The mtime check is meant to preserve files that are downloaded but not actually
+used in a compose, like a subpackage that is not included in any variant. Every
+time its existence in the local cache is checked, the mtime is updated.
+
+
+Race conditions?
+----------------
+
+It should be safe to have multiple compose hosts share the same storage volume
+for generated composes and local cache.
+
+If a cache file is accessed and it exists, there's no risk of race condition.
+
+If two composes need the same file at the same time and it is not present yet,
+one of them will take a lock on it and start downloading. The other will wait
+until the download is finished.
+
+The lock is only valid for a set amount of time (5 minutes) to avoid issues
+where the downloading process is killed in a way that blocks it from releasing
+the lock.
+
+If the file is large and network slow, the limit may not be enough finish
+downloading. In that case the second process will steal the lock while the
+first process is still downloading. This will result in the same file being
+downloaded twice.
+
+When the first process finishes the download, it will put the file into the
+local cache location. When the second process finishes, it will atomically
+replace it, but since it's the same file it will be the same file.
+
+If the first compose already managed to hardlink the file before it gets
+replaced, there will be two copies of the file present locally.
+
+
+Integrity checking
+------------------
+
+There is minimal integrity checking. RPM packages belonging to real builds will
+be check to match the checksum provided by Koji hub.
+
+There is no checking for scratch builds or any images.

doc/phases.rst

@@ -30,17 +30,14 @@ packages to architectures.
 Buildinstall
 ------------
 
-Spawns a bunch of threads, each of which runs either ``lorax`` or
-``buildinstall`` command (the latter coming from ``anaconda`` package). The
+Spawns a bunch of threads, each of which runs the ``lorax`` command. The
 commands create ``boot.iso`` and other boot configuration files. The image is
 finally linked into the ``compose/`` directory as netinstall media.
 
 The created images are also needed for creating live media or other images in
 later phases.
 
-With ``lorax`` this phase runs one task per variant.arch combination. For
-``buildinstall`` command there is only one task per architecture and
-``product.img`` should be used to customize the results.
+With ``lorax`` this phase runs one task per variant.arch combination.
 
 Gather
 ------
@@ -115,6 +112,12 @@ ImageBuild
 This phase wraps up ``koji image-build``. It also updates the metadata
 ultimately responsible for ``images.json`` manifest.
 
+KiwiBuild
+---------
+
+Similarly to image build, this phases creates a koji `kiwiBuild` task. In the
+background it uses Kiwi to create images.
+
 OSBuild
 -------
 

doc/scm_support.rst

@@ -41,6 +41,14 @@ which can contain following keys.
 * ``command`` -- defines a shell command to run after Git clone to generate the
   needed file (for example to run ``make``). Only supported in Git backend.
 
+* ``options`` -- a dictionary of additional configuration options. These are
+  specific to different backends.
+
+  Currently supported values for Git:
+
+  * ``credential_helper`` -- path to a credential helper used to supply
+    username/password for remotes that require authentication.
+
 
 Koji examples
 -------------

pungi/arch.py

@@ -93,6 +93,11 @@ def split_name_arch(name_arch):
 
 def is_excluded(package, arches, logger=None):
     """Check if package is excluded from given architectures."""
+    if any(
+        getBaseArch(exc_arch) == 'x86_64' for exc_arch in package.exclusivearch
+    ) and 'x86_64_v2' not in package.exclusivearch:
+        package.exclusivearch.append('x86_64_v2')
+
     if package.excludearch and set(package.excludearch) & set(arches):
         if logger:
             logger.debug(

pungi/arch_utils.py

@@ -34,6 +34,8 @@ arches = {
     "x86_64": "athlon",
     "amd64": "x86_64",
     "ia32e": "x86_64",
+    # x86-64-v2
+    "x86_64_v2": "noarch",
     # ppc64le
     "ppc64le": "noarch",
     # ppc

pungi/checks.py

@@ -387,6 +387,7 @@ def _extend_with_default_and_alias(validator_class, offline=False):
                 instance[property]["branch"] = resolver(
                     instance[property]["repo"],
                     instance[property].get("branch") or "HEAD",
+                    instance[property].get("options"),
                 )
 
         for error in _hook_errors(properties, instance, schema):
@@ -520,6 +521,13 @@ def make_schema():
                     "file": {"type": "string"},
                     "dir": {"type": "string"},
                     "command": {"type": "string"},
+                    "options": {
+                        "type": "object",
+                        "properties": {
+                            "credential_helper": {"type": "string"},
+                        },
+                        "additionalProperties": False,
+                    },
                 },
                 "additionalProperties": False,
             },
@@ -545,27 +553,6 @@ def make_schema():
             "list_of_strings": {"type": "array", "items": {"type": "string"}},
             "strings": _one_or_list({"type": "string"}),
             "optional_string": {"anyOf": [{"type": "string"}, {"type": "null"}]},
-            "live_image_config": {
-                "type": "object",
-                "properties": {
-                    "kickstart": {"type": "string"},
-                    "ksurl": {"type": "url"},
-                    "name": {"type": "string"},
-                    "subvariant": {"type": "string"},
-                    "target": {"type": "string"},
-                    "version": {"type": "string"},
-                    "repo": {"$ref": "#/definitions/repos"},
-                    "specfile": {"type": "string"},
-                    "scratch": {"type": "boolean"},
-                    "type": {"type": "string"},
-                    "sign": {"type": "boolean"},
-                    "failable": {"type": "boolean"},
-                    "release": {"$ref": "#/definitions/optional_string"},
-                },
-                "required": ["kickstart"],
-                "additionalProperties": False,
-                "type": "object",
-            },
             "osbs_config": {
                 "type": "object",
                 "properties": {
@@ -601,6 +588,7 @@ def make_schema():
             "release_discinfo_description": {"type": "string"},
             "treeinfo_version": {"type": "string"},
             "compose_type": {"type": "string", "enum": COMPOSE_TYPES},
+            "label": {"type": "string"},
             "base_product_name": {"type": "string"},
             "base_product_short": {"type": "string"},
             "base_product_version": {"type": "string"},
@@ -678,7 +666,11 @@ def make_schema():
             "pkgset_allow_reuse": {"type": "boolean", "default": True},
             "createiso_allow_reuse": {"type": "boolean", "default": True},
             "extraiso_allow_reuse": {"type": "boolean", "default": True},
-            "pkgset_source": {"type": "string", "enum": ["koji", "repos"]},
+            "pkgset_source": {"type": "string", "enum": [
+                "koji",
+                "repos",
+                "kojimock",
+            ]},
             "createrepo_c": {"type": "boolean", "default": True},
             "createrepo_checksum": {
                 "type": "string",
@@ -805,7 +797,15 @@ def make_schema():
             "buildinstall_allow_reuse": {"type": "boolean", "default": False},
             "buildinstall_method": {
                 "type": "string",
-                "enum": ["lorax", "buildinstall"],
+                "enum": ["lorax"],
+            },
+            # In phase `buildinstall` we should add to compose only the
+            # images that will be used only as netinstall
+            "netinstall_variants": {
+                "$ref": "#/definitions/list_of_strings",
+                "default": [
+                    "BaseOS",
+                ],
             },
             "buildinstall_topdir": {"type": "string"},
             "buildinstall_kickstart": {"$ref": "#/definitions/str_or_scm_dict"},
@@ -824,8 +824,11 @@ def make_schema():
             "pdc_insecure": {"deprecated": "Koji is queried instead"},
             "cts_url": {"type": "string"},
             "cts_keytab": {"type": "string"},
+            "cts_oidc_token_url": {"type": "url"},
+            "cts_oidc_client_id": {"type": "string"},
             "koji_profile": {"type": "string"},
             "koji_event": {"type": "number"},
+            "koji_cache": {"type": "string"},
             "pkgset_koji_tag": {"$ref": "#/definitions/strings"},
             "pkgset_koji_builds": {"$ref": "#/definitions/strings"},
             "pkgset_koji_scratch_tasks": {"$ref": "#/definitions/strings"},
@@ -859,7 +862,10 @@ def make_schema():
             "paths_module": {"type": "string"},
             "skip_phases": {
                 "type": "array",
-                "items": {"type": "string", "enum": PHASES_NAMES + ["productimg"]},
+                "items": {
+                    "type": "string",
+                    "enum": PHASES_NAMES + ["productimg", "live_images"],
+                },
                 "default": [],
             },
             "image_name_format": {
@@ -893,11 +899,6 @@ def make_schema():
             },
             "restricted_volid": {"type": "boolean", "default": False},
             "volume_id_substitutions": {"type": "object", "default": {}},
-            "live_images_no_rename": {"type": "boolean", "default": False},
-            "live_images_ksurl": {"type": "url"},
-            "live_images_target": {"type": "string"},
-            "live_images_release": {"$ref": "#/definitions/optional_string"},
-            "live_images_version": {"type": "string"},
             "image_build_ksurl": {"type": "url"},
             "image_build_target": {"type": "string"},
             "image_build_release": {"$ref": "#/definitions/optional_string"},
@@ -930,8 +931,6 @@ def make_schema():
             "product_id": {"$ref": "#/definitions/str_or_scm_dict"},
             "product_id_allow_missing": {"type": "boolean", "default": False},
             "product_id_allow_name_prefix": {"type": "boolean", "default": True},
-            # Deprecated in favour of regular local/phase/global setting.
-            "live_target": {"type": "string"},
             "tree_arches": {"$ref": "#/definitions/list_of_strings", "default": []},
             "tree_variants": {"$ref": "#/definitions/list_of_strings", "default": []},
             "translate_paths": {"$ref": "#/definitions/string_pairs", "default": []},
@@ -1049,11 +1048,13 @@ def make_schema():
                                         "config_branch": {"type": "string"},
                                         "tag_ref": {"type": "boolean"},
                                         "ostree_ref": {"type": "string"},
+                                        "runroot_packages": {
+                                            "$ref": "#/definitions/list_of_strings",
+                                        },
                                     },
                                     "required": [
                                         "treefile",
                                         "config_url",
-                                        "repo",
                                         "ostree_repo",
                                     ],
                                     "additionalProperties": False,
@@ -1091,6 +1092,39 @@ def make_schema():
                     ),
                 ]
             },
+            "ostree_container": {
+                "type": "object",
+                "patternProperties": {
+                    # Warning: this pattern is a variant uid regex, but the
+                    # format does not let us validate it as there is no regular
+                    # expression to describe all regular expressions.
+                    ".+": _one_or_list(
+                        {
+                            "type": "object",
+                            "properties": {
+                                "treefile": {"type": "string"},
+                                "config_url": {"type": "string"},
+                                "repo": {"$ref": "#/definitions/repos"},
+                                "keep_original_sources": {"type": "boolean"},
+                                "config_branch": {"type": "string"},
+                                "arches": {"$ref": "#/definitions/list_of_strings"},
+                                "failable": {"$ref": "#/definitions/list_of_strings"},
+                                "version": {"type": "string"},
+                                "tag_ref": {"type": "boolean"},
+                                "runroot_packages": {
+                                    "$ref": "#/definitions/list_of_strings",
+                                },
+                            },
+                            "required": [
+                                "treefile",
+                                "config_url",
+                            ],
+                            "additionalProperties": False,
+                        }
+                    ),
+                },
+                "additionalProperties": False,
+            },
             "ostree_installer": _variant_arch_mapping(
                 {
                     "type": "object",
@@ -1115,11 +1149,9 @@ def make_schema():
                 }
             ),
             "ostree_use_koji_plugin": {"type": "boolean", "default": False},
+            "ostree_container_use_koji_plugin": {"type": "boolean", "default": False},
             "ostree_installer_use_koji_plugin": {"type": "boolean", "default": False},
             "ostree_installer_overwrite": {"type": "boolean", "default": False},
-            "live_images": _variant_arch_mapping(
-                _one_or_list({"$ref": "#/definitions/live_image_config"})
-            ),
             "image_build_allow_reuse": {"type": "boolean", "default": False},
             "image_build": {
                 "type": "object",
@@ -1170,6 +1202,50 @@ def make_schema():
                 },
                 "additionalProperties": False,
             },
+            "kiwibuild": {
+                "type": "object",
+                "patternProperties": {
+                    # Warning: this pattern is a variant uid regex, but the
+                    # format does not let us validate it as there is no regular
+                    # expression to describe all regular expressions.
+                    ".+": {
+                        "type": "array",
+                        "items": {
+                            "type": "object",
+                            "properties": {
+                                "target": {"type": "string"},
+                                "description_scm": {"type": "url"},
+                                "description_path": {"type": "string"},
+                                "kiwi_profile": {"type": "string"},
+                                "release": {"type": "string"},
+                                "arches": {"$ref": "#/definitions/list_of_strings"},
+                                "repos": {"$ref": "#/definitions/list_of_strings"},
+                                "failable": {"$ref": "#/definitions/list_of_strings"},
+                                "subvariant": {"type": "string"},
+                                "type": {"type": "string"},
+                                "type_attr": {"$ref": "#/definitions/list_of_strings"},
+                                "bundle_name_format": {"type": "string"},
+                            },
+                            "required": [
+                                # description_scm and description_path
+                                # are really required, but as they can
+                                # be set at the phase level we cannot
+                                # enforce that here
+                                "kiwi_profile",
+                            ],
+                            "additionalProperties": False,
+                        },
+                    }
+                },
+                "additionalProperties": False,
+            },
+            "kiwibuild_description_scm": {"type": "url"},
+            "kiwibuild_description_path": {"type": "string"},
+            "kiwibuild_target": {"type": "string"},
+            "kiwibuild_release": {"$ref": "#/definitions/optional_string"},
+            "kiwibuild_type": {"type": "string"},
+            "kiwibuild_type_attr": {"$ref": "#/definitions/list_of_strings"},
+            "kiwibuild_bundle_name_format": {"type": "string"},
             "osbuild_target": {"type": "string"},
             "osbuild_release": {"$ref": "#/definitions/optional_string"},
             "osbuild_version": {"type": "string"},
@@ -1230,6 +1306,11 @@ def make_schema():
                                 "ostree_url": {"type": "string"},
                                 "ostree_ref": {"type": "string"},
                                 "ostree_parent": {"type": "string"},
+                                "manifest_type": {"type": "string"},
+                                "customizations": {
+                                    "type": "object",
+                                    "additionalProperties": True,
+                                },
                                 "upload_options": {
                                     # this should be really 'oneOf', but the minimal
                                     # required properties in AWSEC2 and GCP options
@@ -1346,9 +1427,6 @@ def make_schema():
                 {"$ref": "#/definitions/strings"}
             ),
             "lorax_use_koji_plugin": {"type": "boolean", "default": False},
-            "signing_key_id": {"type": "string"},
-            "signing_key_password_file": {"type": "string"},
-            "signing_command": {"type": "string"},
             "productimg": {
                 "deprecated": "remove it. Productimg phase has been removed"
             },
@@ -1483,7 +1561,6 @@ def get_num_cpus():
 CONFIG_DEPS = {
     "buildinstall_method": {
         "conflicts": (
-            (lambda val: val == "buildinstall", ["lorax_options"]),
             (lambda val: not val, ["lorax_options", "buildinstall_kickstart"]),
         ),
     },

pungi/compose.py

@@ -39,6 +39,7 @@ from dogpile.cache import make_region
 from pungi.graph import SimpleAcyclicOrientedGraph
 from pungi.wrappers.variants import VariantsXmlParser
 from pungi.paths import Paths
+from pungi.wrappers.kojiwrapper import KojiDownloadProxy
 from pungi.wrappers.scm import get_file_from_scm
 from pungi.util import (
     makedirs,
@@ -70,39 +71,82 @@ def is_status_fatal(status_code):
 
 
 @retry(wait_on=RequestException)
-def retry_request(method, url, data=None, auth=None):
+def retry_request(method, url, data=None, json_data=None, auth=None):
+    """
+    :param str method: Reqest method.
+    :param str url: Target URL.
+    :param dict data: form-urlencoded data to send in the body of the request.
+    :param dict json_data: json data to send in the body of the request.
+    """
     request_method = getattr(requests, method)
-    rv = request_method(url, json=data, auth=auth)
+    rv = request_method(url, data=data, json=json_data, auth=auth)
     if is_status_fatal(rv.status_code):
         try:
-            error = rv.json()["message"]
+            error = rv.json()
         except ValueError:
             error = rv.text
-        raise RuntimeError("CTS responded with %d: %s" % (rv.status_code, error))
+        raise RuntimeError("%s responded with %d: %s" % (url, rv.status_code, error))
     rv.raise_for_status()
     return rv
 
 
-@contextlib.contextmanager
-def cts_auth(cts_keytab):
-    auth = None
-    if cts_keytab:
-        # requests-kerberos cannot accept custom keytab, we need to use
-        # environment variable for this. But we need to change environment
-        # only temporarily just for this single requests.post.
-        # So at first backup the current environment and revert to it
-        # after the requests call.
-        from requests_kerberos import HTTPKerberosAuth
+class BearerAuth(requests.auth.AuthBase):
+    def __init__(self, token):
+        self.token = token
 
-        auth = HTTPKerberosAuth()
-        environ_copy = dict(os.environ)
-        if "$HOSTNAME" in cts_keytab:
-            cts_keytab = cts_keytab.replace("$HOSTNAME", socket.gethostname())
-        os.environ["KRB5_CLIENT_KTNAME"] = cts_keytab
-        os.environ["KRB5CCNAME"] = "DIR:%s" % tempfile.mkdtemp()
+    def __call__(self, r):
+        r.headers["authorization"] = "Bearer " + self.token
+        return r
+
+
+@contextlib.contextmanager
+def cts_auth(pungi_conf):
+    """
+    :param dict pungi_conf: dict obj of pungi.json config.
+    """
+    auth = None
+    token = None
+    cts_keytab = pungi_conf.get("cts_keytab")
+    cts_oidc_token_url = os.environ.get("CTS_OIDC_TOKEN_URL", "") or pungi_conf.get(
+        "cts_oidc_token_url"
+    )
 
     try:
+        if cts_keytab:
+            # requests-kerberos cannot accept custom keytab, we need to use
+            # environment variable for this. But we need to change environment
+            # only temporarily just for this single requests.post.
+            # So at first backup the current environment and revert to it
+            # after the requests call.
+            from requests_kerberos import HTTPKerberosAuth
+
+            auth = HTTPKerberosAuth()
+            environ_copy = dict(os.environ)
+            if "$HOSTNAME" in cts_keytab:
+                cts_keytab = cts_keytab.replace("$HOSTNAME", socket.gethostname())
+            os.environ["KRB5_CLIENT_KTNAME"] = cts_keytab
+            os.environ["KRB5CCNAME"] = "DIR:%s" % tempfile.mkdtemp()
+        elif cts_oidc_token_url:
+            cts_oidc_client_id = os.environ.get(
+                "CTS_OIDC_CLIENT_ID", ""
+            ) or pungi_conf.get("cts_oidc_client_id", "")
+            token = retry_request(
+                "post",
+                cts_oidc_token_url,
+                data={
+                    "grant_type": "client_credentials",
+                    "client_id": cts_oidc_client_id,
+                    "client_secret": os.environ.get("CTS_OIDC_CLIENT_SECRET", ""),
+                },
+            ).json()["access_token"]
+            auth = BearerAuth(token)
+            del token
+
         yield auth
+    except Exception as e:
+        # Avoid leaking client secret in trackback
+        e.show_locals = False
+        raise e
     finally:
         if cts_keytab:
             shutil.rmtree(os.environ["KRB5CCNAME"].split(":", 1)[1])
@@ -150,8 +194,8 @@ def get_compose_info(
             "parent_compose_ids": parent_compose_ids,
             "respin_of": respin_of,
         }
-        with cts_auth(conf.get("cts_keytab")) as authentication:
-            rv = retry_request("post", url, data=data, auth=authentication)
+        with cts_auth(conf) as authentication:
+            rv = retry_request("post", url, json_data=data, auth=authentication)
 
         # Update local ComposeInfo with received ComposeInfo.
         cts_ci = ComposeInfo()
@@ -187,8 +231,8 @@ def update_compose_url(compose_id, compose_dir, conf):
             "action": "set_url",
             "compose_url": compose_url,
         }
-        with cts_auth(conf.get("cts_keytab")) as authentication:
-            return retry_request("patch", url, data=data, auth=authentication)
+        with cts_auth(conf) as authentication:
+            return retry_request("patch", url, json_data=data, auth=authentication)
 
 
 def get_compose_dir(
@@ -366,6 +410,8 @@ class Compose(kobo.log.LoggingBase):
         else:
             self.cache_region = make_region().configure("dogpile.cache.null")
 
+        self.koji_downloader = KojiDownloadProxy.from_config(self.conf, self._logger)
+
     get_compose_info = staticmethod(get_compose_info)
     write_compose_info = staticmethod(write_compose_info)
     get_compose_dir = staticmethod(get_compose_dir)
@@ -661,7 +707,7 @@ class Compose(kobo.log.LoggingBase):
                 separators=(",", ": "),
             )
 
-    def traceback(self, detail=None):
+    def traceback(self, detail=None, show_locals=True):
         """Store an extended traceback. This method should only be called when
         handling an exception.
 
@@ -672,8 +718,10 @@ class Compose(kobo.log.LoggingBase):
             basename += "-" + detail
         tb_path = self.paths.log.log_file("global", basename)
         self.log_error("Extended traceback in: %s", tb_path)
-        with open(tb_path, "wb") as f:
-            f.write(kobo.tback.Traceback().get_traceback())
+        tback = kobo.tback.Traceback(show_locals=show_locals).get_traceback()
+        # Kobo 0.36.0 returns traceback as str, older versions return bytes
+        with open(tb_path, "wb" if isinstance(tback, bytes) else "w") as f:
+            f.write(tback)
 
     def load_old_compose_config(self):
         """

pungi/createiso.py

@@ -67,10 +67,6 @@ def make_image(f, opts):
                 os.path.join("$TEMPLATE", "config_files/ppc"),
                 hfs_compat=opts.hfs_compat,
             )
-        elif opts.buildinstall_method == "buildinstall":
-            mkisofs_kwargs["boot_args"] = iso.get_boot_options(
-                opts.arch, "/usr/lib/anaconda-runtime/boot"
-            )
 
     # ppc(64) doesn't seem to support utf-8
     if opts.arch in ("ppc", "ppc64", "ppc64le"):
@@ -163,15 +159,11 @@ def write_xorriso_commands(opts):
 
     script = os.path.join(opts.script_dir, "xorriso-%s.txt" % id(opts))
     with open(script, "w") as f:
-        emit(f, "-indev %s" % opts.boot_iso)
-        emit(f, "-outdev %s" % os.path.join(opts.output_dir, opts.iso_name))
-        emit(f, "-boot_image any replay")
+        for cmd in iso.xorriso_commands(
+            opts.arch, opts.boot_iso, os.path.join(opts.output_dir, opts.iso_name)
+        ):
+            emit(f, " ".join(cmd))
         emit(f, "-volid %s" % opts.volid)
-        # isoinfo -J uses the Joliet tree, and it's used by virt-install
-        emit(f, "-joliet on")
-        # Support long filenames in the Joliet trees. Repodata is particularly
-        # likely to run into this limit.
-        emit(f, "-compliance joliet_long_names")
 
         with open(opts.graft_points) as gp:
             for line in gp:
@@ -182,10 +174,6 @@ def write_xorriso_commands(opts):
                 emit(f, "%s %s %s" % (cmd, fs_path, iso_path))
                 emit(f, "-chmod 0%o %s" % (_get_perms(fs_path), iso_path))
 
-        if opts.arch == "ppc64le":
-            # This is needed for the image to be bootable.
-            emit(f, "-as mkisofs -U --")
-
         emit(f, "-chown_r 0 /")
         emit(f, "-chgrp_r 0 /")
         emit(f, "-end")

pungi/gather_dnf.py

@@ -17,6 +17,7 @@
 from enum import Enum
 from functools import cmp_to_key
 from itertools import count, groupby
+import errno
 import logging
 import os
 import re
@@ -39,6 +40,20 @@ def get_source_name(pkg):
     return pkg.sourcerpm.rsplit("-", 2)[0]
 
 
+def filter_dotarch(queue, pattern, **kwargs):
+    """Filter queue for packages matching the pattern. If pattern matches the
+    dotarch format of <name>.<arch>, it is processed as such. Otherwise it is
+    treated as just a name.
+    """
+    kwargs["name__glob"] = pattern
+    if "." in pattern:
+        name, arch = pattern.split(".", 1)
+        if arch in arch_utils.arches or arch == "noarch":
+            kwargs["name__glob"] = name
+            kwargs["arch"] = arch
+    return queue.filter(**kwargs).apply()
+
+
 class GatherOptions(pungi.common.OptionsBase):
     def __init__(self, **kwargs):
         super(GatherOptions, self).__init__()
@@ -450,12 +465,16 @@ class Gather(GatherBase):
                         name__glob=pattern[:-4], reponame__neq=self.opts.lookaside_repos
                     )
                 elif pungi.util.pkg_is_debug(pattern):
-                    pkgs = self.q_debug_packages.filter(
-                        name__glob=pattern, reponame__neq=self.opts.lookaside_repos
+                    pkgs = filter_dotarch(
+                        self.q_debug_packages,
+                        pattern,
+                        reponame__neq=self.opts.lookaside_repos,
                     )
                 else:
-                    pkgs = self.q_binary_packages.filter(
-                        name__glob=pattern, reponame__neq=self.opts.lookaside_repos
+                    pkgs = filter_dotarch(
+                        self.q_binary_packages,
+                        pattern,
+                        reponame__neq=self.opts.lookaside_repos,
                     )
 
                 exclude.update(pkgs)
@@ -521,28 +540,19 @@ class Gather(GatherBase):
                                 name__glob=pattern[:-2]
                             ).apply()
                         else:
-                            pkgs = self.q_debug_packages.filter(
-                                name__glob=pattern
-                            ).apply()
+                            pkgs = filter_dotarch(self.q_debug_packages, pattern)
                     else:
                         if pattern.endswith(".+"):
                             pkgs = self.q_multilib_binary_packages.filter(
                                 name__glob=pattern[:-2]
                             ).apply()
                         else:
-                            kwargs = {"name__glob": pattern}
-                            if "." in pattern:
-                                # The pattern could be name.arch. Check if the
-                                # arch is valid, and if yes, make a more
-                                # specific query.
-                                name, arch = pattern.split(".", 1)
-                                if arch in arch_utils.arches:
-                                    kwargs["name__glob"] = name
-                                    kwargs["arch__eq"] = arch
-                            pkgs = self.q_binary_packages.filter(**kwargs).apply()
+                            pkgs = filter_dotarch(self.q_binary_packages, pattern)
 
                 if not pkgs:
-                    self.logger.error("No package matches pattern %s" % pattern)
+                    self.logger.error(
+                        "Could not find a match for %s in any configured repo", pattern
+                    )
 
                 # The pattern could have been a glob. In that case we want to
                 # group the packages by name and get best match in those
@@ -829,6 +839,12 @@ class Gather(GatherBase):
                     continue
                 if self.is_from_lookaside(i):
                     self._set_flag(i, PkgFlag.lookaside)
+                srpm_name = i.sourcerpm.rsplit("-", 2)[0]
+                if srpm_name in self.opts.fulltree_excludes:
+                    self._set_flag(i, PkgFlag.fulltree_exclude)
+                if PkgFlag.input in self.result_package_flags.get(srpm_name, set()):
+                    # If src rpm is marked as input, mark debuginfo as input too
+                    self._set_flag(i, PkgFlag.input)
                 if i not in self.result_debug_packages:
                     added.add(i)
                     debug_pkgs.append(i)
@@ -1065,9 +1081,12 @@ class Gather(GatherBase):
             # Link downloaded package in (or link package from file repo)
             try:
                 linker.link(pkg.localPkg(), target)
-            except Exception:
-                self.logger.error("Unable to link %s from the yum cache." % pkg.name)
-                raise
+            except Exception as ex:
+                if ex.errno == errno.EEXIST:
+                    self.logger.warning("Downloaded package exists in %s", target)
+                else:
+                    self.logger.error("Unable to link %s from the yum cache.", pkg.name)
+                    raise
 
     def log_count(self, msg, method, *args):
         """

pungi/ostree/__init__.py

@@ -19,6 +19,7 @@ import logging
 
 from .tree import Tree
 from .installer import Installer
+from .container import Container
 
 
 def main(args=None):
@@ -71,6 +72,43 @@ def main(args=None):
         help="use unified core mode in rpm-ostree",
     )
 
+    container = subparser.add_parser(
+        "container", help="Compose OSTree native container"
+    )
+    container.set_defaults(_class=Container, func="run")
+    container.add_argument(
+        "--name",
+        required=True,
+        help="the name of the the OCI archive (required)",
+    )
+    container.add_argument(
+        "--path",
+        required=True,
+        help="where to output the OCI archive (required)",
+    )
+    container.add_argument(
+        "--treefile",
+        metavar="FILE",
+        required=True,
+        help="treefile for rpm-ostree (required)",
+    )
+    container.add_argument(
+        "--log-dir",
+        metavar="DIR",
+        required=True,
+        help="where to log output (required).",
+    )
+    container.add_argument(
+        "--extra-config", metavar="FILE", help="JSON file contains extra configurations"
+    )
+    container.add_argument(
+        "-v",
+        "--version",
+        metavar="VERSION",
+        required=True,
+        help="version identifier (required)",
+    )
+
     installerp = subparser.add_parser(
         "installer", help="Create an OSTree installer image"
     )

pungi/ostree/container.py

@@ -0,0 +1,86 @@
+# -*- coding: utf-8 -*-
+
+
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; version 2 of the License.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Library General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, see <https://gnu.org/licenses/>.
+
+
+import os
+import json
+import six
+from six.moves import shlex_quote
+
+
+from .base import OSTree
+from .utils import tweak_treeconf
+
+
+def emit(cmd):
+    """Print line of shell code into the stream."""
+    if isinstance(cmd, six.string_types):
+        print(cmd)
+    else:
+        print(" ".join([shlex_quote(x) for x in cmd]))
+
+
+class Container(OSTree):
+    def _make_container(self):
+        """Compose OSTree Container Native image"""
+        stamp_file = os.path.join(self.logdir, "%s.stamp" % self.name)
+        cmd = [
+            "rpm-ostree",
+            "compose",
+            "image",
+            # Always initialize for now
+            "--initialize",
+            # Touch the file if a new commit was created. This can help us tell
+            # if the commitid file is missing because no commit was created or
+            # because something went wrong.
+            "--touch-if-changed=%s" % stamp_file,
+            self.treefile,
+        ]
+        fullpath = os.path.join(self.path, "%s.ociarchive" % self.name)
+        cmd.append(fullpath)
+
+        # Set the umask to be more permissive so directories get group write
+        # permissions. See https://pagure.io/releng/issue/8811#comment-629051
+        emit("umask 0002")
+        emit(cmd)
+
+    def run(self):
+        self.name = self.args.name
+        self.path = self.args.path
+        self.treefile = self.args.treefile
+        self.logdir = self.args.log_dir
+        self.extra_config = self.args.extra_config
+
+        if self.extra_config:
+            self.extra_config = json.load(open(self.extra_config, "r"))
+            repos = self.extra_config.get("repo", [])
+            keep_original_sources = self.extra_config.get(
+                "keep_original_sources", False
+            )
+        else:
+            # missing extra_config mustn't affect tweak_treeconf call
+            repos = []
+            keep_original_sources = True
+
+        update_dict = {"automatic-version-prefix": self.args.version}
+
+        self.treefile = tweak_treeconf(
+            self.treefile,
+            source_repos=repos,
+            keep_original_sources=keep_original_sources,
+            update_dict=update_dict,
+        )
+
+        self._make_container()

pungi/phases/__init__.py

@@ -25,9 +25,9 @@ from .buildinstall import BuildinstallPhase  # noqa
 from .extra_files import ExtraFilesPhase  # noqa
 from .createiso import CreateisoPhase  # noqa
 from .extra_isos import ExtraIsosPhase  # noqa
-from .live_images import LiveImagesPhase  # noqa
 from .image_build import ImageBuildPhase  # noqa
 from .image_container import ImageContainerPhase  # noqa
+from .kiwibuild import KiwiBuildPhase  # noqa
 from .osbuild import OSBuildPhase  # noqa
 from .repoclosure import RepoclosurePhase  # noqa
 from .test import TestPhase  # noqa
@@ -35,6 +35,7 @@ from .image_checksum import ImageChecksumPhase  # noqa
 from .livemedia_phase import LiveMediaPhase  # noqa
 from .ostree import OSTreePhase  # noqa
 from .ostree_installer import OstreeInstallerPhase  # noqa
+from .ostree_container import OSTreeContainerPhase  # noqa
 from .osbs import OSBSPhase  # noqa
 from .phases_metadata import gather_phases_metadata  # noqa
 

pungi/phases/buildinstall.py

@@ -31,14 +31,14 @@ from six.moves import shlex_quote
 from pungi.arch import get_valid_arches
 from pungi.util import get_volid, get_arch_variant_data
 from pungi.util import get_file_size, get_mtime, failable, makedirs
-from pungi.util import copy_all, translate_path, move_all
+from pungi.util import copy_all, translate_path
 from pungi.wrappers.lorax import LoraxWrapper
 from pungi.wrappers import iso
 from pungi.wrappers.scm import get_file
 from pungi.wrappers.scm import get_file_from_scm
 from pungi.wrappers import kojiwrapper
 from pungi.phases.base import PhaseBase
-from pungi.runroot import Runroot
+from pungi.runroot import Runroot, download_and_extract_archive
 
 
 class BuildinstallPhase(PhaseBase):
@@ -144,7 +144,7 @@ class BuildinstallPhase(PhaseBase):
         )
         if self.compose.has_comps:
             comps_repo = self.compose.paths.work.comps_repo(arch, variant)
-            if final_output_dir != output_dir:
+            if final_output_dir != output_dir or self.lorax_use_koji_plugin:
                 comps_repo = translate_path(self.compose, comps_repo)
             repos.append(comps_repo)
 
@@ -169,7 +169,6 @@ class BuildinstallPhase(PhaseBase):
                 "rootfs-size": rootfs_size,
                 "dracut-args": dracut_args,
                 "skip_branding": skip_branding,
-                "outputdir": output_dir,
                 "squashfs_only": squashfs_only,
                 "configuration_file": configuration_file,
             }
@@ -219,10 +218,6 @@ class BuildinstallPhase(PhaseBase):
         return repos
 
     def run(self):
-        lorax = LoraxWrapper()
-        product = self.compose.conf["release_name"]
-        version = self.compose.conf["release_version"]
-        release = self.compose.conf["release_version"]
         disc_type = self.compose.conf["disc_types"].get("dvd", "dvd")
 
         # Prepare kickstart file for final images.
@@ -239,7 +234,7 @@ class BuildinstallPhase(PhaseBase):
             )
             makedirs(final_output_dir)
             repo_baseurls = self.get_repos(arch)
-            if final_output_dir != output_dir:
+            if final_output_dir != output_dir or self.lorax_use_koji_plugin:
                 repo_baseurls = [translate_path(self.compose, r) for r in repo_baseurls]
 
             if self.buildinstall_method == "lorax":
@@ -275,23 +270,6 @@ class BuildinstallPhase(PhaseBase):
                             ),
                         )
                     )
-            elif self.buildinstall_method == "buildinstall":
-                volid = get_volid(self.compose, arch, disc_type=disc_type)
-                commands.append(
-                    (
-                        None,
-                        lorax.get_buildinstall_cmd(
-                            product,
-                            version,
-                            release,
-                            repo_baseurls,
-                            output_dir,
-                            is_final=self.compose.supported,
-                            buildarch=arch,
-                            volid=volid,
-                        ),
-                    )
-                )
             else:
                 raise ValueError(
                     "Unsupported buildinstall method: %s" % self.buildinstall_method
@@ -543,10 +521,20 @@ def link_boot_iso(compose, arch, variant, can_fail):
     setattr(img, "can_fail", can_fail)
     setattr(img, "deliverable", "buildinstall")
     try:
-        img.volume_id = iso.get_volume_id(new_boot_iso_path)
+        img.volume_id = iso.get_volume_id(
+            new_boot_iso_path,
+            compose.conf.get("createiso_use_xorrisofs"),
+        )
     except RuntimeError:
         pass
-    compose.im.add(variant.uid, arch, img)
+    # In this phase we should add to compose only the images that
+    # will be used only as netinstall.
+    # On this step lorax generates environment
+    # for creating isos and create them.
+    # On step `extra_isos` we overwrite the not needed iso `boot Minimal` by
+    # new iso. It already contains necessary packages from incldued variants.
+    if variant.uid in compose.conf['netinstall_variants']:
+        compose.im.add(variant.uid, arch, img)
     compose.log_info("[DONE ] %s" % msg)
 
 
@@ -731,8 +719,8 @@ class BuildinstallThread(WorkerThread):
         # input on RPM level.
         cmd_copy = copy(cmd)
         for key in ["outputdir", "sources"]:
-            del cmd_copy[key]
-            del old_metadata["cmd"][key]
+            cmd_copy.pop(key, None)
+            old_metadata["cmd"].pop(key, None)
 
         # Do not reuse if command line arguments are not the same.
         if old_metadata["cmd"] != cmd_copy:
@@ -827,8 +815,6 @@ class BuildinstallThread(WorkerThread):
         if buildinstall_method == "lorax":
             packages += ["lorax"]
             chown_paths.append(_get_log_dir(compose, variant, arch))
-        elif buildinstall_method == "buildinstall":
-            packages += ["anaconda"]
         packages += get_arch_variant_data(
             compose.conf, "buildinstall_packages", arch, variant
         )
@@ -849,13 +835,13 @@ class BuildinstallThread(WorkerThread):
 
         # Start the runroot task.
         runroot = Runroot(compose, phase="buildinstall")
+        task_id = None
         if buildinstall_method == "lorax" and lorax_use_koji_plugin:
-            runroot.run_pungi_buildinstall(
+            task_id = runroot.run_pungi_buildinstall(
                 cmd,
                 log_file=log_file,
                 arch=arch,
                 packages=packages,
-                mounts=[compose.topdir],
                 weight=compose.conf["runroot_weights"].get("buildinstall"),
             )
         else:
@@ -888,19 +874,17 @@ class BuildinstallThread(WorkerThread):
             log_dir = os.path.join(output_dir, "logs")
             copy_all(log_dir, final_log_dir)
         elif lorax_use_koji_plugin:
-            # If Koji pungi-buildinstall is used, then the buildinstall results are
-            # not stored directly in `output_dir` dir, but in "results" and "logs"
-            # subdirectories. We need to move them to final_output_dir.
-            results_dir = os.path.join(output_dir, "results")
-            move_all(results_dir, final_output_dir, rm_src_dir=True)
+            # If Koji pungi-buildinstall is used, then the buildinstall results
+            # are attached as outputs to the Koji task. Download and unpack
+            # them to the correct location.
+            download_and_extract_archive(
+                compose, task_id, "results.tar.gz", final_output_dir
+            )
 
-            # Get the log_dir into which we should copy the resulting log files.
+            # Download the logs into proper location too.
             log_fname = "buildinstall-%s-logs/dummy" % variant.uid
             final_log_dir = os.path.dirname(compose.paths.log.log_file(arch, log_fname))
-            if not os.path.exists(final_log_dir):
-                makedirs(final_log_dir)
-            log_dir = os.path.join(output_dir, "logs")
-            move_all(log_dir, final_log_dir, rm_src_dir=True)
+            download_and_extract_archive(compose, task_id, "logs.tar.gz", final_log_dir)
 
         rpms = runroot.get_buildroot_rpms()
         self._write_buildinstall_metadata(

pungi/phases/createiso.py

@@ -14,6 +14,7 @@
 # along with this program; if not, see <https://gnu.org/licenses/>.
 
 
+import itertools
 import os
 import random
 import shutil
@@ -23,7 +24,7 @@ import json
 import productmd.treeinfo
 from productmd.images import Image
 from kobo.threads import ThreadPool, WorkerThread
-from kobo.shortcuts import run, relative_path
+from kobo.shortcuts import run, relative_path, compute_file_checksums
 from six.moves import shlex_quote
 
 from pungi.wrappers import iso
@@ -154,6 +155,13 @@ class CreateisoPhase(PhaseLoggerMixin, PhaseBase):
             disc_num=cmd["disc_num"],
             disc_count=cmd["disc_count"],
         )
+        if self.compose.notifier:
+            self.compose.notifier.send(
+                "createiso-imagedone",
+                file=cmd["iso_path"],
+                arch=arch,
+                variant=str(variant),
+            )
 
     def try_reuse(self, cmd, variant, arch, opts):
         """Try to reuse image from previous compose.
@@ -181,6 +189,14 @@ class CreateisoPhase(PhaseLoggerMixin, PhaseBase):
         if not old_config:
             self.logger.info("%s - no config for old compose", log_msg)
             return False
+
+        # Disable reuse if unsigned packages are allowed. The older compose
+        # could have unsigned packages, and those may have been signed since
+        # then. We want to regenerate the ISO to have signatures.
+        if None in self.compose.conf["sigkeys"]:
+            self.logger.info("%s - unsigned packages are allowed", log_msg)
+            return False
+
         # Convert current configuration to JSON and back to encode it similarly
         # to the old one
         config = json.loads(json.dumps(self.compose.conf))
@@ -450,7 +466,14 @@ class CreateIsoThread(WorkerThread):
 
         try:
             run_createiso_command(
-                num, compose, bootable, arch, cmd["cmd"], mounts, log_file
+                num,
+                compose,
+                bootable,
+                arch,
+                cmd["cmd"],
+                mounts,
+                log_file,
+                cmd["iso_path"],
             )
         except Exception:
             self.fail(compose, cmd, variant, arch)
@@ -517,7 +540,10 @@ def add_iso_to_metadata(
     setattr(img, "can_fail", compose.can_fail(variant, arch, "iso"))
     setattr(img, "deliverable", "iso")
     try:
-        img.volume_id = iso.get_volume_id(iso_path)
+        img.volume_id = iso.get_volume_id(
+            iso_path,
+            compose.conf.get("createiso_use_xorrisofs"),
+        )
     except RuntimeError:
         pass
     if arch == "src":
@@ -528,7 +554,9 @@ def add_iso_to_metadata(
     return img
 
 
-def run_createiso_command(num, compose, bootable, arch, cmd, mounts, log_file):
+def run_createiso_command(
+    num, compose, bootable, arch, cmd, mounts, log_file, iso_path
+):
     packages = [
         "coreutils",
         "xorriso" if compose.conf.get("createiso_use_xorrisofs") else "genisoimage",
@@ -539,7 +567,6 @@ def run_createiso_command(num, compose, bootable, arch, cmd, mounts, log_file):
     if bootable:
         extra_packages = {
             "lorax": ["lorax", "which"],
-            "buildinstall": ["anaconda"],
         }
         packages.extend(extra_packages[compose.conf["buildinstall_method"]])
 
@@ -571,6 +598,76 @@ def run_createiso_command(num, compose, bootable, arch, cmd, mounts, log_file):
         weight=compose.conf["runroot_weights"].get("createiso"),
     )
 
+    if bootable and compose.conf.get("createiso_use_xorrisofs"):
+        fix_treeinfo_checksums(compose, iso_path, arch)
+
+
+def fix_treeinfo_checksums(compose, iso_path, arch):
+    """It is possible for the ISO to contain a .treefile with incorrect
+    checksums. By modifying the ISO (adding files) some of the images may
+    change.
+
+    This function fixes that after the fact by looking for incorrect checksums,
+    recalculating them and updating the .treeinfo file. Since the size of the
+    file doesn't change, this seems to not change any images.
+    """
+    modified = False
+    with iso.mount(iso_path, compose._logger) as mountpoint:
+        ti = productmd.TreeInfo()
+        ti.load(os.path.join(mountpoint, ".treeinfo"))
+        for image, (type_, expected) in ti.checksums.checksums.items():
+            checksums = compute_file_checksums(os.path.join(mountpoint, image), [type_])
+            actual = checksums[type_]
+            if actual == expected:
+                # Everything fine here, skip to next image.
+                continue
+
+            compose.log_debug("%s: %s: checksum mismatch", iso_path, image)
+            # Update treeinfo with correct checksum
+            ti.checksums.checksums[image] = (type_, actual)
+            modified = True
+
+    if not modified:
+        compose.log_debug("%s: All checksums match, nothing to do.", iso_path)
+        return
+
+    try:
+        tmpdir = compose.mkdtemp(arch, prefix="fix-checksum-")
+        # Write modified .treeinfo
+        ti_path = os.path.join(tmpdir, ".treeinfo")
+        compose.log_debug("Storing modified .treeinfo in %s", ti_path)
+        ti.dump(ti_path)
+        # Write a modified DVD into a temporary path, that is atomically moved
+        # over the original file.
+        fixed_path = os.path.join(tmpdir, "fixed-checksum-dvd.iso")
+        cmd = ["xorriso"]
+        cmd.extend(
+            itertools.chain.from_iterable(
+                iso.xorriso_commands(arch, iso_path, fixed_path)
+            )
+        )
+        cmd.extend(["-map", ti_path, ".treeinfo"])
+        run(
+            cmd,
+            logfile=compose.paths.log.log_file(
+                arch, "checksum-fix_generate_%s" % os.path.basename(iso_path)
+            ),
+        )
+        # The modified ISO no longer has implanted MD5, so that needs to be
+        # fixed again.
+        compose.log_debug("Implanting new MD5 to %s", fixed_path)
+        run(
+            iso.get_implantisomd5_cmd(fixed_path, compose.supported),
+            logfile=compose.paths.log.log_file(
+                arch, "checksum-fix_implantisomd5_%s" % os.path.basename(iso_path)
+            ),
+        )
+        # All done, move the updated image to the final location.
+        compose.log_debug("Updating %s", iso_path)
+        os.rename(fixed_path, iso_path)
+    finally:
+        shutil.rmtree(tmpdir)
+
 
 def split_iso(compose, arch, variant, no_split=False, logger=None):
     """

pungi/phases/extra_isos.py

@@ -166,6 +166,7 @@ class ExtraIsosThread(WorkerThread):
                 log_file=compose.paths.log.log_file(
                     arch, "extraiso-%s" % os.path.basename(iso_path)
                 ),
+                iso_path=iso_path,
             )
 
         img = add_iso_to_metadata(
@@ -204,6 +205,14 @@ class ExtraIsosThread(WorkerThread):
         if not old_config:
             self.pool.log_info("%s - no config for old compose", log_msg)
             return False
+
+        # Disable reuse if unsigned packages are allowed. The older compose
+        # could have unsigned packages, and those may have been signed since
+        # then. We want to regenerate the ISO to have signatures.
+        if None in compose.conf["sigkeys"]:
+            self.pool.log_info("%s - unsigned packages are allowed", log_msg)
+            return False
+
         # Convert current configuration to JSON and back to encode it similarly
         # to the old one
         config = json.loads(json.dumps(compose.conf))
@@ -420,6 +429,12 @@ def get_iso_contents(
         original_treeinfo,
         os.path.join(extra_files_dir, ".treeinfo"),
     )
+    tweak_repo_treeinfo(
+        compose,
+        include_variants,
+        original_treeinfo,
+        original_treeinfo,
+    )
 
     # Add extra files specific for the ISO
     files.update(
@@ -431,6 +446,45 @@ def get_iso_contents(
     return gp
 
 
+def tweak_repo_treeinfo(compose, include_variants, source_file, dest_file):
+    """
+    The method includes the variants to file .treeinfo of a variant. It takes
+    the variants which are described
+    by options `extra_isos -> include_variants`.
+    """
+    ti = productmd.treeinfo.TreeInfo()
+    ti.load(source_file)
+    main_variant = next(iter(ti.variants))
+    for variant_uid in include_variants:
+        variant = compose.all_variants[variant_uid]
+        var = productmd.treeinfo.Variant(ti)
+        var.id = variant.id
+        var.uid = variant.uid
+        var.name = variant.name
+        var.type = variant.type
+        ti.variants.add(var)
+
+    for variant_id in ti.variants:
+        var = ti.variants[variant_id]
+        if variant_id == main_variant:
+            var.paths.packages = 'Packages'
+            var.paths.repository = '.'
+        else:
+            var.paths.packages = os.path.join(
+                '../../..',
+                var.uid,
+                var.arch,
+                'os/Packages',
+            )
+            var.paths.repository = os.path.join(
+                '../../..',
+                var.uid,
+                var.arch,
+                'os',
+            )
+    ti.dump(dest_file, main_variant=main_variant)
+
+
 def tweak_treeinfo(compose, include_variants, source_file, dest_file):
     ti = load_and_tweak_treeinfo(source_file)
     for variant_uid in include_variants:
@@ -446,7 +500,6 @@ def tweak_treeinfo(compose, include_variants, source_file, dest_file):
         var = ti.variants[variant_id]
         var.paths.packages = os.path.join(var.uid, "Packages")
         var.paths.repository = var.uid
-
     ti.dump(dest_file)
 
 

pungi/phases/gather/__init__.py

@@ -23,6 +23,7 @@ import threading
 from kobo.rpmlib import parse_nvra
 from kobo.shortcuts import run
 from productmd.rpms import Rpms
+from pungi.phases.pkgset.common import get_all_arches
 from six.moves import cPickle as pickle
 
 try:
@@ -644,8 +645,14 @@ def _make_lookaside_repo(compose, variant, arch, pkg_map, package_sets=None):
             compose.paths.work.topdir(arch="global"), "download"
         )
         + "/",
-        "koji": lambda: pungi.wrappers.kojiwrapper.KojiWrapper(
-            compose
+        "koji": lambda: compose.conf.get(
+            "koji_cache",
+            pungi.wrappers.kojiwrapper.KojiWrapper(compose).koji_module.config.topdir,
+        ).rstrip("/")
+        + "/",
+        "kojimock": lambda: pungi.wrappers.kojiwrapper.KojiMockWrapper(
+            compose,
+            get_all_arches(compose),
         ).koji_module.config.topdir.rstrip("/")
         + "/",
     }
@@ -659,6 +666,11 @@ def _make_lookaside_repo(compose, variant, arch, pkg_map, package_sets=None):
                 # we need a union of all SRPMs.
                 if pkg_type == "srpm" or pkg_arch == arch:
                     for pkg in packages:
+                        if "lookaside" in pkg.get("flags", []):
+                            # We want to ignore lookaside packages, those will
+                            # be visible to the depending variants from the
+                            # lookaside repo directly.
+                            continue
                         pkg = pkg["path"]
                         if path_prefix and pkg.startswith(path_prefix):
                             pkg = pkg[len(path_prefix) :]

pungi/phases/image_build.py

@@ -346,7 +346,9 @@ class CreateImageBuildThread(WorkerThread):
             # let's not change filename of koji outputs
             image_dest = os.path.join(image_dir, os.path.basename(image_info["path"]))
 
-            src_file = os.path.realpath(image_info["path"])
+            src_file = compose.koji_downloader.get_file(
+                os.path.realpath(image_info["path"])
+            )
             linker.link(src_file, image_dest, link_type=cmd["link_type"])
 
             # Update image manifest

pungi/phases/image_container.py

@@ -76,7 +76,7 @@ class ImageContainerThread(WorkerThread):
         )
         if koji.watch_task(task_id, log_file) != 0:
             raise RuntimeError(
-                "ImageContainer: task %s failed: see %s for details"
+                "ImageContainer task failed: %s. See %s for details"
                 % (task_id, log_file)
             )
 

pungi/phases/kiwibuild.py

@@ -0,0 +1,229 @@
+# -*- coding: utf-8 -*-
+
+import os
+from kobo.threads import ThreadPool, WorkerThread
+from kobo import shortcuts
+from productmd.images import Image
+
+from . import base
+from .. import util
+from ..linker import Linker
+from ..wrappers import kojiwrapper
+from .image_build import EXTENSIONS
+
+KIWIEXTENSIONS = [
+    ("vhd-compressed", ["vhdfixed.xz"], "vhd.xz"),
+    ("vagrant-libvirt", ["vagrant.libvirt.box"], "vagrant-libvirt.box"),
+    ("vagrant-virtualbox", ["vagrant.virtualbox.box"], "vagrant-virtualbox.box"),
+]
+
+
+class KiwiBuildPhase(
+    base.PhaseLoggerMixin, base.ImageConfigMixin, base.ConfigGuardedPhase
+):
+    name = "kiwibuild"
+
+    def __init__(self, compose):
+        super(KiwiBuildPhase, self).__init__(compose)
+        self.pool = ThreadPool(logger=self.logger)
+
+    def _get_arches(self, image_conf, arches):
+        """Get an intersection of arches in the config dict and the given ones."""
+        if "arches" in image_conf:
+            arches = set(image_conf["arches"]) & arches
+        return sorted(arches)
+
+    @staticmethod
+    def _get_repo_urls(compose, repos, arch="$basearch"):
+        """
+        Get list of repos with resolved repo URLs. Preserve repos defined
+        as dicts.
+        """
+        resolved_repos = []
+
+        for repo in repos:
+            repo = util.get_repo_url(compose, repo, arch=arch)
+            if repo is None:
+                raise RuntimeError("Failed to resolve repo URL for %s" % repo)
+            resolved_repos.append(repo)
+
+        return resolved_repos
+
+    def _get_repo(self, image_conf, variant):
+        """
+        Get a list of repos. First included are those explicitly listed in
+        config, followed by by repo for current variant if it's not included in
+        the list already.
+        """
+        repos = shortcuts.force_list(image_conf.get("repos", []))
+
+        if not variant.is_empty and variant.uid not in repos:
+            repos.append(variant.uid)
+
+        return KiwiBuildPhase._get_repo_urls(self.compose, repos, arch="$arch")
+
+    def run(self):
+        for variant in self.compose.get_variants():
+            arches = set([x for x in variant.arches if x != "src"])
+
+            for image_conf in self.get_config_block(variant):
+                build_arches = self._get_arches(image_conf, arches)
+                if not build_arches:
+                    self.log_debug("skip: no arches")
+                    continue
+
+                # these properties can be set per-image *or* as e.g.
+                # kiwibuild_description_scm or global_release in the config
+                generics = {
+                    "release": self.get_release(image_conf),
+                    "target": self.get_config(image_conf, "target"),
+                    "descscm": self.get_config(image_conf, "description_scm"),
+                    "descpath": self.get_config(image_conf, "description_path"),
+                    "type": self.get_config(image_conf, "type"),
+                    "type_attr": self.get_config(image_conf, "type_attr"),
+                    "bundle_name_format": self.get_config(
+                        image_conf, "bundle_name_format"
+                    ),
+                }
+
+                repo = self._get_repo(image_conf, variant)
+
+                failable_arches = image_conf.pop("failable", [])
+                if failable_arches == ["*"]:
+                    failable_arches = image_conf["arches"]
+
+                self.pool.add(RunKiwiBuildThread(self.pool))
+                self.pool.queue_put(
+                    (
+                        self.compose,
+                        variant,
+                        image_conf,
+                        build_arches,
+                        generics,
+                        repo,
+                        failable_arches,
+                    )
+                )
+
+        self.pool.start()
+
+
+class RunKiwiBuildThread(WorkerThread):
+    def process(self, item, num):
+        (compose, variant, config, arches, generics, repo, failable_arches) = item
+        self.failable_arches = failable_arches
+        # the Koji task as a whole can only fail if *all* arches are failable
+        can_task_fail = set(failable_arches).issuperset(set(arches))
+        self.num = num
+        with util.failable(
+            compose,
+            can_task_fail,
+            variant,
+            "*",
+            "kiwibuild",
+            logger=self.pool._logger,
+        ):
+            self.worker(compose, variant, config, arches, generics, repo)
+
+    def worker(self, compose, variant, config, arches, generics, repo):
+        msg = "kiwibuild task for variant %s" % variant.uid
+        self.pool.log_info("[BEGIN] %s" % msg)
+        koji = kojiwrapper.KojiWrapper(compose)
+        koji.login()
+
+        task_id = koji.koji_proxy.kiwiBuild(
+            generics["target"],
+            arches,
+            generics["descscm"],
+            generics["descpath"],
+            profile=config["kiwi_profile"],
+            release=generics["release"],
+            repos=repo,
+            type=generics["type"],
+            type_attr=generics["type_attr"],
+            result_bundle_name_format=generics["bundle_name_format"],
+            # this ensures the task won't fail if only failable arches fail
+            optional_arches=self.failable_arches,
+        )
+
+        koji.save_task_id(task_id)
+
+        # Wait for it to finish and capture the output into log file.
+        log_dir = os.path.join(compose.paths.log.topdir(), "kiwibuild")
+        util.makedirs(log_dir)
+        log_file = os.path.join(
+            log_dir, "%s-%s-watch-task.log" % (variant.uid, self.num)
+        )
+        if koji.watch_task(task_id, log_file) != 0:
+            raise RuntimeError(
+                "kiwiBuild task failed: %s. See %s for details" % (task_id, log_file)
+            )
+
+        # Refresh koji session which may have timed out while the task was
+        # running. Watching is done via a subprocess, so the session is
+        # inactive.
+        koji = kojiwrapper.KojiWrapper(compose)
+
+        linker = Linker(logger=self.pool._logger)
+
+        # Process all images in the build. There should be one for each
+        # architecture, but we don't verify that.
+        paths = koji.get_image_paths(task_id)
+
+        for arch, paths in paths.items():
+            for path in paths:
+                type_, format_ = _find_type_and_format(path)
+                if not format_:
+                    # Path doesn't match any known type.
+                    continue
+
+                # image_dir is absolute path to which the image should be copied.
+                # We also need the same path as relative to compose directory for
+                # including in the metadata.
+                image_dir = compose.paths.compose.image_dir(variant) % {"arch": arch}
+                rel_image_dir = compose.paths.compose.image_dir(
+                    variant, relative=True
+                ) % {"arch": arch}
+                util.makedirs(image_dir)
+
+                filename = os.path.basename(path)
+
+                image_dest = os.path.join(image_dir, filename)
+
+                src_file = compose.koji_downloader.get_file(path)
+
+                linker.link(src_file, image_dest, link_type=compose.conf["link_type"])
+
+                # Update image manifest
+                img = Image(compose.im)
+
+                # Get the manifest type from the config if supplied, otherwise we
+                # determine the manifest type based on the koji output
+                img.type = type_
+                img.format = format_
+                img.path = os.path.join(rel_image_dir, filename)
+                img.mtime = util.get_mtime(image_dest)
+                img.size = util.get_file_size(image_dest)
+                img.arch = arch
+                img.disc_number = 1  # We don't expect multiple disks
+                img.disc_count = 1
+                img.bootable = False
+                img.subvariant = config.get("subvariant", variant.uid)
+                setattr(img, "can_fail", arch in self.failable_arches)
+                setattr(img, "deliverable", "kiwibuild")
+                compose.im.add(variant=variant.uid, arch=arch, image=img)
+
+        self.pool.log_info("[DONE ] %s (task id: %s)" % (msg, task_id))
+
+
+def _find_type_and_format(path):
+    for type_, suffixes in EXTENSIONS.items():
+        for suffix in suffixes:
+            if path.endswith(suffix):
+                return type_, suffix
+    # these are our kiwi-exclusive mappings for images whose extensions
+    # aren't quite the same as imagefactory
+    for type_, suffixes, format_ in KIWIEXTENSIONS:
+        if any(path.endswith(suffix) for suffix in suffixes):
+            return type_, format_
+    return None, None

pungi/phases/live_images.py

@@ -1,406 +0,0 @@
-# -*- coding: utf-8 -*-
-
-
-# This program is free software; you can redistribute it and/or modify
-# it under the terms of the GNU General Public License as published by
-# the Free Software Foundation; version 2 of the License.
-#
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU Library General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License
-# along with this program; if not, see <https://gnu.org/licenses/>.
-
-
-import os
-import sys
-import time
-import shutil
-
-from kobo.threads import ThreadPool, WorkerThread
-from kobo.shortcuts import run, save_to_file, force_list
-from productmd.images import Image
-from six.moves import shlex_quote
-
-from pungi.wrappers.kojiwrapper import KojiWrapper
-from pungi.wrappers import iso
-from pungi.phases import base
-from pungi.util import makedirs, get_mtime, get_file_size, failable
-from pungi.util import get_repo_urls
-
-
-# HACK: define cmp in python3
-if sys.version_info[0] == 3:
-
-    def cmp(a, b):
-        return (a > b) - (a < b)
-
-
-class LiveImagesPhase(
-    base.PhaseLoggerMixin, base.ImageConfigMixin, base.ConfigGuardedPhase
-):
-    name = "live_images"
-
-    def __init__(self, compose):
-        super(LiveImagesPhase, self).__init__(compose)
-        self.pool = ThreadPool(logger=self.logger)
-
-    def _get_repos(self, arch, variant, data):
-        repos = []
-        if not variant.is_empty:
-            repos.append(variant.uid)
-        repos.extend(force_list(data.get("repo", [])))
-        return get_repo_urls(self.compose, repos, arch=arch)
-
-    def run(self):
-        symlink_isos_to = self.compose.conf.get("symlink_isos_to")
-        commands = []
-
-        for variant in self.compose.all_variants.values():
-            for arch in variant.arches + ["src"]:
-                for data in self.get_config_block(variant, arch):
-                    subvariant = data.get("subvariant", variant.uid)
-                    type = data.get("type", "live")
-
-                    if type == "live":
-                        dest_dir = self.compose.paths.compose.iso_dir(
-                            arch, variant, symlink_to=symlink_isos_to
-                        )
-                    elif type == "appliance":
-                        dest_dir = self.compose.paths.compose.image_dir(
-                            variant, symlink_to=symlink_isos_to
-                        )
-                        dest_dir = dest_dir % {"arch": arch}
-                        makedirs(dest_dir)
-                    else:
-                        raise RuntimeError("Unknown live image type %s" % type)
-                    if not dest_dir:
-                        continue
-
-                    cmd = {
-                        "name": data.get("name"),
-                        "version": self.get_version(data),
-                        "release": self.get_release(data),
-                        "dest_dir": dest_dir,
-                        "build_arch": arch,
-                        "ks_file": data["kickstart"],
-                        "ksurl": self.get_ksurl(data),
-                        # Used for images wrapped in RPM
-                        "specfile": data.get("specfile", None),
-                        # Scratch (only taken in consideration if specfile
-                        # specified) For images wrapped in rpm is scratch
-                        # disabled by default For other images is scratch
-                        # always on
-                        "scratch": data.get("scratch", False),
-                        "sign": False,
-                        "type": type,
-                        "label": "",  # currently not used
-                        "subvariant": subvariant,
-                        "failable_arches": data.get("failable", []),
-                        # First see if live_target is specified, then fall back
-                        # to regular setup of local, phase and global setting.
-                        "target": self.compose.conf.get("live_target")
-                        or self.get_config(data, "target"),
-                    }
-
-                    cmd["repos"] = self._get_repos(arch, variant, data)
-
-                    # Signing of the rpm wrapped image
-                    if not cmd["scratch"] and data.get("sign"):
-                        cmd["sign"] = True
-
-                    cmd["filename"] = self._get_file_name(
-                        arch, variant, cmd["name"], cmd["version"]
-                    )
-
-                    commands.append((cmd, variant, arch))
-
-        for cmd, variant, arch in commands:
-            self.pool.add(CreateLiveImageThread(self.pool))
-            self.pool.queue_put((self.compose, cmd, variant, arch))
-
-        self.pool.start()
-
-    def _get_file_name(self, arch, variant, name=None, version=None):
-        if self.compose.conf["live_images_no_rename"]:
-            return None
-
-        disc_type = self.compose.conf["disc_types"].get("live", "live")
-
-        format = (
-            "%(compose_id)s-%(variant)s-%(arch)s-%(disc_type)s%(disc_num)s%(suffix)s"
-        )
-        # Custom name (prefix)
-        if name:
-            custom_iso_name = name
-            if version:
-                custom_iso_name += "-%s" % version
-            format = (
-                custom_iso_name
-                + "-%(variant)s-%(arch)s-%(disc_type)s%(disc_num)s%(suffix)s"
-            )
-
-        # XXX: hardcoded disc_num
-        return self.compose.get_image_name(
-            arch, variant, disc_type=disc_type, disc_num=None, format=format
-        )
-
-
-class CreateLiveImageThread(WorkerThread):
-    EXTS = (".iso", ".raw.xz")
-
-    def process(self, item, num):
-        compose, cmd, variant, arch = item
-        self.failable_arches = cmd.get("failable_arches", [])
-        self.can_fail = bool(self.failable_arches)
-        with failable(
-            compose,
-            self.can_fail,
-            variant,
-            arch,
-            "live",
-            cmd.get("subvariant"),
-            logger=self.pool._logger,
-        ):
-            self.worker(compose, cmd, variant, arch, num)
-
-    def worker(self, compose, cmd, variant, arch, num):
-        self.basename = "%(name)s-%(version)s-%(release)s" % cmd
-        log_file = compose.paths.log.log_file(arch, "liveimage-%s" % self.basename)
-
-        subvariant = cmd.pop("subvariant")
-
-        imgname = "%s-%s-%s-%s" % (
-            compose.ci_base.release.short,
-            subvariant,
-            "Live" if cmd["type"] == "live" else "Disk",
-            arch,
-        )
-
-        msg = "Creating ISO (arch: %s, variant: %s): %s" % (
-            arch,
-            variant,
-            self.basename,
-        )
-        self.pool.log_info("[BEGIN] %s" % msg)
-
-        koji_wrapper = KojiWrapper(compose)
-        _, version = compose.compose_id.rsplit("-", 1)
-        name = cmd["name"] or imgname
-        version = cmd["version"] or version
-        archive = False
-        if cmd["specfile"] and not cmd["scratch"]:
-            # Non scratch build are allowed only for rpm wrapped images
-            archive = True
-        koji_cmd = koji_wrapper.get_create_image_cmd(
-            name,
-            version,
-            cmd["target"],
-            cmd["build_arch"],
-            cmd["ks_file"],
-            cmd["repos"],
-            image_type=cmd["type"],
-            wait=True,
-            archive=archive,
-            specfile=cmd["specfile"],
-            release=cmd["release"],
-            ksurl=cmd["ksurl"],
-        )
-
-        # avoid race conditions?
-        # Kerberos authentication failed:
-        #   Permission denied in replay cache code (-1765328215)
-        time.sleep(num * 3)
-
-        output = koji_wrapper.run_blocking_cmd(koji_cmd, log_file=log_file)
-        if output["retcode"] != 0:
-            raise RuntimeError(
-                "LiveImage task failed: %s. See %s for more details."
-                % (output["task_id"], log_file)
-            )
-
-        # copy finished image to isos/
-        image_path = [
-            path
-            for path in koji_wrapper.get_image_path(output["task_id"])
-            if self._is_image(path)
-        ]
-        if len(image_path) != 1:
-            raise RuntimeError(
-                "Got %d images from task %d, expected 1."
-                % (len(image_path), output["task_id"])
-            )
-        image_path = image_path[0]
-        filename = cmd.get("filename") or os.path.basename(image_path)
-        destination = os.path.join(cmd["dest_dir"], filename)
-        shutil.copy2(image_path, destination)
-
-        # copy finished rpm to isos/ (if rpm wrapped ISO was built)
-        if cmd["specfile"]:
-            rpm_paths = koji_wrapper.get_wrapped_rpm_path(output["task_id"])
-
-            if cmd["sign"]:
-                # Sign the rpm wrapped images and get their paths
-                self.pool.log_info(
-                    "Signing rpm wrapped images in task_id: %s (expected key ID: %s)"
-                    % (output["task_id"], compose.conf.get("signing_key_id"))
-                )
-                signed_rpm_paths = self._sign_image(
-                    koji_wrapper, compose, cmd, output["task_id"]
-                )
-                if signed_rpm_paths:
-                    rpm_paths = signed_rpm_paths
-
-            for rpm_path in rpm_paths:
-                shutil.copy2(rpm_path, cmd["dest_dir"])
-
-        if cmd["type"] == "live":
-            # ISO manifest only makes sense for live images
-            self._write_manifest(destination)
-
-        self._add_to_images(
-            compose,
-            variant,
-            subvariant,
-            arch,
-            cmd["type"],
-            self._get_format(image_path),
-            destination,
-        )
-
-        self.pool.log_info("[DONE ] %s (task id: %s)" % (msg, output["task_id"]))
-
-    def _add_to_images(self, compose, variant, subvariant, arch, type, format, path):
-        """Adds the image to images.json"""
-        img = Image(compose.im)
-        img.type = "raw-xz" if type == "appliance" else type
-        img.format = format
-        img.path = os.path.relpath(path, compose.paths.compose.topdir())
-        img.mtime = get_mtime(path)
-        img.size = get_file_size(path)
-        img.arch = arch
-        img.disc_number = 1  # We don't expect multiple disks
-        img.disc_count = 1
-        img.bootable = True
-        img.subvariant = subvariant
-        setattr(img, "can_fail", self.can_fail)
-        setattr(img, "deliverable", "live")
-        compose.im.add(variant=variant.uid, arch=arch, image=img)
-
-    def _is_image(self, path):
-        for ext in self.EXTS:
-            if path.endswith(ext):
-                return True
-        return False
-
-    def _get_format(self, path):
-        """Get format based on extension."""
-        for ext in self.EXTS:
-            if path.endswith(ext):
-                return ext[1:]
-        raise RuntimeError("Getting format for unknown image %s" % path)
-
-    def _write_manifest(self, iso_path):
-        """Generate manifest for ISO at given path.
-
-        :param iso_path: (str) absolute path to the ISO
-        """
-        dir, filename = os.path.split(iso_path)
-        run("cd %s && %s" % (shlex_quote(dir), iso.get_manifest_cmd(filename)))
-
-    def _sign_image(self, koji_wrapper, compose, cmd, koji_task_id):
-        signing_key_id = compose.conf.get("signing_key_id")
-        signing_command = compose.conf.get("signing_command")
-
-        if not signing_key_id:
-            self.pool.log_warning(
-                "Signing is enabled but signing_key_id is not specified"
-            )
-            self.pool.log_warning("Signing skipped")
-            return None
-        if not signing_command:
-            self.pool.log_warning(
-                "Signing is enabled but signing_command is not specified"
-            )
-            self.pool.log_warning("Signing skipped")
-            return None
-
-        # Prepare signing log file
-        signing_log_file = compose.paths.log.log_file(
-            cmd["build_arch"], "live_images-signing-%s" % self.basename
-        )
-
-        # Sign the rpm wrapped images
-        try:
-            sign_builds_in_task(
-                koji_wrapper,
-                koji_task_id,
-                signing_command,
-                log_file=signing_log_file,
-                signing_key_password=compose.conf.get("signing_key_password"),
-            )
-        except RuntimeError:
-            self.pool.log_error(
-                "Error while signing rpm wrapped images. See log: %s" % signing_log_file
-            )
-            raise
-
-        # Get pats to the signed rpms
-        signing_key_id = signing_key_id.lower()  # Koji uses lowercase in paths
-        rpm_paths = koji_wrapper.get_signed_wrapped_rpms_paths(
-            koji_task_id, signing_key_id
-        )
-
-        # Wait until files are available
-        if wait_paths(rpm_paths, 60 * 15):
-            # Files are ready
-            return rpm_paths
-
-        # Signed RPMs are not available
-        self.pool.log_warning("Signed files are not available: %s" % rpm_paths)
-        self.pool.log_warning("Unsigned files will be used")
-        return None
-
-
-def wait_paths(paths, timeout=60):
-    started = time.time()
-    remaining = paths[:]
-    while True:
-        for path in remaining[:]:
-            if os.path.exists(path):
-                remaining.remove(path)
-        if not remaining:
-            break
-        time.sleep(1)
-        if timeout >= 0 and (time.time() - started) > timeout:
-            return False
-    return True
-
-
-def sign_builds_in_task(
-    koji_wrapper, task_id, signing_command, log_file=None, signing_key_password=None
-):
-    # Get list of nvrs that should be signed
-    nvrs = koji_wrapper.get_build_nvrs(task_id)
-    if not nvrs:
-        # No builds are available (scratch build, etc.?)
-        return
-
-    # Append builds to sign_cmd
-    for nvr in nvrs:
-        signing_command += " '%s'" % nvr
-
-    # Log signing command before password is filled in it
-    if log_file:
-        save_to_file(log_file, signing_command, append=True)
-
-    # Fill password into the signing command
-    if signing_key_password:
-        signing_command = signing_command % {
-            "signing_key_password": signing_key_password
-        }
-
-    # Sign the builds
-    run(signing_command, can_fail=False, show_cmd=False, logfile=log_file)

pungi/phases/livemedia_phase.py

@@ -182,7 +182,9 @@ class LiveMediaThread(WorkerThread):
             # let's not change filename of koji outputs
             image_dest = os.path.join(image_dir, os.path.basename(image_info["path"]))
 
-            src_file = os.path.realpath(image_info["path"])
+            src_file = compose.koji_downloader.get_file(
+                os.path.realpath(image_info["path"])
+            )
             linker.link(src_file, image_dest, link_type=link_type)
 
             # Update image manifest

pungi/phases/osbs.py

@@ -134,7 +134,7 @@ class OSBSThread(WorkerThread):
         # though there is not much there).
         if koji.watch_task(task_id, log_file) != 0:
             raise RuntimeError(
-                "OSBS: task %s failed: see %s for details" % (task_id, log_file)
+                "OSBS task failed: %s. See %s for details" % (task_id, log_file)
             )
 
         scratch = config.get("scratch", False)
@@ -154,7 +154,7 @@ class OSBSThread(WorkerThread):
             reuse_file,
         )
 
-        self.pool.log_info("[DONE ] %s" % msg)
+        self.pool.log_info("[DONE ] %s (task id: %s)" % (msg, task_id))
 
     def _get_image_conf(self, compose, config):
         """Get image-build.conf from git repo.

pungi/phases/osbuild.py

@@ -159,6 +159,10 @@ class RunOSBuildThread(WorkerThread):
         if upload_options:
             opts["upload_options"] = upload_options
 
+        customizations = config.get("customizations")
+        if customizations:
+            opts["customizations"] = customizations
+
         if release:
             opts["release"] = release
         task_id = koji.koji_proxy.osbuildImage(
@@ -181,7 +185,7 @@ class RunOSBuildThread(WorkerThread):
         )
         if koji.watch_task(task_id, log_file) != 0:
             raise RuntimeError(
-                "OSBuild: task %s failed: see %s for details" % (task_id, log_file)
+                "OSBuild task failed: %s. See %s for details" % (task_id, log_file)
             )
 
         # Refresh koji session which may have timed out while the task was
@@ -228,8 +232,11 @@ class RunOSBuildThread(WorkerThread):
 
             image_dest = os.path.join(image_dir, archive["filename"])
 
-            src_file = os.path.join(
-                koji.koji_module.pathinfo.imagebuild(build_info), archive["filename"]
+            src_file = compose.koji_downloader.get_file(
+                os.path.join(
+                    koji.koji_module.pathinfo.imagebuild(build_info),
+                    archive["filename"],
+                ),
             )
 
             linker.link(src_file, image_dest, link_type=compose.conf["link_type"])
@@ -246,7 +253,24 @@ class RunOSBuildThread(WorkerThread):
 
             # Update image manifest
             img = Image(compose.im)
-            img.type = archive["type_name"] if archive["type_name"] != "iso" else "dvd"
+
+            # Get the manifest type from the config if supplied, otherwise we
+            # determine the manifest type based on the koji output
+            img.type = config.get("manifest_type")
+            if not img.type:
+                if archive["type_name"] != "iso":
+                    img.type = archive["type_name"]
+                else:
+                    fn = archive["filename"].lower()
+                    if "ostree" in fn:
+                        img.type = "dvd-ostree-osbuild"
+                    elif "live" in fn:
+                        img.type = "live-osbuild"
+                    elif "netinst" in fn or "boot" in fn:
+                        img.type = "boot"
+                    else:
+                        img.type = "dvd"
+
             img.format = suffix
             img.path = os.path.join(rel_image_dir, archive["filename"])
             img.mtime = util.get_mtime(image_dest)

pungi/phases/ostree.py

@@ -85,7 +85,7 @@ class OSTreeThread(WorkerThread):
         comps_repo = compose.paths.work.comps_repo(
             "$basearch", variant=variant, create_dir=False
         )
-        repos = shortcuts.force_list(config["repo"]) + self.repos
+        repos = shortcuts.force_list(config.get("repo", [])) + self.repos
         if compose.has_comps:
             repos.append(translate_path(compose, comps_repo))
         repos = get_repo_dicts(repos, logger=self.pool)
@@ -168,7 +168,9 @@ class OSTreeThread(WorkerThread):
                 ("unified-core", config.get("unified_core", False)),
             ]
         )
-        packages = ["pungi", "ostree", "rpm-ostree"]
+        default_packages = ["pungi", "ostree", "rpm-ostree"]
+        additional_packages = config.get("runroot_packages", [])
+        packages = default_packages + additional_packages
         log_file = os.path.join(self.logdir, "runroot.log")
         mounts = [compose.topdir, config["ostree_repo"]]
         runroot = Runroot(compose, phase="ostree")

pungi/phases/ostree_container.py

@@ -0,0 +1,190 @@
+# -*- coding: utf-8 -*-
+
+import copy
+import json
+import os
+from kobo import shortcuts
+from kobo.threads import ThreadPool, WorkerThread
+
+from productmd.images import Image
+
+from pungi.runroot import Runroot
+from .base import ConfigGuardedPhase
+from .. import util
+from ..util import get_repo_dicts, translate_path
+from ..wrappers import scm
+
+
+class OSTreeContainerPhase(ConfigGuardedPhase):
+    name = "ostree_container"
+
+    def __init__(self, compose, pkgset_phase=None):
+        super(OSTreeContainerPhase, self).__init__(compose)
+        self.pool = ThreadPool(logger=self.compose._logger)
+        self.pkgset_phase = pkgset_phase
+
+    def get_repos(self):
+        return [
+            translate_path(
+                self.compose,
+                self.compose.paths.work.pkgset_repo(
+                    pkgset.name, "$basearch", create_dir=False
+                ),
+            )
+            for pkgset in self.pkgset_phase.package_sets
+        ]
+
+    def _enqueue(self, variant, arch, conf):
+        self.pool.add(OSTreeContainerThread(self.pool, self.get_repos()))
+        self.pool.queue_put((self.compose, variant, arch, conf))
+
+    def run(self):
+        if isinstance(self.compose.conf.get(self.name), dict):
+            for variant in self.compose.get_variants():
+                for conf in self.get_config_block(variant):
+                    for arch in conf.get("arches", []) or variant.arches:
+                        self._enqueue(variant, arch, conf)
+        else:
+            # Legacy code path to support original configuration.
+            for variant in self.compose.get_variants():
+                for arch in variant.arches:
+                    for conf in self.get_config_block(variant, arch):
+                        self._enqueue(variant, arch, conf)
+
+        self.pool.start()
+
+
+class OSTreeContainerThread(WorkerThread):
+    def __init__(self, pool, repos):
+        super(OSTreeContainerThread, self).__init__(pool)
+        self.repos = repos
+
+    def process(self, item, num):
+        compose, variant, arch, config = item
+        self.num = num
+        failable_arches = config.get("failable", [])
+        self.can_fail = util.can_arch_fail(failable_arches, arch)
+        with util.failable(compose, self.can_fail, variant, arch, "ostree-container"):
+            self.worker(compose, variant, arch, config)
+
+    def worker(self, compose, variant, arch, config):
+        msg = "OSTree container phase for variant %s, arch %s" % (variant.uid, arch)
+        self.pool.log_info("[BEGIN] %s" % msg)
+        workdir = compose.paths.work.topdir("ostree-container-%d" % self.num)
+        self.logdir = compose.paths.log.topdir(
+            "%s/%s/ostree-container-%d" % (arch, variant.uid, self.num)
+        )
+        repodir = os.path.join(workdir, "config_repo")
+        self._clone_repo(
+            compose,
+            repodir,
+            config["config_url"],
+            config.get("config_branch", "main"),
+        )
+
+        repos = shortcuts.force_list(config.get("repo", [])) + self.repos
+        repos = get_repo_dicts(repos, logger=self.pool)
+
+        # copy the original config and update before save to a json file
+        new_config = copy.copy(config)
+
+        # repos in configuration can have repo url set to variant UID,
+        # update it to have the actual url that we just translated.
+        new_config.update({"repo": repos})
+
+        # remove unnecessary (for 'pungi-make-ostree container' script ) elements
+        # from config, it doesn't hurt to have them, however remove them can
+        # reduce confusion
+        for k in [
+            "treefile",
+            "config_url",
+            "config_branch",
+            "failable",
+            "version",
+        ]:
+            new_config.pop(k, None)
+
+        # write a json file to save the configuration, so 'pungi-make-ostree tree'
+        # can take use of it
+        extra_config_file = os.path.join(workdir, "extra_config.json")
+        with open(extra_config_file, "w") as f:
+            json.dump(new_config, f, indent=4)
+
+        self._run_ostree_container_cmd(
+            compose, variant, arch, config, repodir, extra_config_file=extra_config_file
+        )
+
+        self.pool.log_info("[DONE ] %s" % (msg))
+
+    def _run_ostree_container_cmd(
+        self, compose, variant, arch, config, config_repo, extra_config_file=None
+    ):
+        target_dir = compose.paths.compose.image_dir(variant) % {"arch": arch}
+        util.makedirs(target_dir)
+        version = util.version_generator(compose, config.get("version"))
+        archive_name = "%s-%s-%s" % (
+            compose.conf["release_short"],
+            variant.uid,
+            version,
+        )
+
+        # Run the pungi-make-ostree command locally to create a script to
+        # execute in runroot environment.
+        cmd = [
+            "pungi-make-ostree",
+            "container",
+            "--log-dir=%s" % self.logdir,
+            "--name=%s" % archive_name,
+            "--path=%s" % target_dir,
+            "--treefile=%s" % os.path.join(config_repo, config["treefile"]),
+            "--extra-config=%s" % extra_config_file,
+            "--version=%s" % version,
+        ]
+
+        _, runroot_script = shortcuts.run(cmd, universal_newlines=True)
+
+        default_packages = ["ostree", "rpm-ostree", "selinux-policy-targeted"]
+        additional_packages = config.get("runroot_packages", [])
+        packages = default_packages + additional_packages
+        log_file = os.path.join(self.logdir, "runroot.log")
+        # TODO: Use to get previous build
+        mounts = [compose.topdir]
+
+        runroot = Runroot(compose, phase="ostree_container")
+        runroot.run(
+            " && ".join(runroot_script.splitlines()),
+            log_file=log_file,
+            arch=arch,
+            packages=packages,
+            mounts=mounts,
+            new_chroot=True,
+            weight=compose.conf["runroot_weights"].get("ostree"),
+        )
+
+        fullpath = os.path.join(target_dir, "%s.ociarchive" % archive_name)
+
+        # Update image manifest
+        img = Image(compose.im)
+
+        # Get the manifest type from the config if supplied, otherwise we
+        # determine the manifest type based on the koji output
+        img.type = "ociarchive"
+        img.format = "ociarchive"
+        img.path = os.path.relpath(fullpath, compose.paths.compose.topdir())
+        img.mtime = util.get_mtime(fullpath)
+        img.size = util.get_file_size(fullpath)
+        img.arch = arch
+        img.disc_number = 1
+        img.disc_count = 1
+        img.bootable = False
+        img.subvariant = config.get("subvariant", variant.uid)
+        setattr(img, "can_fail", self.can_fail)
+        setattr(img, "deliverable", "ostree-container")
+        compose.im.add(variant=variant.uid, arch=arch, image=img)
+
+    def _clone_repo(self, compose, repodir, url, branch):
+        scm.get_dir_from_scm(
+            {"scm": "git", "repo": url, "branch": branch, "dir": "."},
+            repodir,
+            compose=compose,
+        )

pungi/phases/pkgset/pkgsets.py

@@ -23,11 +23,15 @@ import itertools
 import json
 import os
 import time
+import pgpy
+import rpm
 from six.moves import cPickle as pickle
+from functools import partial
 
 import kobo.log
 import kobo.pkgset
 import kobo.rpmlib
+from kobo.shortcuts import compute_file_checksums
 
 from kobo.threads import WorkerThread, ThreadPool
 
@@ -150,9 +154,15 @@ class PackageSetBase(kobo.log.LoggingBase):
         """
 
         def nvr_formatter(package_info):
-            # joins NVR parts of the package with '-' character.
-            return "-".join(
-                (package_info["name"], package_info["version"], package_info["release"])
+            epoch_suffix = ''
+            if package_info['epoch'] is not None:
+                epoch_suffix = ':' + package_info['epoch']
+            return (
+                f"{package_info['name']}"
+                f"{epoch_suffix}-"
+                f"{package_info['version']}-"
+                f"{package_info['release']}."
+                f"{package_info['arch']}"
             )
 
         def get_error(sigkeys, infos):
@@ -354,6 +364,7 @@ class KojiPackageSet(PackageSetBase):
         extra_tasks=None,
         signed_packages_retries=0,
         signed_packages_wait=30,
+        downloader=None,
     ):
         """
         Creates new KojiPackageSet.
@@ -408,6 +419,8 @@ class KojiPackageSet(PackageSetBase):
         self.signed_packages_retries = signed_packages_retries
         self.signed_packages_wait = signed_packages_wait
 
+        self.downloader = downloader
+
     def __getstate__(self):
         result = self.__dict__.copy()
         del result["koji_wrapper"]
@@ -498,7 +511,8 @@ class KojiPackageSet(PackageSetBase):
 
         response = None
         if self.cache_region:
-            cache_key = "KojiPackageSet.get_latest_rpms_%s_%s_%s" % (
+            cache_key = "%s.get_latest_rpms_%s_%s_%s" % (
+                str(self.__class__.__name__),
                 str(tag),
                 str(event),
                 str(inherit),
@@ -520,17 +534,36 @@ class KojiPackageSet(PackageSetBase):
 
         return response
 
+
+
     def get_package_path(self, queue_item):
         rpm_info, build_info = queue_item
 
         # Check if this RPM is coming from scratch task. In this case, we already
         # know the path.
         if "path_from_task" in rpm_info:
-            return rpm_info["path_from_task"]
+            return self.downloader.get_file(rpm_info["path_from_task"])
 
         pathinfo = self.koji_wrapper.koji_module.pathinfo
         paths = []
 
+        if "getRPMChecksums" in self.koji_proxy.system.listMethods():
+
+            def checksum_validator(keyname, pkg_path):
+                checksums = self.koji_proxy.getRPMChecksums(
+                    rpm_info["id"], checksum_types=("sha256",)
+                )
+                if "sha256" in checksums.get(keyname, {}):
+                    computed = compute_file_checksums(pkg_path, ("sha256",))
+                    if computed["sha256"] != checksums[keyname]["sha256"]:
+                        raise RuntimeError("Checksum mismatch for %s" % pkg_path)
+
+        else:
+
+            def checksum_validator(keyname, pkg_path):
+                # Koji doesn't support checksums yet
+                pass
+
         attempts_left = self.signed_packages_retries + 1
         while attempts_left > 0:
             for sigkey in self.sigkey_ordering:
@@ -543,8 +576,11 @@ class KojiPackageSet(PackageSetBase):
                 )
                 if rpm_path not in paths:
                     paths.append(rpm_path)
-                if os.path.isfile(rpm_path):
-                    return rpm_path
+                path = self.downloader.get_file(
+                    rpm_path, partial(checksum_validator, sigkey)
+                )
+                if path:
+                    return path
 
             # No signed copy was found, wait a little and try again.
             attempts_left -= 1
@@ -557,16 +593,18 @@ class KojiPackageSet(PackageSetBase):
             # use an unsigned copy (if allowed)
             rpm_path = os.path.join(pathinfo.build(build_info), pathinfo.rpm(rpm_info))
             paths.append(rpm_path)
-            if os.path.isfile(rpm_path):
-                return rpm_path
+            path = self.downloader.get_file(rpm_path, partial(checksum_validator, ""))
+            if path:
+                return path
 
         if self._allow_invalid_sigkeys and rpm_info["name"] not in self.packages:
             # use an unsigned copy (if allowed)
             rpm_path = os.path.join(pathinfo.build(build_info), pathinfo.rpm(rpm_info))
             paths.append(rpm_path)
-            if os.path.isfile(rpm_path):
+            path = self.downloader.get_file(rpm_path)
+            if path:
                 self._invalid_sigkey_rpms.append(rpm_info)
-                return rpm_path
+                return path
 
         self._invalid_sigkey_rpms.append(rpm_info)
         self.log_error(
@@ -587,7 +625,7 @@ class KojiPackageSet(PackageSetBase):
         result_srpms = []
         include_packages = set(include_packages or [])
 
-        if type(event) is dict:
+        if isinstance(event, dict):
             event = event["id"]
 
         msg = "Getting latest RPMs (tag: %s, event: %s, inherit: %s)" % (
@@ -734,6 +772,7 @@ class KojiPackageSet(PackageSetBase):
                         "exclusive_noarch": compose.conf[
                             "pkgset_exclusive_arch_considers_noarch"
                         ],
+                        "module_defaults_dir": compose.conf.get("module_defaults_dir"),
                     },
                     f,
                     protocol=pickle.HIGHEST_PROTOCOL,
@@ -830,6 +869,7 @@ class KojiPackageSet(PackageSetBase):
 
         inherit_to_noarch = compose.conf["pkgset_inherit_exclusive_arch_to_noarch"]
         exclusive_noarch = compose.conf["pkgset_exclusive_arch_considers_noarch"]
+        module_defaults_dir = compose.conf.get("module_defaults_dir")
         if (
             reuse_data["allow_invalid_sigkeys"] == self._allow_invalid_sigkeys
             and reuse_data["packages"] == self.packages
@@ -841,6 +881,7 @@ class KojiPackageSet(PackageSetBase):
             # generated with older version of Pungi. Best to not reuse.
             and reuse_data.get("inherit_to_noarch") == inherit_to_noarch
             and reuse_data.get("exclusive_noarch") == exclusive_noarch
+            and reuse_data.get("module_defaults_dir") == module_defaults_dir
         ):
             self.log_info("Copying repo data for reuse: %s" % old_repo_dir)
             copy_all(old_repo_dir, repo_dir)
@@ -855,6 +896,67 @@ class KojiPackageSet(PackageSetBase):
             return False
 
 
+class KojiMockPackageSet(KojiPackageSet):
+
+    def _is_rpm_signed(self, rpm_path) -> bool:
+        ts = rpm.TransactionSet()
+        ts.setVSFlags(rpm._RPMVSF_NOSIGNATURES)
+        sigkeys = [
+            sigkey.lower() for sigkey in self.sigkey_ordering
+            if sigkey is not None
+        ]
+        if not sigkeys:
+            return True
+        with open(rpm_path, 'rb') as fd:
+            header = ts.hdrFromFdno(fd)
+            signature = header[rpm.RPMTAG_SIGGPG] or header[rpm.RPMTAG_SIGPGP]
+            if signature is None:
+                return False
+            pgp_msg = pgpy.PGPMessage.from_blob(signature)
+            return any(
+                signature.signer.lower() in sigkeys
+                for signature in pgp_msg.signatures
+            )
+
+    def get_package_path(self, queue_item):
+        rpm_info, build_info = queue_item
+
+        # Check if this RPM is coming from scratch task.
+        # In this case, we already know the path.
+        if "path_from_task" in rpm_info:
+            return rpm_info["path_from_task"]
+
+        # we replaced this part because pungi uses way
+        # of guessing path of package on koji based on sigkey
+        # we don't need that because all our packages will
+        # be ready for release
+        # signature verification is still done during deps resolution
+        pathinfo = self.koji_wrapper.koji_module.pathinfo
+
+        rpm_path = os.path.join(pathinfo.topdir, pathinfo.rpm(rpm_info))
+        if os.path.isfile(rpm_path):
+            if not self._is_rpm_signed(rpm_path):
+                self._invalid_sigkey_rpms.append(rpm_info)
+                self.log_error(
+                    'RPM "%s" not found for sigs: "%s". Path checked: "%s"',
+                    rpm_info, self.sigkey_ordering, rpm_path
+                )
+                return
+            return rpm_path
+        else:
+            self.log_warning("RPM %s not found" % rpm_path)
+            return None
+
+    def populate(self, tag, event=None, inherit=True, include_packages=None):
+        result = super().populate(
+            tag=tag,
+            event=event,
+            inherit=inherit,
+            include_packages=include_packages,
+        )
+        return result
+
+
 def _is_src(rpm_info):
     """Check if rpm info object returned by Koji refers to source packages."""
     return rpm_info["arch"] in ("src", "nosrc")

pungi/phases/pkgset/sources/__init__.py

@@ -15,8 +15,10 @@
 
 from .source_koji import PkgsetSourceKoji
 from .source_repos import PkgsetSourceRepos
+from .source_kojimock import PkgsetSourceKojiMock
 
 ALL_SOURCES = {
     "koji": PkgsetSourceKoji,
     "repos": PkgsetSourceRepos,
+    "kojimock": PkgsetSourceKojiMock,
 }

pungi/phases/pkgset/sources/source_koji.py

@@ -193,17 +193,13 @@ class PkgsetSourceKoji(pungi.phases.pkgset.source.PkgsetSourceBase):
     def __call__(self):
         compose = self.compose
         self.koji_wrapper = pungi.wrappers.kojiwrapper.KojiWrapper(compose)
-        # path prefix must contain trailing '/'
-        path_prefix = self.koji_wrapper.koji_module.config.topdir.rstrip("/") + "/"
-        package_sets = get_pkgset_from_koji(
-            self.compose, self.koji_wrapper, path_prefix
-        )
-        return (package_sets, path_prefix)
+        package_sets = get_pkgset_from_koji(self.compose, self.koji_wrapper)
+        return (package_sets, self.compose.koji_downloader.path_prefix)
 
 
-def get_pkgset_from_koji(compose, koji_wrapper, path_prefix):
+def get_pkgset_from_koji(compose, koji_wrapper):
     event_info = get_koji_event_info(compose, koji_wrapper)
-    return populate_global_pkgset(compose, koji_wrapper, path_prefix, event_info)
+    return populate_global_pkgset(compose, koji_wrapper, event_info)
 
 
 def _add_module_to_variant(
@@ -226,20 +222,23 @@ def _add_module_to_variant(
     """
     mmds = {}
     archives = koji_wrapper.koji_proxy.listArchives(build["id"])
+    available_arches = set()
     for archive in archives:
         if archive["btype"] != "module":
             # Skip non module archives
             continue
         typedir = koji_wrapper.koji_module.pathinfo.typedir(build, archive["btype"])
         filename = archive["filename"]
-        file_path = os.path.join(typedir, filename)
+        file_path = compose.koji_downloader.get_file(os.path.join(typedir, filename))
         try:
             # If there are two dots, the arch is in the middle. MBS uploads
             # files with actual architecture in the filename, but Pungi deals
             # in basearch. This assumes that each arch in the build maps to a
             # unique basearch.
             _, arch, _ = filename.split(".")
-            filename = "modulemd.%s.txt" % getBaseArch(arch)
+            basearch = getBaseArch(arch)
+            filename = "modulemd.%s.txt" % basearch
+            available_arches.add(basearch)
         except ValueError:
             pass
         mmds[filename] = file_path
@@ -264,15 +263,26 @@ def _add_module_to_variant(
             compose.log_debug("Module %s is filtered from %s.%s", nsvc, variant, arch)
             continue
 
+        if arch not in available_arches:
+            compose.log_debug(
+                "Module %s is not available for arch %s.%s", nsvc, variant, arch
+            )
+            continue
+
         filename = "modulemd.%s.txt" % arch
         if filename not in mmds:
             raise RuntimeError(
                 "Module %s does not have metadata for arch %s and is not filtered "
                 "out via filter_modules option." % (nsvc, arch)
             )
-        mod_stream = read_single_module_stream_from_file(
-            mmds[filename], compose, arch, build
-        )
+        try:
+            mod_stream = read_single_module_stream_from_file(
+                mmds[filename], compose, arch, build
+            )
+        except Exception as exc:
+            # libmodulemd raises various GLib exceptions with not very helpful
+            # messages. Let's replace it with something more useful.
+            raise RuntimeError("Failed to read %s: %s", mmds[filename], str(exc))
         if mod_stream:
             added = True
         variant.arch_mmds.setdefault(arch, {})[nsvc] = mod_stream
@@ -395,7 +405,13 @@ def _is_filtered_out(compose, variant, arch, module_name, module_stream):
 
 
 def _get_modules_from_koji(
-    compose, koji_wrapper, event, variant, variant_tags, tag_to_mmd, exclude_module_ns
+    compose,
+    koji_wrapper,
+    event,
+    variant,
+    variant_tags,
+    tag_to_mmd,
+    exclude_module_ns,
 ):
     """
     Loads modules for given `variant` from koji `session`, adds them to
@@ -480,7 +496,16 @@ def filter_inherited(koji_proxy, event, module_builds, top_tag):
         # And keep only builds from that topmost tag
         result.extend(build for build in builds if build["tag_name"] == tag)
 
-    return result
+    # If the same module was inherited multiple times, it will be in result
+    # multiple times. We need to deduplicate.
+    deduplicated_result = []
+    included_nvrs = set()
+    for build in result:
+        if build["nvr"] not in included_nvrs:
+            deduplicated_result.append(build)
+            included_nvrs.add(build["nvr"])
+
+    return deduplicated_result
 
 
 def filter_by_whitelist(compose, module_builds, input_modules, expected_modules):
@@ -670,7 +695,7 @@ def _get_modules_from_koji_tags(
         )
 
 
-def populate_global_pkgset(compose, koji_wrapper, path_prefix, event):
+def populate_global_pkgset(compose, koji_wrapper, event):
     all_arches = get_all_arches(compose)
 
     # List of compose tags from which we create this compose
@@ -764,7 +789,12 @@ def populate_global_pkgset(compose, koji_wrapper, path_prefix, event):
 
         if extra_modules:
             _add_extra_modules_to_variant(
-                compose, koji_wrapper, variant, extra_modules, variant_tags, tag_to_mmd
+                compose,
+                koji_wrapper,
+                variant,
+                extra_modules,
+                variant_tags,
+                tag_to_mmd,
             )
 
         variant_scratch_modules = get_variant_data(
@@ -821,6 +851,7 @@ def populate_global_pkgset(compose, koji_wrapper, path_prefix, event):
             cache_region=compose.cache_region,
             signed_packages_retries=compose.conf["signed_packages_retries"],
             signed_packages_wait=compose.conf["signed_packages_wait"],
+            downloader=compose.koji_downloader,
             **kwargs
         )
 
@@ -879,7 +910,13 @@ def populate_global_pkgset(compose, koji_wrapper, path_prefix, event):
         if pkgset.reuse is None:
             pkgset.populate(
                 compose_tag,
-                event,
+                # We care about packages as they existed on the specified
+                # event. However, modular content tags are not expected to
+                # change, so the event doesn't matter there. If an exact NSVC
+                # of a module is specified, the code above would happily find
+                # its content tag, but fail here if the content tag doesn't
+                # exist at the given event.
+                event=event if is_traditional else None,
                 inherit=should_inherit,
                 include_packages=modular_packages,
             )
@@ -907,7 +944,7 @@ def populate_global_pkgset(compose, koji_wrapper, path_prefix, event):
                 MaterializedPackageSet.create,
                 compose,
                 pkgset,
-                path_prefix,
+                compose.koji_downloader.path_prefix,
                 mmd=tag_to_mmd.get(pkgset.name),
             )
         )

pungi/runroot.py

@@ -13,13 +13,19 @@
 # You should have received a copy of the GNU General Public License
 # along with this program; if not, see <https://gnu.org/licenses/>.
 
+import contextlib
 import os
 import re
+import shutil
+import tarfile
+
+import requests
 import six
 from six.moves import shlex_quote
 import kobo.log
 from kobo.shortcuts import run
 
+from pungi import util
 from pungi.wrappers import kojiwrapper
 
 
@@ -94,7 +100,7 @@ class Runroot(kobo.log.LoggingBase):
         log_file = os.path.join(log_dir, "program.log")
         try:
             with open(log_file) as f:
-                for line in f:
+                for line in f.readlines():
                     if "losetup: cannot find an unused loop device" in line:
                         return True
                     if re.match("losetup: .* failed to set up loop device", line):
@@ -230,9 +236,9 @@ class Runroot(kobo.log.LoggingBase):
                 fmt_dict["runroot_key"] = runroot_key
             self._ssh_run(hostname, user, run_template, fmt_dict, log_file=log_file)
 
-            fmt_dict[
-                "command"
-            ] = "rpm -qa --qf='%{name}-%{version}-%{release}.%{arch}\n'"
+            fmt_dict["command"] = (
+                "rpm -qa --qf='%{name}-%{version}-%{release}.%{arch}\n'"
+            )
             buildroot_rpms = self._ssh_run(
                 hostname,
                 user,
@@ -314,7 +320,8 @@ class Runroot(kobo.log.LoggingBase):
             arch,
             args,
             channel=runroot_channel,
-            chown_uid=os.getuid(),
+            # We want to change owner only if shared NFS directory is used.
+            chown_uid=os.getuid() if kwargs.get("mounts") else None,
             **kwargs
         )
 
@@ -325,6 +332,7 @@ class Runroot(kobo.log.LoggingBase):
                 % (output["task_id"], log_file)
             )
         self._result = output
+        return output["task_id"]
 
     def run_pungi_ostree(self, args, log_file=None, arch=None, **kwargs):
         """
@@ -381,3 +389,72 @@ class Runroot(kobo.log.LoggingBase):
             return self._result
         else:
             raise ValueError("Unknown runroot_method %r." % self.runroot_method)
+
+
+@util.retry(wait_on=requests.exceptions.RequestException)
+def _download_file(url, dest):
+    # contextlib.closing is only needed in requests<2.18
+    with contextlib.closing(requests.get(url, stream=True, timeout=5)) as r:
+        if r.status_code == 404:
+            raise RuntimeError("Archive %s not found" % url)
+        r.raise_for_status()
+        with open(dest, "wb") as f:
+            shutil.copyfileobj(r.raw, f)
+
+
+def _download_archive(task_id, fname, archive_url, dest_dir):
+    """Download file from URL to a destination, with retries."""
+    temp_file = os.path.join(dest_dir, fname)
+    _download_file(archive_url, temp_file)
+    return temp_file
+
+
+def _extract_archive(task_id, fname, archive_file, dest_path):
+    """Extract the archive into given destination.
+
+    All items of the archive must match the name of the archive, i.e. all
+    paths in foo.tar.gz must start with foo/.
+    """
+    basename = os.path.basename(fname).split(".")[0]
+    strip_prefix = basename + "/"
+    with tarfile.open(archive_file, "r") as archive:
+        for member in archive.getmembers():
+            # Check if each item is either the root directory or is within it.
+            if member.name != basename and not member.name.startswith(strip_prefix):
+                raise RuntimeError(
+                    "Archive %s from task %s contains file without expected prefix: %s"
+                    % (fname, task_id, member)
+                )
+            dest = os.path.join(dest_path, member.name[len(strip_prefix) :])
+            if member.isdir():
+                # Create directories where needed...
+                util.makedirs(dest)
+            elif member.isfile():
+                # ... and extract files into them.
+                with open(dest, "wb") as dest_obj:
+                    shutil.copyfileobj(archive.extractfile(member), dest_obj)
+            elif member.islnk():
+                # We have a hardlink. Let's also link it.
+                linked_file = os.path.join(
+                    dest_path, member.linkname[len(strip_prefix) :]
+                )
+                os.link(linked_file, dest)
+            else:
+                # Any other file type is an error.
+                raise RuntimeError(
+                    "Unexpected file type in %s from task %s: %s"
+                    % (fname, task_id, member)
+                )
+
+
+def download_and_extract_archive(compose, task_id, fname, destination):
+    """Download a tar archive from task outputs and extract it to the destination."""
+    koji = kojiwrapper.KojiWrapper(compose).koji_module
+    # Koji API provides downloadTaskOutput method, but it's not usable as it
+    # will attempt to load the entire file into memory.
+    # So instead let's generate a patch and attempt to convert it to a URL.
+    server_path = os.path.join(koji.pathinfo.task(task_id), fname)
+    archive_url = server_path.replace(koji.config.topdir, koji.config.topurl)
+    with util.temp_dir(prefix="buildinstall-download") as tmp_dir:
+        local_path = _download_archive(task_id, fname, archive_url, tmp_dir)
+        _extract_archive(task_id, fname, local_path, destination)

pungi/scripts/cache_cleanup.py

@@ -0,0 +1,63 @@
+import argparse
+import os
+import re
+import time
+
+from pungi.util import format_size
+
+
+LOCK_RE = re.compile(r".*\.lock(\|[A-Za-z0-9]+)*$")
+
+
+def should_be_cleaned_up(path, st, threshold):
+    if st.st_nlink == 1 and st.st_mtime < threshold:
+        # No other instances, older than limit
+        return True
+
+    if LOCK_RE.match(path) and st.st_mtime < threshold:
+        # Suspiciously old lock
+        return True
+
+    return False
+
+
+def main():
+    parser = argparse.ArgumentParser()
+    parser.add_argument("CACHE_DIR")
+    parser.add_argument("-n", "--dry-run", action="store_true")
+    parser.add_argument("--verbose", action="store_true")
+    parser.add_argument(
+        "--max-age",
+        help="how old files should be considered for deletion",
+        default=7,
+        type=int,
+    )
+
+    args = parser.parse_args()
+
+    topdir = os.path.abspath(args.CACHE_DIR)
+    max_age = args.max_age * 24 * 3600
+
+    cleaned_up = 0
+
+    threshold = time.time() - max_age
+    for dirpath, dirnames, filenames in os.walk(topdir):
+        for f in filenames:
+            filepath = os.path.join(dirpath, f)
+            st = os.stat(filepath)
+            if should_be_cleaned_up(filepath, st, threshold):
+                if args.verbose:
+                    print("RM %s" % filepath)
+                cleaned_up += st.st_size
+                if not args.dry_run:
+                    os.remove(filepath)
+        if not dirnames and not filenames:
+            if args.verbose:
+                print("RMDIR %s" % dirpath)
+            if not args.dry_run:
+                os.rmdir(dirpath)
+
+    if args.dry_run:
+        print("Would reclaim %s bytes." % format_size(cleaned_up))
+    else:
+        print("Reclaimed %s bytes." % format_size(cleaned_up))

pungi/scripts/config_validate.py

@@ -128,7 +128,6 @@ def run(config, topdir, has_old, offline, defined_variables, schema_overrides):
         pungi.phases.OSTreePhase(compose),
         pungi.phases.CreateisoPhase(compose, buildinstall_phase),
         pungi.phases.ExtraIsosPhase(compose, buildinstall_phase),
-        pungi.phases.LiveImagesPhase(compose),
         pungi.phases.LiveMediaPhase(compose),
         pungi.phases.ImageBuildPhase(compose),
         pungi.phases.ImageChecksumPhase(compose),

pungi/scripts/create_extra_repo.py

@@ -0,0 +1,441 @@
+# coding=utf-8
+
+import argparse
+import os
+import subprocess
+import tempfile
+from shutil import rmtree
+from typing import (
+    AnyStr,
+    List,
+    Dict,
+    Optional,
+)
+
+import createrepo_c as cr
+import requests
+import yaml
+from dataclasses import dataclass, field
+
+from .create_packages_json import (
+    PackagesGenerator,
+    RepoInfo,
+    VariantInfo,
+)
+
+
+@dataclass
+class ExtraVariantInfo(VariantInfo):
+
+    modules: List[AnyStr] = field(default_factory=list)
+    packages: List[AnyStr] = field(default_factory=list)
+
+
+class CreateExtraRepo(PackagesGenerator):
+
+    def __init__(
+            self,
+            variants: List[ExtraVariantInfo],
+            bs_auth_token: AnyStr,
+            local_repository_path: AnyStr,
+            clear_target_repo: bool = True,
+    ):
+        self.variants = []  # type: List[ExtraVariantInfo]
+        super().__init__(variants, [], [])
+        self.auth_headers = {
+            'Authorization': f'Bearer {bs_auth_token}',
+        }
+        # modules data of modules.yaml.gz from an existing local repo
+        self.local_modules_data = []
+        self.local_repository_path = local_repository_path
+        # path to modules.yaml, which generated by the class
+        self.default_modules_yaml_path = os.path.join(
+            local_repository_path,
+            'modules.yaml',
+        )
+        if clear_target_repo:
+            if os.path.exists(self.local_repository_path):
+                rmtree(self.local_repository_path)
+            os.makedirs(self.local_repository_path, exist_ok=True)
+        else:
+            self._read_local_modules_yaml()
+
+    def _read_local_modules_yaml(self):
+        """
+        Read modules data from an existin local repo
+        """
+        repomd_file_path = os.path.join(
+            self.local_repository_path,
+            'repodata',
+            'repomd.xml',
+        )
+        repomd_object = self._parse_repomd(repomd_file_path)
+        for repomd_record in repomd_object.records:
+            if repomd_record.type != 'modules':
+                continue
+            modules_yaml_path = os.path.join(
+                self.local_repository_path,
+                repomd_record.location_href,
+            )
+            self.local_modules_data = list(self._parse_modules_file(
+                modules_yaml_path,
+            ))
+            break
+
+    def _dump_local_modules_yaml(self):
+        """
+        Dump merged modules data to an local repo
+        """
+        if self.local_modules_data:
+            with open(self.default_modules_yaml_path, 'w') as yaml_file:
+                yaml.dump_all(
+                    self.local_modules_data,
+                    yaml_file,
+                )
+
+    @staticmethod
+    def get_repo_info_from_bs_repo(
+            auth_token: AnyStr,
+            build_id: AnyStr,
+            arch: AnyStr,
+            packages: Optional[List[AnyStr]] = None,
+            modules: Optional[List[AnyStr]] = None,
+    ) -> List[ExtraVariantInfo]:
+        """
+        Get info about a BS repo and save it to
+            an object of class ExtraRepoInfo
+        :param auth_token: Auth token to Build System
+        :param build_id: ID of a build from BS
+        :param arch: an architecture of repo which will be used
+        :param packages: list of names of packages which will be put to an
+                         local repo from a BS repo
+        :param modules: list of names of modules which will be put to an
+                        local repo from a BS repo
+        :return: list of ExtraRepoInfo with info about the BS repos
+        """
+
+        bs_url = 'https://build.cloudlinux.com'
+        api_uri = 'api/v1'
+        bs_repo_suffix = 'build_repos'
+
+        variants_info = []
+
+        # get the full info about a BS repo
+        repo_request = requests.get(
+            url=os.path.join(
+                bs_url,
+                api_uri,
+                'builds',
+                build_id,
+            ),
+            headers={
+                'Authorization': f'Bearer {auth_token}',
+            },
+        )
+        repo_request.raise_for_status()
+        result = repo_request.json()
+        for build_platform in result['build_platforms']:
+            platform_name = build_platform['name']
+            for architecture in build_platform['architectures']:
+                # skip repo with unsuitable architecture
+                if architecture != arch:
+                    continue
+                variant_info = ExtraVariantInfo(
+                    name=f'{build_id}-{platform_name}-{architecture}',
+                    arch=architecture,
+                    packages=packages,
+                    modules=modules,
+                    repos=[
+                        RepoInfo(
+                            path=os.path.join(
+                                bs_url,
+                                bs_repo_suffix,
+                                build_id,
+                                platform_name,
+                            ),
+                            folder=architecture,
+                            is_remote=True,
+                        )
+                    ]
+                )
+                variants_info.append(variant_info)
+        return variants_info
+
+    def _create_local_extra_repo(self):
+        """
+        Call `createrepo_c <path_to_repo>` for creating a local repo
+        """
+        subprocess.call(
+            f'createrepo_c {self.local_repository_path}',
+            shell=True,
+        )
+        # remove an unnecessary temporary modules.yaml
+        if os.path.exists(self.default_modules_yaml_path):
+            os.remove(self.default_modules_yaml_path)
+
+    def get_remote_file_content(
+            self,
+            file_url: AnyStr,
+    ) -> AnyStr:
+        """
+        Get content from a remote file and write it to a temp file
+        :param file_url: url of a remote file
+        :return: path to a temp file
+        """
+
+        file_request = requests.get(
+            url=file_url,
+            # for the case when we get a file from BS
+            headers=self.auth_headers,
+        )
+        file_request.raise_for_status()
+        with tempfile.NamedTemporaryFile(delete=False) as file_stream:
+            file_stream.write(file_request.content)
+        return file_stream.name
+
+    def _download_rpm_to_local_repo(
+            self,
+            package_location: AnyStr,
+            repo_info: RepoInfo,
+    ) -> None:
+        """
+        Download a rpm package from a remote repo and save it to a local repo
+        :param package_location: relative uri of a package in a remote repo
+        :param repo_info: info about a remote repo which contains a specific
+                          rpm package
+        """
+        rpm_package_remote_path = os.path.join(
+            repo_info.path,
+            repo_info.folder,
+            package_location,
+        )
+        rpm_package_local_path = os.path.join(
+            self.local_repository_path,
+            os.path.basename(package_location),
+        )
+        rpm_request = requests.get(
+            url=rpm_package_remote_path,
+            headers=self.auth_headers,
+        )
+        rpm_request.raise_for_status()
+        with open(rpm_package_local_path, 'wb') as rpm_file:
+            rpm_file.write(rpm_request.content)
+
+    def _download_packages(
+            self,
+            packages: Dict[AnyStr, cr.Package],
+            variant_info: ExtraVariantInfo
+    ):
+        """
+        Download all defined packages from a remote repo
+        :param packages: information about all packages (including
+                         modularity) in a remote repo
+        :param variant_info: information about a remote variant
+        """
+        for package in packages.values():
+            package_name = package.name
+            # Skip a current package from a remote repo if we defined
+            # the list packages and a current package doesn't belong to it
+            if variant_info.packages and \
+                    package_name not in variant_info.packages:
+                continue
+            for repo_info in variant_info.repos:
+                self._download_rpm_to_local_repo(
+                    package_location=package.location_href,
+                    repo_info=repo_info,
+                )
+
+    def _download_modules(
+            self,
+            modules_data: List[Dict],
+            variant_info: ExtraVariantInfo,
+            packages: Dict[AnyStr, cr.Package]
+    ):
+        """
+        Download all defined modularity packages and their data from
+            a remote repo
+        :param modules_data: information about all modules in a remote repo
+        :param variant_info: information about a remote variant
+        :param packages: information about all packages (including
+                         modularity) in a remote repo
+        """
+        for module in modules_data:
+            module_data = module['data']
+            # Skip a current module from a remote repo if we defined
+            # the list modules and a current module doesn't belong to it
+            if variant_info.modules and \
+                    module_data['name'] not in variant_info.modules:
+                continue
+            # we should add info about a module if the local repodata
+            # doesn't have it
+            if module not in self.local_modules_data:
+                self.local_modules_data.append(module)
+            # just skip a module's record if it doesn't have rpm artifact
+            if module['document'] != 'modulemd' or \
+                    'artifacts' not in module_data or \
+                    'rpms' not in module_data['artifacts']:
+                continue
+            for rpm in module['data']['artifacts']['rpms']:
+                # Empty repo_info.packages means that we will download
+                # all packages from repo including
+                # the modularity packages
+                if not variant_info.packages:
+                    break
+                # skip a rpm if it doesn't belong to a processed repo
+                if rpm not in packages:
+                    continue
+                for repo_info in variant_info.repos:
+                    self._download_rpm_to_local_repo(
+                        package_location=packages[rpm].location_href,
+                        repo_info=repo_info,
+                    )
+
+    def create_extra_repo(self):
+        """
+        1. Get from the remote repos the specific (or all) packages/modules
+        2. Save them to a local repo
+        3. Save info about the modules to a local repo
+        3. Call `createrepo_c` which creates a local repo
+           with the right repodata
+        """
+        for variant_info in self.variants:
+            for repo_info in variant_info.repos:
+                repomd_records = self._get_repomd_records(
+                    repo_info=repo_info,
+                )
+                packages_iterator = self.get_packages_iterator(repo_info)
+                # parse the repodata (including modules.yaml.gz)
+                modules_data = self._parse_module_repomd_record(
+                    repo_info=repo_info,
+                    repomd_records=repomd_records,
+                )
+                # convert the packages dict to more usable form
+                # for future checking that a rpm from the module's artifacts
+                # belongs to a processed repository
+                packages = {
+                    f'{package.name}-{package.epoch}:{package.version}-'
+                    f'{package.release}.{package.arch}':
+                        package for package in packages_iterator
+                }
+                self._download_modules(
+                    modules_data=modules_data,
+                    variant_info=variant_info,
+                    packages=packages,
+                )
+                self._download_packages(
+                    packages=packages,
+                    variant_info=variant_info,
+                )
+
+        self._dump_local_modules_yaml()
+        self._create_local_extra_repo()
+
+
+def create_parser():
+    parser = argparse.ArgumentParser()
+    parser.add_argument(
+        '--bs-auth-token',
+        help='Auth token for Build System',
+    )
+    parser.add_argument(
+        '--local-repo-path',
+        help='Path to a local repo. E.g. /var/repo/test_repo',
+        required=True,
+    )
+    parser.add_argument(
+        '--clear-local-repo',
+        help='Clear a local repo before creating a new',
+        action='store_true',
+        default=False,
+    )
+    parser.add_argument(
+        '--repo',
+        action='append',
+        help='Path to a folder with repofolders or build id. E.g. '
+             '"http://koji.cloudlinux.com/mirrors/rhel_mirror" or '
+             '"601809b3c2f5b0e458b14cd3"',
+        required=True,
+    )
+    parser.add_argument(
+        '--repo-folder',
+        action='append',
+        help='A folder which contains folder repodata . E.g. "baseos-stream"',
+        required=True,
+    )
+    parser.add_argument(
+        '--repo-arch',
+        action='append',
+        help='What architecture packages a repository contains. E.g. "x86_64"',
+        required=True,
+    )
+    parser.add_argument(
+        '--packages',
+        action='append',
+        type=str,
+        default=[],
+        help='A list of packages names which we want to download to local '
+             'extra repo. We will download all of packages if param is empty',
+        required=True,
+    )
+    parser.add_argument(
+        '--modules',
+        action='append',
+        type=str,
+        default=[],
+        help='A list of modules names which we want to download to local '
+             'extra repo. We will download all of modules if param is empty',
+        required=True,
+    )
+
+    return parser
+
+
+def cli_main():
+    args = create_parser().parse_args()
+    repos_info = []
+    for repo, repo_folder, repo_arch, packages, modules in zip(
+        args.repo,
+        args.repo_folder,
+        args.repo_arch,
+        args.packages,
+        args.modules,
+    ):
+        modules = modules.split()
+        packages = packages.split()
+        if repo.startswith('http://'):
+            repos_info.append(
+                ExtraVariantInfo(
+                    name=repo_folder,
+                    arch=repo_arch,
+                    repos=[
+                        RepoInfo(
+                            path=repo,
+                            folder=repo_folder,
+                            is_remote=True,
+                        )
+                    ],
+                    modules=modules,
+                    packages=packages,
+                )
+            )
+        else:
+            repos_info.extend(
+                CreateExtraRepo.get_repo_info_from_bs_repo(
+                    auth_token=args.bs_auth_token,
+                    build_id=repo,
+                    arch=repo_arch,
+                    modules=modules,
+                    packages=packages,
+                )
+            )
+    cer = CreateExtraRepo(
+        variants=repos_info,
+        bs_auth_token=args.bs_auth_token,
+        local_repository_path=args.local_repo_path,
+        clear_target_repo=args.clear_local_repo,
+    )
+    cer.create_extra_repo()
+
+
+if __name__ == '__main__':
+    cli_main()

pungi/scripts/create_packages_json.py

@@ -0,0 +1,514 @@
+# coding=utf-8
+"""
+The tool allow to generate package.json. This file is used by pungi
+# as parameter `gather_prepopulate`
+Sample of using repodata files taken from
+https://github.com/rpm-software-management/createrepo_c/blob/master/examples/python/repodata_parsing.py
+"""
+
+import argparse
+import gzip
+import json
+import logging
+import lzma
+import os
+import re
+import tempfile
+from collections import defaultdict
+from itertools import tee
+from pathlib import Path
+from typing import (
+    AnyStr,
+    Dict,
+    List,
+    Any,
+    Iterator,
+    Optional,
+    Tuple,
+    Union,
+)
+
+import binascii
+from urllib.parse import urljoin
+
+import requests
+import rpm
+import yaml
+from createrepo_c import (
+    Package,
+    PackageIterator,
+    Repomd,
+    RepomdRecord,
+)
+from dataclasses import dataclass, field
+from kobo.rpmlib import parse_nvra
+
+logging.basicConfig(level=logging.INFO)
+
+
+def _is_compressed_file(first_two_bytes: bytes, initial_bytes: bytes):
+    return binascii.hexlify(first_two_bytes) == initial_bytes
+
+
+def is_gzip_file(first_two_bytes):
+    return _is_compressed_file(
+        first_two_bytes=first_two_bytes,
+        initial_bytes=b'1f8b',
+    )
+
+
+def is_xz_file(first_two_bytes):
+    return _is_compressed_file(
+        first_two_bytes=first_two_bytes,
+        initial_bytes=b'fd37',
+    )
+
+
+@dataclass
+class RepoInfo:
+    # path to a directory with repo directories. E.g. '/var/repos' contains
+    # 'appstream', 'baseos', etc.
+    # Or 'http://koji.cloudlinux.com/mirrors/rhel_mirror' if you are
+    # using remote repo
+    path: str
+    # name of folder with a repodata folder. E.g. 'baseos', 'appstream', etc
+    folder: str
+    # Is a repo remote or local
+    is_remote: bool
+    # Is a reference repository (usually it's a RHEL repo)
+    # Layout of packages from such repository will be taken as example
+    # Only layout of specific package (which doesn't exist
+    # in a reference repository) will be taken as example
+    is_reference: bool = False
+    # The packages from 'present' repo will be added to a variant.
+    # The packages from 'absent' repo will be removed from a variant.
+    repo_type: str = 'present'
+
+
+@dataclass
+class VariantInfo:
+    # name of variant. E.g. 'BaseOS', 'AppStream', etc
+    name: AnyStr
+    # architecture of variant. E.g. 'x86_64', 'i686', etc
+    arch: AnyStr
+    # The packages which will be not added to a variant
+    excluded_packages: List[str] = field(default_factory=list)
+    # Repos of a variant
+    repos: List[RepoInfo] = field(default_factory=list)
+
+
+class PackagesGenerator:
+
+    repo_arches = defaultdict(lambda: list(('noarch',)))
+    addon_repos = {
+        'x86_64': ['i686'],
+        'ppc64le': [],
+        'aarch64': [],
+        's390x': [],
+        'i686': [],
+    }
+
+    def __init__(
+        self,
+        variants: List[VariantInfo],
+        excluded_packages: List[AnyStr],
+        included_packages: List[AnyStr],
+    ):
+        self.variants = variants
+        self.pkgs = dict()
+        self.excluded_packages = excluded_packages
+        self.included_packages = included_packages
+        self.tmp_files = []  # type: list[Path]
+        for arch, arch_list in self.addon_repos.items():
+            self.repo_arches[arch].extend(arch_list)
+            self.repo_arches[arch].append(arch)
+
+    def __del__(self):
+        for tmp_file in self.tmp_files:
+            if tmp_file.exists():
+                tmp_file.unlink()
+
+    @staticmethod
+    def _get_full_repo_path(repo_info: RepoInfo):
+        result = os.path.join(
+            repo_info.path,
+            repo_info.folder
+        )
+        if repo_info.is_remote:
+            result = urljoin(
+                repo_info.path + '/',
+                repo_info.folder,
+            )
+        return result
+
+    @staticmethod
+    def _warning_callback(warning_type, message):
+        """
+        Warning callback for createrepo_c parsing functions
+        """
+        print(f'Warning message: "{message}"; warning type: "{warning_type}"')
+        return True
+
+    def get_remote_file_content(self, file_url: AnyStr) -> AnyStr:
+        """
+        Get content from a remote file and write it to a temp file
+        :param file_url: url of a remote file
+        :return: path to a temp file
+        """
+
+        file_request = requests.get(
+            url=file_url,
+        )
+        file_request.raise_for_status()
+        with tempfile.NamedTemporaryFile(delete=False) as file_stream:
+            file_stream.write(file_request.content)
+        self.tmp_files.append(Path(file_stream.name))
+        return file_stream.name
+
+    @staticmethod
+    def _parse_repomd(repomd_file_path: AnyStr) -> Repomd:
+        """
+        Parse file repomd.xml and create object Repomd
+        :param repomd_file_path: path to local repomd.xml
+        """
+        return Repomd(repomd_file_path)
+
+    @classmethod
+    def _parse_modules_file(
+            cls,
+            modules_file_path: AnyStr,
+
+    ) -> Iterator[Any]:
+        """
+        Parse modules.yaml.gz and returns parsed data
+        :param modules_file_path: path to local modules.yaml.gz
+        :return: List of dict for each module in a repo
+        """
+
+        with open(modules_file_path, 'rb') as modules_file:
+            data = modules_file.read()
+            if is_gzip_file(data[:2]):
+                data = gzip.decompress(data)
+            elif is_xz_file(data[:2]):
+                data = lzma.decompress(data)
+            return yaml.load_all(
+                data,
+                Loader=yaml.BaseLoader,
+            )
+
+    def _get_repomd_records(
+            self,
+            repo_info: RepoInfo,
+    ) -> List[RepomdRecord]:
+        """
+        Get, parse file repomd.xml and extract from it repomd records
+        :param repo_info: structure which contains info about a current repo
+        :return: list with repomd records
+        """
+        repomd_file_path = os.path.join(
+            repo_info.path,
+            repo_info.folder,
+            'repodata',
+            'repomd.xml',
+        )
+        if repo_info.is_remote:
+            repomd_file_path = urljoin(
+                urljoin(
+                    repo_info.path + '/',
+                    repo_info.folder
+                ) + '/',
+                'repodata/repomd.xml'
+            )
+            repomd_file_path = self.get_remote_file_content(repomd_file_path)
+
+        repomd_object = self._parse_repomd(repomd_file_path)
+        if repo_info.is_remote:
+            os.remove(repomd_file_path)
+        return repomd_object.records
+
+    def _download_repomd_records(
+            self,
+            repo_info: RepoInfo,
+            repomd_records: List[RepomdRecord],
+            repomd_records_dict: Dict[str, str],
+    ):
+        """
+        Download repomd records
+        :param repo_info: structure which contains info about a current repo
+        :param repomd_records: list with repomd records
+        :param repomd_records_dict: dict with paths to repodata files
+        """
+        for repomd_record in repomd_records:
+            if repomd_record.type not in (
+                    'primary',
+                    'filelists',
+                    'other',
+            ):
+                continue
+            repomd_record_file_path = os.path.join(
+                repo_info.path,
+                repo_info.folder,
+                repomd_record.location_href,
+            )
+            if repo_info.is_remote:
+                repomd_record_file_path = self.get_remote_file_content(
+                    repomd_record_file_path)
+            repomd_records_dict[repomd_record.type] = repomd_record_file_path
+
+    def _parse_module_repomd_record(
+            self,
+            repo_info: RepoInfo,
+            repomd_records: List[RepomdRecord],
+    ) -> List[Dict]:
+        """
+        Download repomd records
+        :param repo_info: structure which contains info about a current repo
+        :param repomd_records: list with repomd records
+        """
+        for repomd_record in repomd_records:
+            if repomd_record.type != 'modules':
+                continue
+            repomd_record_file_path = os.path.join(
+                repo_info.path,
+                repo_info.folder,
+                repomd_record.location_href,
+            )
+            if repo_info.is_remote:
+                repomd_record_file_path = self.get_remote_file_content(
+                    repomd_record_file_path)
+            return list(self._parse_modules_file(
+                repomd_record_file_path,
+            ))
+        return []
+
+    @staticmethod
+    def compare_pkgs_version(package_1: Package, package_2: Package) -> int:
+        version_tuple_1 = (
+            package_1.epoch,
+            package_1.version,
+            package_1.release,
+        )
+        version_tuple_2 = (
+            package_2.epoch,
+            package_2.version,
+            package_2.release,
+        )
+        return rpm.labelCompare(version_tuple_1, version_tuple_2)
+
+    def get_packages_iterator(
+            self,
+            repo_info: RepoInfo,
+    ) -> Union[PackageIterator, Iterator]:
+        full_repo_path = self._get_full_repo_path(repo_info)
+        pkgs_iterator = self.pkgs.get(full_repo_path)
+        if pkgs_iterator is None:
+            repomd_records = self._get_repomd_records(
+                repo_info=repo_info,
+            )
+            repomd_records_dict = {}  # type: Dict[str, str]
+            self._download_repomd_records(
+                repo_info=repo_info,
+                repomd_records=repomd_records,
+                repomd_records_dict=repomd_records_dict,
+            )
+            pkgs_iterator = PackageIterator(
+                primary_path=repomd_records_dict['primary'],
+                filelists_path=repomd_records_dict['filelists'],
+                other_path=repomd_records_dict['other'],
+                warningcb=self._warning_callback,
+            )
+            pkgs_iterator, self.pkgs[full_repo_path] = tee(pkgs_iterator)
+        return pkgs_iterator
+
+    def get_package_arch(
+            self,
+            package: Package,
+            variant_arch: str,
+    ) -> str:
+        result = variant_arch
+        if package.arch in self.repo_arches[variant_arch]:
+            result = package.arch
+        return result
+
+    def is_skipped_module_package(
+            self,
+            package: Package,
+            variant_arch: str,
+    ) -> bool:
+        package_key = self.get_package_key(package, variant_arch)
+        # Even a module package will be added to packages.json if
+        # it presents in the list of included packages
+        return 'module' in package.release and not any(
+            re.search(
+                f'^{included_pkg}$',
+                package_key,
+            ) or included_pkg in (package.name, package_key)
+            for included_pkg in self.included_packages
+        )
+
+    def is_excluded_package(
+            self,
+            package: Package,
+            variant_arch: str,
+            excluded_packages: List[str],
+    ) -> bool:
+        package_key = self.get_package_key(package, variant_arch)
+        return any(
+            re.search(
+                f'^{excluded_pkg}$',
+                package_key,
+            ) or excluded_pkg in (package.name, package_key)
+            for excluded_pkg in excluded_packages
+        )
+
+    @staticmethod
+    def get_source_rpm_name(package: Package) -> str:
+        source_rpm_nvra = parse_nvra(package.rpm_sourcerpm)
+        return source_rpm_nvra['name']
+
+    def get_package_key(self, package: Package, variant_arch: str) -> str:
+        return (
+            f'{package.name}.'
+            f'{self.get_package_arch(package, variant_arch)}'
+        )
+
+    def generate_packages_json(
+            self
+    ) -> Dict[AnyStr, Dict[AnyStr, Dict[AnyStr, List[AnyStr]]]]:
+        """
+        Generate packages.json
+        """
+        packages = defaultdict(lambda: defaultdict(lambda: {
+            'variants': list(),
+        }))
+        for variant_info in self.variants:
+            for repo_info in variant_info.repos:
+                is_reference = repo_info.is_reference
+                for package in self.get_packages_iterator(repo_info=repo_info):
+                    if self.is_skipped_module_package(
+                            package=package,
+                            variant_arch=variant_info.arch,
+                    ):
+                        continue
+                    if self.is_excluded_package(
+                        package=package,
+                        variant_arch=variant_info.arch,
+                        excluded_packages=self.excluded_packages,
+                    ):
+                        continue
+                    if self.is_excluded_package(
+                        package=package,
+                        variant_arch=variant_info.arch,
+                        excluded_packages=variant_info.excluded_packages,
+                    ):
+                        continue
+                    package_key = self.get_package_key(
+                        package,
+                        variant_info.arch,
+                    )
+                    source_rpm_name = self.get_source_rpm_name(package)
+                    package_info = packages[source_rpm_name][package_key]
+                    if 'is_reference' not in package_info:
+                        package_info['variants'].append(variant_info.name)
+                        package_info['is_reference'] = is_reference
+                        package_info['package'] = package
+                    elif not package_info['is_reference'] or \
+                            package_info['is_reference'] == is_reference and \
+                            self.compare_pkgs_version(
+                        package_1=package,
+                        package_2=package_info['package'],
+                    ) > 0:
+                        package_info['variants'] = [variant_info.name]
+                        package_info['is_reference'] = is_reference
+                        package_info['package'] = package
+                    elif self.compare_pkgs_version(
+                        package_1=package,
+                        package_2=package_info['package'],
+                    ) == 0 and repo_info.repo_type != 'absent':
+                        package_info['variants'].append(variant_info.name)
+        result = defaultdict(lambda: defaultdict(
+            lambda: defaultdict(list),
+        ))
+        for variant_info in self.variants:
+            for source_rpm_name, packages_info in packages.items():
+                for package_key, package_info in packages_info.items():
+                    variant_pkgs = result[variant_info.name][variant_info.arch]
+                    if variant_info.name not in package_info['variants']:
+                        continue
+                    variant_pkgs[source_rpm_name].append(package_key)
+        return result
+
+
+def create_parser():
+    parser = argparse.ArgumentParser()
+    parser.add_argument(
+        '-c',
+        '--config',
+        type=Path,
+        default=Path('config.yaml'),
+        required=False,
+        help='Path to a config',
+    )
+    parser.add_argument(
+        '-o',
+        '--json-output-path',
+        type=str,
+        help='Full path to output json file',
+        required=True,
+    )
+
+    return parser
+
+
+def read_config(config_path: Path) -> Optional[Dict]:
+    if not config_path.exists():
+        logging.error('A config by path "%s" does not exist', config_path)
+        exit(1)
+    with config_path.open('r') as config_fd:
+        return yaml.safe_load(config_fd)
+
+
+def process_config(config_data: Dict) -> Tuple[
+    List[VariantInfo],
+    List[str],
+    List[str],
+]:
+    excluded_packages = config_data.get('excluded_packages', [])
+    included_packages = config_data.get('included_packages', [])
+    variants = [VariantInfo(
+        name=variant_name,
+        arch=variant_info['arch'],
+        excluded_packages=variant_info.get('excluded_packages', []),
+        repos=[RepoInfo(
+            path=variant_repo['path'],
+            folder=variant_repo['folder'],
+            is_remote=variant_repo['remote'],
+            is_reference=variant_repo['reference'],
+            repo_type=variant_repo.get('repo_type', 'present'),
+        ) for variant_repo in variant_info['repos']]
+    ) for variant_name, variant_info in config_data['variants'].items()]
+    return variants, excluded_packages, included_packages
+
+
+def cli_main():
+    args = create_parser().parse_args()
+    variants, excluded_packages, included_packages = process_config(
+        config_data=read_config(args.config)
+    )
+    pg = PackagesGenerator(
+        variants=variants,
+        excluded_packages=excluded_packages,
+        included_packages=included_packages,
+    )
+    result = pg.generate_packages_json()
+    with open(args.json_output_path, 'w') as packages_file:
+        json.dump(
+            result,
+            packages_file,
+            indent=4,
+            sort_keys=True,
+        )
+
+
+if __name__ == '__main__':
+    cli_main()

pungi/scripts/gather_modules.py

@@ -0,0 +1,255 @@
+import gzip
+import lzma
+import os
+from argparse import ArgumentParser, FileType
+from glob import iglob
+from io import BytesIO
+from pathlib import Path
+from typing import List, AnyStr, Iterable, Union, Optional
+import logging
+from urllib.parse import urljoin
+
+import yaml
+import createrepo_c as cr
+from typing.io import BinaryIO
+
+from .create_packages_json import PackagesGenerator, is_gzip_file, is_xz_file
+
+EMPTY_FILE = '.empty'
+
+
+def read_modules_yaml(modules_yaml_path: Union[str, Path]) -> BytesIO:
+    with open(modules_yaml_path, 'rb') as fp:
+        return BytesIO(fp.read())
+
+
+def grep_list_of_modules_yaml(repos_path: AnyStr) -> Iterable[BytesIO]:
+    """
+    Find all of valid *modules.yaml.gz in repos
+    :param repos_path: path to a directory which contains repo dirs
+    :return: iterable object of content from *modules.yaml.*
+    """
+
+    return (
+        read_modules_yaml_from_specific_repo(repo_path=Path(path).parent)
+        for path in iglob(
+            str(Path(repos_path).joinpath('**/repodata')),
+            recursive=True
+        )
+    )
+
+
+def _is_remote(path: str):
+    return any(str(path).startswith(protocol)
+               for protocol in ('http', 'https'))
+
+
+def read_modules_yaml_from_specific_repo(
+        repo_path: Union[str, Path]
+) -> Optional[BytesIO]:
+    """
+    Read modules_yaml from a specific repo (remote or local)
+    :param repo_path: path/url to a specific repo
+                      (final dir should contain dir `repodata`)
+    :return: iterable object of content from *modules.yaml.*
+    """
+
+    if _is_remote(repo_path):
+        repomd_url = urljoin(
+            repo_path + '/',
+            'repodata/repomd.xml',
+        )
+        packages_generator = PackagesGenerator(
+            variants=[],
+            excluded_packages=[],
+            included_packages=[],
+        )
+        repomd_file_path = packages_generator.get_remote_file_content(
+            file_url=repomd_url
+        )
+    else:
+        repomd_file_path = os.path.join(
+            repo_path,
+            'repodata/repomd.xml',
+        )
+    repomd_obj = cr.Repomd(str(repomd_file_path))
+    for record in repomd_obj.records:
+        if record.type != 'modules':
+            continue
+        else:
+            if _is_remote(repo_path):
+                modules_yaml_url = urljoin(
+                    repo_path + '/',
+                    record.location_href,
+                )
+                packages_generator = PackagesGenerator(
+                    variants=[],
+                    excluded_packages=[],
+                    included_packages=[],
+                )
+                modules_yaml_path = packages_generator.get_remote_file_content(
+                    file_url=modules_yaml_url
+                )
+            else:
+                modules_yaml_path = os.path.join(
+                    repo_path,
+                    record.location_href,
+                )
+            return read_modules_yaml(modules_yaml_path=modules_yaml_path)
+    else:
+        return None
+
+
+def _should_grep_defaults(
+        document_type: str,
+        grep_only_modules_data: bool = False,
+        grep_only_modules_defaults_data: bool = False,
+) -> bool:
+    xor_flag = grep_only_modules_data == grep_only_modules_defaults_data
+    if document_type == 'modulemd' and (xor_flag or grep_only_modules_data):
+        return True
+    return False
+
+
+def _should_grep_modules(
+        document_type: str,
+        grep_only_modules_data: bool = False,
+        grep_only_modules_defaults_data: bool = False,
+) -> bool:
+    xor_flag = grep_only_modules_data == grep_only_modules_defaults_data
+    if document_type == 'modulemd-defaults' and \
+            (xor_flag or grep_only_modules_defaults_data):
+        return True
+    return False
+
+
+def collect_modules(
+        modules_paths: List[BinaryIO],
+        target_dir: str,
+        grep_only_modules_data: bool = False,
+        grep_only_modules_defaults_data: bool = False,
+):
+    """
+    Read given modules.yaml.gz files and export modules
+    and modulemd files from it.
+    Returns:
+        object:
+    """
+    xor_flag = grep_only_modules_defaults_data is grep_only_modules_data
+    modules_path = os.path.join(target_dir, 'modules')
+    module_defaults_path = os.path.join(target_dir, 'module_defaults')
+    if grep_only_modules_data or xor_flag:
+        os.makedirs(modules_path, exist_ok=True)
+    if grep_only_modules_defaults_data or xor_flag:
+        os.makedirs(module_defaults_path, exist_ok=True)
+    # Defaults modules can be empty, but pungi detects
+    # empty folder while copying and raises the exception in this case
+    Path(os.path.join(module_defaults_path, EMPTY_FILE)).touch()
+
+    for module_file in modules_paths:
+        data = module_file.read()
+        if is_gzip_file(data[:2]):
+            data = gzip.decompress(data)
+        elif is_xz_file(data[:2]):
+            data = lzma.decompress(data)
+        documents = yaml.load_all(data, Loader=yaml.BaseLoader)
+        for doc in documents:
+            path = None
+            if _should_grep_modules(
+                doc['document'],
+                grep_only_modules_data,
+                grep_only_modules_defaults_data,
+            ):
+                name = f"{doc['data']['module']}.yaml"
+                path = os.path.join(module_defaults_path, name)
+                logging.info('Found %s module defaults', name)
+            elif _should_grep_defaults(
+                    doc['document'],
+                    grep_only_modules_data,
+                    grep_only_modules_defaults_data,
+            ):
+                # pungi.phases.pkgset.sources.source_koji.get_koji_modules
+                stream = doc['data']['stream'].replace('-', '_')
+                doc_data = doc['data']
+                name = f"{doc_data['name']}-{stream}-" \
+                       f"{doc_data['version']}.{doc_data['context']}"
+                arch_dir = os.path.join(
+                    modules_path,
+                    doc_data['arch']
+                )
+                os.makedirs(arch_dir, exist_ok=True)
+                path = os.path.join(
+                    arch_dir,
+                    name,
+                )
+                logging.info('Found module %s', name)
+
+                if 'artifacts' not in doc['data']:
+                    logging.warning(
+                        'RPM %s does not have explicit list of artifacts',
+                        name
+                    )
+            if path is not None:
+                with open(path, 'w') as f:
+                    yaml.dump(doc, f, default_flow_style=False)
+
+
+def cli_main():
+    parser = ArgumentParser()
+    content_type_group = parser.add_mutually_exclusive_group(required=False)
+    content_type_group.add_argument(
+        '--get-only-modules-data',
+        action='store_true',
+        help='Parse and get only modules data',
+    )
+    content_type_group.add_argument(
+        '--get-only-modules-defaults-data',
+        action='store_true',
+        help='Parse and get only modules_defaults data',
+    )
+    path_group = parser.add_mutually_exclusive_group(required=True)
+    path_group.add_argument(
+        '-p', '--path',
+        type=FileType('rb'), nargs='+',
+        help='Path to modules.yaml.gz file. '
+             'You may pass multiple files by passing -p path1 path2'
+    )
+    path_group.add_argument(
+        '-rp', '--repo-path',
+        required=False,
+        type=str,
+        default=None,
+        help='Path to a directory which contains repodirs. E.g. /var/repos'
+    )
+    path_group.add_argument(
+        '-rd', '--repodata-paths',
+        required=False,
+        type=str,
+        nargs='+',
+        default=[],
+        help='Paths/urls to the directories with directory `repodata`',
+    )
+    parser.add_argument('-t', '--target', required=True)
+
+    namespace = parser.parse_args()
+    if namespace.repodata_paths:
+        modules = []
+        for repodata_path in namespace.repodata_paths:
+            modules.append(read_modules_yaml_from_specific_repo(
+                repodata_path,
+            ))
+    elif namespace.path is not None:
+        modules = namespace.path
+    else:
+        modules = grep_list_of_modules_yaml(namespace.repo_path)
+    modules = list(filter(lambda i: i is not None, modules))
+    collect_modules(
+        modules,
+        namespace.target,
+        namespace.get_only_modules_data,
+        namespace.get_only_modules_defaults_data,
+    )
+
+
+if __name__ == '__main__':
+    cli_main()

pungi/scripts/gather_rpms.py

@@ -0,0 +1,96 @@
+import re
+from argparse import ArgumentParser
+
+import os
+from glob import iglob
+from typing import List
+from pathlib import Path
+
+from dataclasses import dataclass
+from productmd.common import parse_nvra
+
+
+@dataclass
+class Package:
+    nvra: dict
+    path: Path
+
+
+def search_rpms(top_dir: Path) -> List[Package]:
+    """
+    Search for all *.rpm files recursively
+    in given top directory
+    Returns:
+        list: list of paths
+    """
+    return [Package(
+        nvra=parse_nvra(Path(path).stem),
+        path=Path(path),
+    ) for path in iglob(str(top_dir.joinpath('**/*.rpm')), recursive=True)]
+
+
+def is_excluded_package(
+        package: Package,
+        excluded_packages: List[str],
+) -> bool:
+    package_key = f'{package.nvra["name"]}.{package.nvra["arch"]}'
+    return any(
+        re.search(
+            f'^{excluded_pkg}$',
+            package_key,
+        ) or excluded_pkg in (package.nvra['name'], package_key)
+        for excluded_pkg in excluded_packages
+    )
+
+
+def copy_rpms(
+        packages: List[Package],
+        target_top_dir: Path,
+        excluded_packages: List[str],
+):
+    """
+    Search synced repos for rpms and prepare
+    koji-like structure for pungi
+
+    Instead of repos, use following structure:
+        # ls /mnt/koji/
+            i686/  noarch/  x86_64/
+    Returns:
+        Nothing:
+    """
+    for package in packages:
+        if is_excluded_package(package, excluded_packages):
+            continue
+        target_arch_dir = target_top_dir.joinpath(package.nvra['arch'])
+        target_file = target_arch_dir.joinpath(package.path.name)
+        os.makedirs(target_arch_dir, exist_ok=True)
+
+        if not target_file.exists():
+            try:
+                os.link(package.path, target_file)
+            except OSError:
+                # hardlink failed, try symlinking
+                package.path.symlink_to(target_file)
+
+
+def cli_main():
+    parser = ArgumentParser()
+    parser.add_argument('-p', '--path', required=True, type=Path)
+    parser.add_argument('-t', '--target', required=True, type=Path)
+    parser.add_argument(
+        '-e',
+        '--excluded-packages',
+        required=False,
+        nargs='+',
+        type=str,
+        default=[],
+    )
+
+    namespace = parser.parse_args()
+
+    rpms = search_rpms(namespace.path)
+    copy_rpms(rpms, namespace.target, namespace.excluded_packages)
+
+
+if __name__ == '__main__':
+    cli_main()

pungi/scripts/pungi.py

@@ -478,8 +478,7 @@ def main():
                 print("RPM size:       %s MiB" % (mypungi.size_packages() / 1024**2))
                 if not opts.nodebuginfo:
                     print(
-                        "DEBUGINFO size: %s MiB"
-                        % (mypungi.size_debuginfo() / 1024**2)
+                        "DEBUGINFO size: %s MiB" % (mypungi.size_debuginfo() / 1024**2)
                     )
                 if not opts.nosource:
                     print("SRPM size:      %s MiB" % (mypungi.size_srpms() / 1024**2))

pungi/scripts/pungi_gather.py

@@ -97,6 +97,7 @@ def main(ns, persistdir, cachedir):
     dnf_conf = Conf(ns.arch)
     dnf_conf.persistdir = persistdir
     dnf_conf.cachedir = cachedir
+    dnf_conf.optional_metadata_types = ["filelists"]
     dnf_obj = DnfWrapper(dnf_conf)
 
     gather_opts = GatherOptions()

pungi/scripts/pungi_koji.py

@@ -23,6 +23,7 @@ from pungi.phases import PHASES_NAMES
 from pungi import get_full_version, util
 from pungi.errors import UnsignedPackagesError
 from pungi.wrappers import kojiwrapper
+from pungi.util import rmtree
 
 
 # force C locales
@@ -251,9 +252,15 @@ def main():
         kobo.log.add_stderr_logger(logger)
 
     conf = util.load_config(opts.config)
-
     compose_type = opts.compose_type or conf.get("compose_type", "production")
-    if compose_type == "production" and not opts.label and not opts.no_label:
+    label = opts.label or conf.get("label")
+    if label:
+        try:
+            productmd.composeinfo.verify_label(label)
+        except ValueError as ex:
+            abort(str(ex))
+
+    if compose_type == "production" and not label and not opts.no_label:
         abort("must specify label for a production compose")
 
     if (
@@ -303,7 +310,7 @@ def main():
             opts.target_dir,
             conf,
             compose_type=compose_type,
-            compose_label=opts.label,
+            compose_label=label,
             parent_compose_ids=opts.parent_compose_id,
             respin_of=opts.respin_of,
         )
@@ -314,7 +321,7 @@ def main():
             ci = Compose.get_compose_info(
                 conf,
                 compose_type=compose_type,
-                compose_label=opts.label,
+                compose_label=label,
                 parent_compose_ids=opts.parent_compose_id,
                 respin_of=opts.respin_of,
             )
@@ -416,11 +423,12 @@ def run_compose(
         compose, buildinstall_phase, pkgset_phase
     )
     ostree_phase = pungi.phases.OSTreePhase(compose, pkgset_phase)
+    ostree_container_phase = pungi.phases.OSTreeContainerPhase(compose, pkgset_phase)
     createiso_phase = pungi.phases.CreateisoPhase(compose, buildinstall_phase)
     extra_isos_phase = pungi.phases.ExtraIsosPhase(compose, buildinstall_phase)
-    liveimages_phase = pungi.phases.LiveImagesPhase(compose)
     livemedia_phase = pungi.phases.LiveMediaPhase(compose)
     image_build_phase = pungi.phases.ImageBuildPhase(compose, buildinstall_phase)
+    kiwibuild_phase = pungi.phases.KiwiBuildPhase(compose)
     osbuild_phase = pungi.phases.OSBuildPhase(compose)
     osbs_phase = pungi.phases.OSBSPhase(compose, pkgset_phase, buildinstall_phase)
     image_container_phase = pungi.phases.ImageContainerPhase(compose)
@@ -437,17 +445,18 @@ def run_compose(
         gather_phase,
         extrafiles_phase,
         createiso_phase,
-        liveimages_phase,
         livemedia_phase,
         image_build_phase,
         image_checksum_phase,
         test_phase,
         ostree_phase,
         ostree_installer_phase,
+        ostree_container_phase,
         extra_isos_phase,
         osbs_phase,
         osbuild_phase,
         image_container_phase,
+        kiwibuild_phase,
     ):
         if phase.skip():
             continue
@@ -462,50 +471,6 @@ def run_compose(
             print(i)
         raise RuntimeError("Configuration is not valid")
 
-    # PREP
-
-    # Note: This may be put into a new method of phase classes (e.g. .prep())
-    # in same way as .validate() or .run()
-
-    # Prep for liveimages - Obtain a password for signing rpm wrapped images
-    if (
-        "signing_key_password_file" in compose.conf
-        and "signing_command" in compose.conf
-        and "%(signing_key_password)s" in compose.conf["signing_command"]
-        and not liveimages_phase.skip()
-    ):
-        # TODO: Don't require key if signing is turned off
-        # Obtain signing key password
-        signing_key_password = None
-
-        # Use appropriate method
-        if compose.conf["signing_key_password_file"] == "-":
-            # Use stdin (by getpass module)
-            try:
-                signing_key_password = getpass.getpass("Signing key password: ")
-            except EOFError:
-                compose.log_debug("Ignoring signing key password")
-                pass
-        else:
-            # Use text file with password
-            try:
-                signing_key_password = (
-                    open(compose.conf["signing_key_password_file"], "r")
-                    .readline()
-                    .rstrip("\n")
-                )
-            except IOError:
-                # Filename is not print intentionally in case someone puts
-                # password directly into the option
-                err_msg = "Cannot load password from file specified by 'signing_key_password_file' option"  # noqa: E501
-                compose.log_error(err_msg)
-                print(err_msg)
-                raise RuntimeError(err_msg)
-
-        if signing_key_password:
-            # Store the password
-            compose.conf["signing_key_password"] = signing_key_password
-
     init_phase.start()
     init_phase.stop()
 
@@ -518,6 +483,7 @@ def run_compose(
         (gather_phase, createrepo_phase),
         extrafiles_phase,
         (ostree_phase, ostree_installer_phase),
+        ostree_container_phase,
     )
     essentials_phase = pungi.phases.WeaverPhase(compose, essentials_schema)
     essentials_phase.start()
@@ -542,10 +508,10 @@ def run_compose(
     compose_images_schema = (
         createiso_phase,
         extra_isos_phase,
-        liveimages_phase,
         image_build_phase,
         livemedia_phase,
         osbuild_phase,
+        kiwibuild_phase,
     )
     post_image_phase = pungi.phases.WeaverPhase(
         compose, (image_checksum_phase, image_container_phase)
@@ -567,10 +533,11 @@ def run_compose(
         and ostree_installer_phase.skip()
         and createiso_phase.skip()
         and extra_isos_phase.skip()
-        and liveimages_phase.skip()
         and livemedia_phase.skip()
         and image_build_phase.skip()
+        and kiwibuild_phase.skip()
         and osbuild_phase.skip()
+        and ostree_container_phase.skip()
     ):
         compose.im.dump(compose.paths.compose.metadata("images.json"))
     compose.dump_containers_metadata()
@@ -684,7 +651,7 @@ def cli_main():
     except (Exception, KeyboardInterrupt) as ex:
         if COMPOSE:
             COMPOSE.log_error("Compose run failed: %s" % ex)
-            COMPOSE.traceback()
+            COMPOSE.traceback(show_locals=getattr(ex, "show_locals", True))
             COMPOSE.log_critical("Compose failed: %s" % COMPOSE.topdir)
             COMPOSE.write_status("DOOMED")
         else:
@@ -693,3 +660,8 @@ def cli_main():
         sys.stdout.flush()
         sys.stderr.flush()
         sys.exit(1)
+    finally:
+        # Remove repositories cloned during ExtraFiles phase
+        process_id = os.getpid()
+        directoy_to_remove = "/tmp/pungi-temp-git-repos-" + str(process_id) + "/"
+        rmtree(directoy_to_remove)

pungi/util.py

@@ -279,7 +279,7 @@ class GitUrlResolveError(RuntimeError):
     pass
 
 
-def resolve_git_ref(repourl, ref):
+def resolve_git_ref(repourl, ref, credential_helper=None):
     """Resolve a reference in a Git repo to a commit.
 
     Raises RuntimeError if there was an error. Most likely cause is failure to
@@ -289,7 +289,7 @@ def resolve_git_ref(repourl, ref):
         # This looks like a commit ID already.
         return ref
     try:
-        _, output = git_ls_remote(repourl, ref)
+        _, output = git_ls_remote(repourl, ref, credential_helper)
     except RuntimeError as e:
         raise GitUrlResolveError(
             "ref does not exist in remote repo %s with the error %s %s"
@@ -316,7 +316,7 @@ def resolve_git_ref(repourl, ref):
     return lines[0].split()[0]
 
 
-def resolve_git_url(url):
+def resolve_git_url(url, credential_helper=None):
     """Given a url to a Git repo specifying HEAD or origin/<branch> as a ref,
     replace that specifier with actual SHA1 of the commit.
 
@@ -335,7 +335,7 @@ def resolve_git_url(url):
     scheme = r.scheme.replace("git+", "")
 
     baseurl = urllib.parse.urlunsplit((scheme, r.netloc, r.path, "", ""))
-    fragment = resolve_git_ref(baseurl, ref)
+    fragment = resolve_git_ref(baseurl, ref, credential_helper)
 
     result = urllib.parse.urlunsplit((r.scheme, r.netloc, r.path, r.query, fragment))
     if "?#" in url:
@@ -354,13 +354,18 @@ class GitUrlResolver(object):
         self.offline = offline
         self.cache = {}
 
-    def __call__(self, url, branch=None):
+    def __call__(self, url, branch=None, options=None):
+        credential_helper = options.get("credential_helper") if options else None
         if self.offline:
             return branch or url
         key = (url, branch)
         if key not in self.cache:
             try:
-                res = resolve_git_ref(url, branch) if branch else resolve_git_url(url)
+                res = (
+                    resolve_git_ref(url, branch, credential_helper)
+                    if branch
+                    else resolve_git_url(url, credential_helper)
+                )
                 self.cache[key] = res
             except GitUrlResolveError as exc:
                 self.cache[key] = exc
@@ -456,6 +461,9 @@ def get_volid(compose, arch, variant=None, disc_type=False, formats=None, **kwar
         if not variant_uid and "%(variant)s" in i:
             continue
         try:
+            # fmt: off
+            # Black wants to add a comma after kwargs, but that's not valid in
+            # Python 2.7
             args = get_format_substs(
                 compose,
                 variant=variant_uid,
@@ -467,6 +475,7 @@ def get_volid(compose, arch, variant=None, disc_type=False, formats=None, **kwar
                 base_product_version=base_product_version,
                 **kwargs
             )
+            # fmt: on
             volid = (i % args).format(**args)
         except KeyError as err:
             raise RuntimeError(
@@ -478,10 +487,7 @@ def get_volid(compose, arch, variant=None, disc_type=False, formats=None, **kwar
         tried.add(volid)
 
     if volid and len(volid) > 32:
-        raise ValueError(
-            "Could not create volume ID longer than 32 bytes, options are %r",
-            sorted(tried, key=len),
-        )
+        volid = volid[:32]
 
     if compose.conf["restricted_volid"]:
         # Replace all non-alphanumeric characters and non-underscores) with
@@ -991,8 +997,12 @@ def retry(timeout=120, interval=30, wait_on=Exception):
 
 
 @retry(wait_on=RuntimeError)
-def git_ls_remote(baseurl, ref):
-    return run(["git", "ls-remote", baseurl, ref], universal_newlines=True)
+def git_ls_remote(baseurl, ref, credential_helper=None):
+    cmd = ["git"]
+    if credential_helper:
+        cmd.extend(["-c", "credential.useHttpPath=true"])
+        cmd.extend(["-c", "credential.helper=%s" % credential_helper])
+    return run(cmd + ["ls-remote", baseurl, ref], universal_newlines=True)
 
 
 def get_tz_offset():
@@ -1137,3 +1147,16 @@ def read_json_file(file_path):
     """A helper function to read a JSON file."""
     with open(file_path) as f:
         return json.load(f)
+
+
+UNITS = ["", "Ki", "Mi", "Gi", "Ti"]
+
+
+def format_size(sz):
+    sz = float(sz)
+    unit = 0
+    while sz > 1024:
+        sz /= 1024
+        unit += 1
+
+    return "%.3g %sB" % (sz, UNITS[unit])

pungi/wrappers/comps.py

@@ -306,6 +306,8 @@ class CompsWrapper(object):
             append_common_info(doc, group_node, group, force_description=True)
             append_bool(doc, group_node, "default", group.default)
             append_bool(doc, group_node, "uservisible", group.uservisible)
+            if group.display_order is not None:
+                append(doc, group_node, "display_order", str(group.display_order))
 
             if group.lang_only:
                 append(doc, group_node, "langonly", group.lang_only)

pungi/wrappers/fus.py

@@ -88,5 +88,12 @@ def parse_output(output):
                 packages.add((name, arch, frozenset(flags)))
             else:
                 name, arch = nevra.rsplit(".", 1)
-                modules.add(name.split(":", 1)[1])
+                # replace dash by underscore in stream of module's nerva
+                # source of name looks like
+                # module:llvm-toolset:rhel8:8040020210411062713:9f9e2e7e.x86_64
+                name = ':'.join(
+                    item.replace('-', '_') if i == 1 else item for
+                    i, item in enumerate(name.split(':')[1:])
+                )
+                modules.add(name)
     return packages, modules

pungi/wrappers/iso.py

@@ -280,14 +280,21 @@ def get_manifest_cmd(iso_name, xorriso=False, output_file=None):
         )
 
 
-def get_volume_id(path):
-    cmd = ["isoinfo", "-d", "-i", path]
-    retcode, output = run(cmd, universal_newlines=True)
+def get_volume_id(path, xorriso=False):
+    if xorriso:
+        cmd = ["xorriso", "-indev", path]
+        retcode, output = run(cmd, universal_newlines=True)
+        for line in output.splitlines():
+            if line.startswith("Volume id"):
+                return line.split("'")[1]
+    else:
+        cmd = ["isoinfo", "-d", "-i", path]
+        retcode, output = run(cmd, universal_newlines=True)
 
-    for line in output.splitlines():
-        line = line.strip()
-        if line.startswith("Volume id:"):
-            return line[11:].strip()
+        for line in output.splitlines():
+            line = line.strip()
+            if line.startswith("Volume id:"):
+                return line[11:].strip()
 
     raise RuntimeError("Could not read Volume ID")
 
@@ -509,3 +516,21 @@ def mount(image, logger=None, use_guestmount=True):
                 util.run_unmount_cmd(["fusermount", "-u", mount_dir], path=mount_dir)
             else:
                 util.run_unmount_cmd(["umount", mount_dir], path=mount_dir)
+
+
+def xorriso_commands(arch, input, output):
+    """List of xorriso commands to modify a bootable image."""
+    commands = [
+        ("-indev", input),
+        ("-outdev", output),
+        # isoinfo -J uses the Joliet tree, and it's used by virt-install
+        ("-joliet", "on"),
+        # Support long filenames in the Joliet trees. Repodata is particularly
+        # likely to run into this limit.
+        ("-compliance", "joliet_long_names"),
+        ("-boot_image", "any", "replay"),
+    ]
+    if arch == "ppc64le":
+        # This is needed for the image to be bootable.
+        commands.append(("-as", "mkisofs", "-U", "--"))
+    return commands

pungi/wrappers/kojimock.py

@@ -0,0 +1,299 @@
+import os
+import time
+from pathlib import Path
+
+from attr import dataclass
+from kobo.rpmlib import parse_nvra
+
+from pungi.module_util import Modulemd
+
+# just a random value which we don't
+# use in mock currently
+# originally builds are filtered by this value
+# to get consistent snapshot of tags and packages
+from pungi.scripts.gather_rpms import search_rpms
+
+LAST_EVENT_ID = 999999
+# last event time is not important but build
+# time should be less then it
+LAST_EVENT_TIME = time.time()
+BUILD_TIME = 0
+# virtual build that collects all
+# packages built for some arch
+RELEASE_BUILD_ID = 15270
+# tag that should have all packages available
+ALL_PACKAGES_TAG = 'dist-c8-compose'
+# tag that should have all modules available
+ALL_MODULES_TAG = 'dist-c8-module-compose'
+
+
+@dataclass
+class Module:
+    build_id: int
+    name: str
+    nvr: str
+    stream: str
+    version: str
+    context: str
+    arch: str
+
+
+class KojiMock:
+    """
+    Class that acts like real koji (for some needed methods)
+    but uses local storage as data source
+    """
+    def __init__(self, packages_dir, modules_dir, all_arches):
+        self._modules = self._gather_modules(modules_dir)
+        self._modules_dir = modules_dir
+        self._packages_dir = packages_dir
+        self._all_arches = all_arches
+
+    @staticmethod
+    def _gather_modules(modules_dir):
+        modules = {}
+        for index, (f, arch) in enumerate(
+            (sub_path.name, sub_path.parent.name)
+            for path in Path(modules_dir).glob('*')
+            for sub_path in path.iterdir()
+        ):
+            parsed = parse_nvra(f)
+            modules[index] = Module(
+                name=parsed['name'],
+                nvr=f,
+                version=parsed['release'],
+                context=parsed['arch'],
+                stream=parsed['version'],
+                build_id=index,
+                arch=arch,
+            )
+        return modules
+
+    @staticmethod
+    def getLastEvent(*args, **kwargs):
+        return {'id': LAST_EVENT_ID, 'ts': LAST_EVENT_TIME}
+
+    def listTagged(self, tag_name, *args, **kwargs):
+        """
+        Returns list of virtual 'builds' that contain packages by given tag
+        There are two kinds of tags: modular and distributive.
+        For now, only one kind, distributive one, is needed.
+        """
+        if tag_name != ALL_MODULES_TAG:
+            raise ValueError("I don't know what tag is %s" % tag_name)
+
+        builds = []
+        for module in self._modules.values():
+            builds.append({
+                'build_id': module.build_id,
+                'owner_name': 'centos',
+                'package_name': module.name,
+                'nvr': module.nvr,
+                'version': module.stream,
+                'release': '%s.%s' % (module.version, module.context),
+                'name': module.name,
+                'id': module.build_id,
+                'tag_name': tag_name,
+                'arch': module.arch,
+                # Following fields are currently not
+                # used but returned by real koji
+                # left them here just for reference
+                #
+                # 'task_id': None,
+                # 'state': 1,
+                # 'start_time': '2020-12-23 16:43:59',
+                # 'creation_event_id': 309485,
+                # 'creation_time': '2020-12-23 17:05:33.553748',
+                # 'epoch': None, 'tag_id': 533,
+                # 'completion_time': '2020-12-23 17:05:23',
+                # 'volume_id': 0,
+                # 'package_id': 3221,
+                # 'owner_id': 11,
+                # 'volume_name': 'DEFAULT',
+            })
+
+        return builds
+
+    @staticmethod
+    def getFullInheritance(*args, **kwargs):
+        """
+        Unneeded because we use local storage.
+        """
+        return []
+
+    def getBuild(self, build_id, *args, **kwargs):
+        """
+        Used to get information about build
+        (used in pungi only for modules currently)
+        """
+        module = self._modules[build_id]
+
+        result = {
+            'id': build_id,
+            'name': module.name,
+            'version': module.stream,
+            'release': '%s.%s' % (module.version, module.context),
+            'completion_ts': BUILD_TIME,
+            'state': 'COMPLETE',
+            'arch': module.arch,
+            'extra': {
+                'typeinfo': {
+                    'module': {
+                        'stream': module.stream,
+                        'version': module.version,
+                        'name': module.name,
+                        'context': module.context,
+                        'content_koji_tag': '-'.join([
+                            module.name,
+                            module.stream,
+                            module.version
+                        ]) + '.' + module.context
+                    }
+                }
+            }
+        }
+        return result
+
+    def listArchives(self, build_id, *args, **kwargs):
+        """
+        Originally lists artifacts for build, but in pungi used
+        only to get list of modulemd files for some module
+        """
+        module = self._modules[build_id]
+
+        return [
+            {
+                'build_id': module.build_id,
+                'filename': f'modulemd.{module.arch}.txt',
+                'btype': 'module'
+            },
+            # noone ever uses this file
+            # but it should be because pungi ignores builds
+            # with len(files) <= 1
+            {
+                'build_id': module.build_id,
+                'filename': 'modulemd.txt',
+                'btype': 'module'
+            }
+        ]
+
+    def listTaggedRPMS(self, tag_name, *args, **kwargs):
+        """
+        Get information about packages that are tagged by tag.
+        There are two kings of tags: per-module and per-distr.
+        """
+        if tag_name == ALL_PACKAGES_TAG:
+            builds, packages = self._get_release_packages()
+        else:
+            builds, packages = self._get_module_packages(tag_name)
+        return [
+            packages,
+            builds
+        ]
+
+    def _get_release_packages(self):
+        """
+        Search packages dir and keep only
+        packages that are non-modular.
+
+        This is quite the way how real koji works:
+         - modular packages are tagged by module-* tag
+         - all other packages are tagged with dist* tag
+        """
+        packages = []
+
+        # get all rpms in folder
+        rpms = search_rpms(Path(self._packages_dir))
+
+        for rpm in rpms:
+            info = parse_nvra(rpm.path.stem)
+            if 'module' in info['release']:
+                continue
+            packages.append({
+                "build_id": RELEASE_BUILD_ID,
+                "name": info['name'],
+                "extra": None,
+                "arch": info['arch'],
+                "epoch": info['epoch'] or None,
+                "version": info['version'],
+                "metadata_only": False,
+                "release": info['release'],
+                # not used currently
+                # "id": 262555,
+                # "size": 0
+            })
+        builds = []
+        return builds, packages
+
+    def _get_module_packages(self, tag_name):
+        """
+        Get list of builds for module and given module tag name.
+        """
+        builds = []
+        packages = []
+        modules = self._get_modules_by_name(tag_name)
+        for module in modules:
+            if module is None:
+                raise ValueError('Module %s is not found' % tag_name)
+            path = os.path.join(
+                self._modules_dir,
+                module.arch,
+                tag_name,
+            )
+
+            builds.append({
+                "build_id": module.build_id,
+                "package_name": module.name,
+                "nvr": module.nvr,
+                "tag_name": module.nvr,
+                "version": module.stream,
+                "release": module.version,
+                "id": module.build_id,
+                "name": module.name,
+                "volume_name": "DEFAULT",
+                # Following fields are currently not
+                # used but returned by real koji
+                # left them here just for reference
+                #
+                # "owner_name": "mbox-mbs-backend",
+                # "task_id": 195937,
+                # "state": 1,
+                # "start_time": "2020-12-22 19:20:12.504578",
+                # "creation_event_id": 306731,
+                # "creation_time": "2020-12-22 19:20:12.504578",
+                # "epoch": None,
+                # "tag_id": 1192,
+                # "completion_time": "2020-12-22 19:34:34.716615",
+                # "volume_id": 0,
+                # "package_id": 104,
+                # "owner_id": 6,
+            })
+
+            if os.path.exists(path):
+                info = Modulemd.ModuleStream.read_string(open(path).read(), strict=True)
+                for art in info.get_rpm_artifacts():
+                    data = parse_nvra(art)
+                    packages.append({
+                        "build_id": module.build_id,
+                        "name": data['name'],
+                        "extra": None,
+                        "arch": data['arch'],
+                        "epoch": data['epoch'] or None,
+                        "version": data['version'],
+                        "metadata_only": False,
+                        "release": data['release'],
+                        "id": 262555,
+                        "size": 0
+                    })
+            else:
+                raise RuntimeError('Unable to find module %s' % path)
+        return builds, packages
+
+    def _get_modules_by_name(self, tag_name):
+        modules = []
+        for arch in self._all_arches:
+            for module in self._modules.values():
+                if module.nvr != tag_name or module.arch != arch:
+                    continue
+                modules.append(module)
+        return modules

pungi/wrappers/kojiwrapper.py

@@ -14,18 +14,25 @@
 # along with this program; if not, see <https://gnu.org/licenses/>.
 
 
+import contextlib
 import os
 import re
+import socket
+import shutil
 import time
 import threading
-import contextlib
+
+import requests
 
 import koji
 from kobo.shortcuts import run, force_list
 import six
 from six.moves import configparser, shlex_quote
 import six.moves.xmlrpc_client as xmlrpclib
+from flufl.lock import Lock
+from datetime import timedelta
 
+from .kojimock import KojiMock
 from .. import util
 from ..arch_utils import getBaseArch
 
@@ -407,92 +414,6 @@ class KojiWrapper(object):
 
         return cmd
 
-    def get_create_image_cmd(
-        self,
-        name,
-        version,
-        target,
-        arch,
-        ks_file,
-        repos,
-        image_type="live",
-        image_format=None,
-        release=None,
-        wait=True,
-        archive=False,
-        specfile=None,
-        ksurl=None,
-    ):
-        # Usage: koji spin-livecd [options] <name> <version> <target> <arch> <kickstart-file>  # noqa: E501
-        # Usage: koji spin-appliance [options] <name> <version> <target> <arch> <kickstart-file>  # noqa: E501
-        # Examples:
-        #  * name: RHEL-7.0
-        #  * name: Satellite-6.0.1-RHEL-6
-        #  ** -<type>.<arch>
-        #  * version: YYYYMMDD[.n|.t].X
-        #  * release: 1
-
-        cmd = self._get_cmd()
-
-        if image_type == "live":
-            cmd.append("spin-livecd")
-        elif image_type == "appliance":
-            cmd.append("spin-appliance")
-        else:
-            raise ValueError("Invalid image type: %s" % image_type)
-
-        if not archive:
-            cmd.append("--scratch")
-
-        cmd.append("--noprogress")
-
-        if wait:
-            cmd.append("--wait")
-        else:
-            cmd.append("--nowait")
-
-        if specfile:
-            cmd.append("--specfile=%s" % specfile)
-
-        if ksurl:
-            cmd.append("--ksurl=%s" % ksurl)
-
-        if isinstance(repos, list):
-            for repo in repos:
-                cmd.append("--repo=%s" % repo)
-        else:
-            cmd.append("--repo=%s" % repos)
-
-        if image_format:
-            if image_type != "appliance":
-                raise ValueError("Format can be specified only for appliance images'")
-            supported_formats = ["raw", "qcow", "qcow2", "vmx"]
-            if image_format not in supported_formats:
-                raise ValueError(
-                    "Format is not supported: %s. Supported formats: %s"
-                    % (image_format, " ".join(sorted(supported_formats)))
-                )
-            cmd.append("--format=%s" % image_format)
-
-        if release is not None:
-            cmd.append("--release=%s" % release)
-
-        # IMPORTANT: all --opts have to be provided *before* args
-        # Usage:
-        #  koji spin-livecd [options] <name> <version> <target> <arch> <kickstart-file>
-
-        cmd.append(name)
-        cmd.append(version)
-        cmd.append(target)
-
-        # i686 -> i386 etc.
-        arch = getBaseArch(arch)
-        cmd.append(arch)
-
-        cmd.append(ks_file)
-
-        return cmd
-
     def _has_connection_error(self, output):
         """Checks if output indicates connection error."""
         return re.search("error: failed to connect\n$", output)
@@ -606,6 +527,7 @@ class KojiWrapper(object):
                 "createImage",
                 "createLiveMedia",
                 "createAppliance",
+                "createKiwiImage",
             ]:
                 continue
 
@@ -785,11 +707,10 @@ class KojiWrapper(object):
         if list_of_args is None and list_of_kwargs is None:
             raise ValueError("One of list_of_args or list_of_kwargs must be set.")
 
-        if type(list_of_args) not in [type(None), list] or type(list_of_kwargs) not in [
-            type(None),
-            list,
-        ]:
-            raise ValueError("list_of_args and list_of_kwargs must be list or None.")
+        if list_of_args is not None and not isinstance(list_of_args, list):
+            raise ValueError("list_of_args must be list or None.")
+        if list_of_kwargs is not None and not isinstance(list_of_kwargs, list):
+            raise ValueError("list_of_kwargs must be list or None.")
 
         if list_of_kwargs is None:
             list_of_kwargs = [{}] * len(list_of_args)
@@ -803,9 +724,9 @@ class KojiWrapper(object):
 
         koji_session.multicall = True
         for args, kwargs in zip(list_of_args, list_of_kwargs):
-            if type(args) != list:
+            if not isinstance(args, list):
                 args = [args]
-            if type(kwargs) != dict:
+            if not isinstance(kwargs, dict):
                 raise ValueError("Every item in list_of_kwargs must be a dict")
             koji_session_fnc(*args, **kwargs)
 
@@ -813,7 +734,7 @@ class KojiWrapper(object):
 
         if not responses:
             return None
-        if type(responses) != list:
+        if not isinstance(responses, list):
             raise ValueError(
                 "Fault element was returned for multicall of method %r: %r"
                 % (koji_session_fnc, responses)
@@ -829,7 +750,7 @@ class KojiWrapper(object):
         # a one-item array containing the result value,
         # or a struct of the form found inside the standard <fault> element.
         for response, args, kwargs in zip(responses, list_of_args, list_of_kwargs):
-            if type(response) == list:
+            if isinstance(response, list):
                 if not response:
                     raise ValueError(
                         "Empty list returned for multicall of method %r with args %r, %r"  # noqa: E501
@@ -864,6 +785,45 @@ class KojiWrapper(object):
             pass
 
 
+class KojiMockWrapper(object):
+    lock = threading.Lock()
+
+    def __init__(self, compose, all_arches):
+        self.all_arches = all_arches
+        self.compose = compose
+        try:
+            self.profile = self.compose.conf["koji_profile"]
+        except KeyError:
+            raise RuntimeError("Koji profile must be configured")
+        with self.lock:
+            self.koji_module = koji.get_profile_module(self.profile)
+            session_opts = {}
+            for key in (
+                "timeout",
+                "keepalive",
+                "max_retries",
+                "retry_interval",
+                "anon_retry",
+                "offline_retry",
+                "offline_retry_interval",
+                "debug",
+                "debug_xmlrpc",
+                "serverca",
+                "use_fast_upload",
+            ):
+                value = getattr(self.koji_module.config, key, None)
+                if value is not None:
+                    session_opts[key] = value
+            self.koji_proxy = KojiMock(
+                packages_dir=self.koji_module.config.topdir,
+                modules_dir=os.path.join(
+                    self.koji_module.config.topdir,
+                    'modules',
+                ),
+                all_arches=self.all_arches,
+            )
+
+
 def get_buildroot_rpms(compose, task_id):
     """Get build root RPMs - either from runroot or local"""
     result = []
@@ -895,3 +855,177 @@ def get_buildroot_rpms(compose, task_id):
                 continue
             result.append(i)
     return sorted(result)
+
+
+class KojiDownloadProxy:
+    def __init__(self, topdir, topurl, cache_dir, logger):
+        if not topdir:
+            # This will only happen if there is either no koji_profile
+            # configured, or the profile doesn't have a topdir. In the first
+            # case there will be no koji interaction, and the second indicates
+            # broken koji configuration.
+            # We can pretend to have local access in both cases to avoid any
+            # external requests.
+            self.has_local_access = True
+            return
+
+        self.cache_dir = cache_dir
+        self.logger = logger
+
+        self.topdir = topdir
+        self.topurl = topurl
+
+        # If cache directory is configured, we want to use it (even if we
+        # actually have local access to the storage).
+        self.has_local_access = not bool(cache_dir)
+        # This is used for temporary downloaded files. The suffix is unique
+        # per-process. To prevent threads in the same process from colliding, a
+        # thread id is added later.
+        self.unique_suffix = "%s.%s" % (socket.gethostname(), os.getpid())
+        self.session = None
+        if not self.has_local_access:
+            self.session = requests.Session()
+
+    @property
+    def path_prefix(self):
+        dir = self.topdir if self.has_local_access else self.cache_dir
+        return dir.rstrip("/") + "/"
+
+    @classmethod
+    def from_config(klass, conf, logger):
+        topdir = None
+        topurl = None
+        cache_dir = None
+        if "koji_profile" in conf:
+            koji_module = koji.get_profile_module(conf["koji_profile"])
+            topdir = koji_module.config.topdir
+            topurl = koji_module.config.topurl
+
+            cache_dir = conf.get("koji_cache")
+            if cache_dir:
+                cache_dir = cache_dir.rstrip("/") + "/"
+        return klass(topdir, topurl, cache_dir, logger)
+
+    @util.retry(wait_on=requests.exceptions.RequestException)
+    def _download(self, url, dest):
+        """Download file into given location
+
+        :param str url: URL of the file to download
+        :param str dest: file path to store the result in
+        :returns: path to the downloaded file (same as dest) or None if the URL
+        """
+        # contextlib.closing is only needed in requests<2.18
+        with contextlib.closing(self.session.get(url, stream=True)) as r:
+            if r.status_code == 404:
+                self.logger.warning("GET %s NOT FOUND", url)
+                return None
+            if r.status_code != 200:
+                self.logger.error("GET %s %s", url, r.status_code)
+                r.raise_for_status()
+                # The exception from here will be retried by the decorator.
+
+            file_size = int(r.headers.get("Content-Length", 0))
+            self.logger.info("GET %s OK %s", url, util.format_size(file_size))
+            with open(dest, "wb") as f:
+                shutil.copyfileobj(r.raw, f)
+        return dest
+
+    def _delete(self, path):
+        """Try to delete file at given path and ignore errors."""
+        try:
+            os.remove(path)
+        except Exception:
+            self.logger.warning("Failed to delete %s", path)
+
+    def _atomic_download(self, url, dest, validator):
+        """Atomically download a file
+
+        :param str url: URL of the file to download
+        :param str dest: file path to store the result in
+        :returns: path to the downloaded file (same as dest) or None if the URL
+                  return 404.
+        """
+        temp_file = "%s.%s.%s" % (dest, self.unique_suffix, threading.get_ident())
+
+        # First download to the temporary location.
+        try:
+            if self._download(url, temp_file) is None:
+                # The file was not found.
+                return None
+        except Exception:
+            # Download failed, let's make sure to clean up potentially partial
+            # temporary file.
+            self._delete(temp_file)
+            raise
+
+        # Check if the temporary file is correct (assuming we were provided a
+        # validator function).
+        try:
+            if validator:
+                validator(temp_file)
+        except Exception:
+            # Validation failed. Let's delete the problematic file and re-raise
+            # the exception.
+            self._delete(temp_file)
+            raise
+
+        # Atomically move the temporary file into final location
+        os.rename(temp_file, dest)
+        return dest
+
+    def _download_file(self, path, validator):
+        """Ensure file on Koji volume in ``path`` is present in the local
+        cache.
+
+        :returns: path to the local file or None if file is not found
+        """
+        url = path.replace(self.topdir, self.topurl)
+        destination_file = path.replace(self.topdir, self.cache_dir)
+        util.makedirs(os.path.dirname(destination_file))
+
+        lock = Lock(destination_file + ".lock")
+        # Hold the lock for this file for 5 minutes. If another compose needs
+        # the same file but it's not downloaded yet, the process will wait.
+        #
+        # If the download finishes in time, the downloaded file will be used
+        # here.
+        #
+        # If the download takes longer, this process will steal the lock and
+        # start its own download.
+        #
+        # That should not be a problem: the same file will be downloaded and
+        # then replaced atomically on the filesystem. If the original process
+        # managed to hardlink the first file already, that hardlink will be
+        # broken, but that will only result in the same file stored twice.
+        lock.lifetime = timedelta(minutes=5)
+
+        with lock:
+            # Check if the file already exists. If yes, return the path.
+            if os.path.exists(destination_file):
+                # Update mtime of the file. This covers the case of packages in the
+                # tag that are not included in the compose. Updating mtime will
+                # exempt them from cleanup for extra time.
+                os.utime(destination_file)
+                return destination_file
+
+            return self._atomic_download(url, destination_file, validator)
+
+    def get_file(self, path, validator=None):
+        """
+        If path refers to an existing file in Koji, return a valid local path
+        to it. If no such file exists, return None.
+
+        :param validator: A callable that will be called with the path to the
+            downloaded file if and only if the file was actually downloaded.
+            Any exception raised from there will be abort the download and be
+            propagated.
+        """
+        if self.has_local_access:
+            # We have koji volume mounted locally. No transformation needed for
+            # the path, just check it exists.
+            if os.path.exists(path):
+                return path
+            return None
+        else:
+            # We need to download the file.
+            return self._download_file(path, validator)

pungi/wrappers/lorax.py

@@ -109,55 +109,3 @@ class LoraxWrapper(object):
         # TODO: workdir
 
         return cmd
-
-    def get_buildinstall_cmd(
-        self,
-        product,
-        version,
-        release,
-        repo_baseurl,
-        output_dir,
-        variant=None,
-        bugurl=None,
-        nomacboot=False,
-        noupgrade=False,
-        is_final=False,
-        buildarch=None,
-        volid=None,
-        brand=None,
-    ):
-        # RHEL 6 compatibility
-        # Usage: buildinstall [--debug] --version <version> --brand <brand> --product <product> --release <comment> --final [--output outputdir] [--discs <discstring>] <root>  # noqa: E501
-
-        brand = brand or "redhat"
-        # HACK: ignore provided release
-        release = "%s %s" % (brand, version)
-        bugurl = bugurl or "https://bugzilla.redhat.com"
-
-        cmd = ["/usr/lib/anaconda-runtime/buildinstall"]
-
-        cmd.append("--debug")
-
-        cmd.extend(["--version", version])
-        cmd.extend(["--brand", brand])
-        cmd.extend(["--product", product])
-        cmd.extend(["--release", release])
-
-        if is_final:
-            cmd.append("--final")
-
-        if buildarch:
-            cmd.extend(["--buildarch", buildarch])
-
-        if bugurl:
-            cmd.extend(["--bugurl", bugurl])
-
-        output_dir = os.path.abspath(output_dir)
-        cmd.extend(["--output", output_dir])
-
-        for i in force_list(repo_baseurl):
-            if "://" not in i:
-                i = "file://%s" % os.path.abspath(i)
-            cmd.append(i)
-
-        return cmd

pungi/wrappers/scm.py

@@ -20,6 +20,7 @@ import os
 import shutil
 import glob
 import six
+import threading
 from six.moves import shlex_quote
 from six.moves.urllib.request import urlretrieve
 from fnmatch import fnmatch
@@ -29,12 +30,15 @@ from kobo.shortcuts import run, force_list
 from pungi.util import explode_rpm_package, makedirs, copy_all, temp_dir, retry
 from .kojiwrapper import KojiWrapper
 
+lock = threading.Lock()
+
 
 class ScmBase(kobo.log.LoggingBase):
-    def __init__(self, logger=None, command=None, compose=None):
+    def __init__(self, logger=None, command=None, compose=None, options=None):
         kobo.log.LoggingBase.__init__(self, logger=logger)
         self.command = command
         self.compose = compose
+        self.options = options or {}
 
     @retry(interval=60, timeout=300, wait_on=RuntimeError)
     def retry_run(self, cmd, **kwargs):
@@ -156,22 +160,31 @@ class GitWrapper(ScmBase):
         if "://" not in repo:
             repo = "file://%s" % repo
 
+        git_cmd = ["git"]
+        if "credential_helper" in self.options:
+            git_cmd.extend(["-c", "credential.useHttpPath=true"])
+            git_cmd.extend(
+                ["-c", "credential.helper=%s" % self.options["credential_helper"]]
+            )
+
         run(["git", "init"], workdir=destdir)
         try:
-            run(["git", "fetch", "--depth=1", repo, branch], workdir=destdir)
+            run(git_cmd + ["fetch", "--depth=1", repo, branch], workdir=destdir)
             run(["git", "checkout", "FETCH_HEAD"], workdir=destdir)
         except RuntimeError as e:
             # Fetch failed, to do a full clone we add a remote to our empty
             # repo, get its content and check out the reference we want.
             self.log_debug(
                 "Trying to do a full clone because shallow clone failed: %s %s"
-                % (e, e.output)
+                % (e, getattr(e, "output", ""))
             )
             try:
                 # Re-run git init in case of previous failure breaking .git dir
                 run(["git", "init"], workdir=destdir)
                 run(["git", "remote", "add", "origin", repo], workdir=destdir)
-                self.retry_run(["git", "remote", "update", "origin"], workdir=destdir)
+                self.retry_run(
+                    git_cmd + ["remote", "update", "origin"], workdir=destdir
+                )
                 run(["git", "checkout", branch], workdir=destdir)
             except RuntimeError:
                 if self.compose:
@@ -185,19 +198,38 @@ class GitWrapper(ScmBase):
                     copy_all(destdir, debugdir)
                 raise
 
-        self.run_process_command(destdir)
+    def get_temp_repo_path(self, scm_root, scm_branch):
+        scm_repo = scm_root.split("/")[-1]
+        process_id = os.getpid()
+        tmp_dir = (
+            "/tmp/pungi-temp-git-repos-"
+            + str(process_id)
+            + "/"
+            + scm_repo
+            + "-"
+            + scm_branch
+        )
+        return tmp_dir
+
+    def setup_repo(self, scm_root, scm_branch):
+        tmp_dir = self.get_temp_repo_path(scm_root, scm_branch)
+        if not os.path.isdir(tmp_dir):
+            makedirs(tmp_dir)
+            self._clone(scm_root, scm_branch, tmp_dir)
+        self.run_process_command(tmp_dir)
+        return tmp_dir
 
     def export_dir(self, scm_root, scm_dir, target_dir, scm_branch=None):
         scm_dir = scm_dir.lstrip("/")
         scm_branch = scm_branch or "master"
 
-        with temp_dir() as tmp_dir:
-            self.log_debug(
-                "Exporting directory %s from git %s (branch %s)..."
-                % (scm_dir, scm_root, scm_branch)
-            )
+        self.log_debug(
+            "Exporting directory %s from git %s (branch %s)..."
+            % (scm_dir, scm_root, scm_branch)
+        )
 
-            self._clone(scm_root, scm_branch, tmp_dir)
+        with lock:
+            tmp_dir = self.setup_repo(scm_root, scm_branch)
 
             copy_all(os.path.join(tmp_dir, scm_dir), target_dir)
 
@@ -205,15 +237,15 @@ class GitWrapper(ScmBase):
         scm_file = scm_file.lstrip("/")
         scm_branch = scm_branch or "master"
 
-        with temp_dir() as tmp_dir:
-            target_path = os.path.join(target_dir, os.path.basename(scm_file))
+        target_path = os.path.join(target_dir, os.path.basename(scm_file))
 
-            self.log_debug(
-                "Exporting file %s from git %s (branch %s)..."
-                % (scm_file, scm_root, scm_branch)
-            )
+        self.log_debug(
+            "Exporting file %s from git %s (branch %s)..."
+            % (scm_file, scm_root, scm_branch)
+        )
 
-            self._clone(scm_root, scm_branch, tmp_dir)
+        with lock:
+            tmp_dir = self.setup_repo(scm_root, scm_branch)
 
             makedirs(target_dir)
             shutil.copy2(os.path.join(tmp_dir, scm_file), target_path)
@@ -361,15 +393,19 @@ def get_file_from_scm(scm_dict, target_path, compose=None):
         scm_file = os.path.abspath(scm_dict)
         scm_branch = None
         command = None
+        options = {}
     else:
         scm_type = scm_dict["scm"]
         scm_repo = scm_dict["repo"]
         scm_file = scm_dict["file"]
         scm_branch = scm_dict.get("branch", None)
         command = scm_dict.get("command")
+        options = scm_dict.get("options", {})
 
     logger = compose._logger if compose else None
-    scm = _get_wrapper(scm_type, logger=logger, command=command, compose=compose)
+    scm = _get_wrapper(
+        scm_type, logger=logger, command=command, compose=compose, options=options
+    )
 
     files_copied = []
     for i in force_list(scm_file):
@@ -450,15 +486,19 @@ def get_dir_from_scm(scm_dict, target_path, compose=None):
         scm_dir = os.path.abspath(scm_dict)
         scm_branch = None
         command = None
+        options = {}
     else:
         scm_type = scm_dict["scm"]
         scm_repo = scm_dict.get("repo", None)
         scm_dir = scm_dict["dir"]
         scm_branch = scm_dict.get("branch", None)
         command = scm_dict.get("command")
+        options = scm_dict.get("options", {})
 
     logger = compose._logger if compose else None
-    scm = _get_wrapper(scm_type, logger=logger, command=command, compose=compose)
+    scm = _get_wrapper(
+        scm_type, logger=logger, command=command, compose=compose, options=options
+    )
 
     with temp_dir(prefix="scm_checkout_") as tmp_dir:
         scm.export_dir(scm_repo, scm_dir, scm_branch=scm_branch, target_dir=tmp_dir)

pungi_utils/unified_isos.py

@@ -148,6 +148,15 @@ class UnifiedISO(object):
                     new_path = os.path.join(self.temp_dir, "trees", arch, old_relpath)
 
                 makedirs(os.path.dirname(new_path))
+                # Resolve symlinks to external files. Symlinks within the
+                # provided `dir` are kept.
+                if os.path.islink(old_path):
+                    real_path = os.readlink(old_path)
+                    abspath = os.path.normpath(
+                        os.path.join(os.path.dirname(old_path), real_path)
+                    )
+                    if not abspath.startswith(dir):
+                        old_path = real_path
                 try:
                     self.linker.link(old_path, new_path)
                 except OSError as exc:

requirements.txt

@@ -1,7 +1,7 @@
 # Some packages must be installed via dnf/yum first, see doc/contributing.rst
-dict.sorted
 dogpile.cache
-funcsigs
+flufl.lock ; python_version >= '3.0'
+flufl.lock < 3.0 ; python_version <= '2.7'
 jsonschema
 kobo
 koji
@@ -12,4 +12,4 @@ ordered_set
 productmd
 pykickstart
 python-multilib
-urlgrabber
+urlgrabber ; python_version < '3.0'

setup.py

@@ -20,7 +20,7 @@ packages = sorted(packages)
 
 setup(
     name="pungi",
-    version="4.4.0",
+    version="4.7.0",
     description="Distribution compose tool",
     url="https://pagure.io/pungi",
     author="Dennis Gilmore",
@@ -41,6 +41,11 @@ setup(
             "pungi-gather = pungi.scripts.pungi_gather:cli_main",
             "pungi-config-dump = pungi.scripts.config_dump:cli_main",
             "pungi-config-validate = pungi.scripts.config_validate:cli_main",
+            "pungi-cache-cleanup = pungi.scripts.cache_cleanup:main",
+            "pungi-gather-modules = pungi.scripts.gather_modules:cli_main",
+            "pungi-gather-rpms = pungi.scripts.gather_rpms:cli_main",
+            "pungi-generate-packages-json = pungi.scripts.create_packages_json:cli_main",  # noqa: E501
+            "pungi-create-extra-repo = pungi.scripts.create_extra_repo:cli_main"
         ]
     },
     scripts=["contrib/yum-dnf-compare/pungi-compare-depsolving"],
@@ -61,5 +66,5 @@ setup(
         "dogpile.cache",
     ],
     extras_require={':python_version=="2.7"': ["enum34", "lockfile"]},
-    tests_require=["mock", "pytest", "pytest-cov"],
+    tests_require=["pytest", "pytest-cov", "pyfakefs"],
 )

sources

@@ -0,0 +1 @@
+SHA512 (pungi-4.7.0.tar.bz2) = 55c7527a0dff6efa8ed13b1ccdfd3628686fadb55b78fb456e552f4972b831aa96f3ff37ac54837462d91df834157f38426e6b66b52216e1e5861628df724eca

test-requirements.txt

@@ -1,5 +1,5 @@
-mock
+mock; python_version < '3.3'
 parameterized
 pytest
 pytest-cov
-unittest2
+unittest2; python_version < '3.0'

tests/Dockerfile-test

@@ -1,4 +1,4 @@
-FROM fedora:33
+FROM registry.fedoraproject.org/fedora:latest
 LABEL \
     name="Pungi test" \
     description="Run tests using tox with Python 3" \

tests/data/test_create_extra_repo/test_repo/repodata/repomd.xml

@@ -0,0 +1,36 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<repomd xmlns="http://linux.duke.edu/metadata/repo" xmlns:rpm="http://linux.duke.edu/metadata/rpm">
+  <revision>1612479076</revision>
+  <data type="primary">
+    <checksum type="sha256">08941fae6bdb14f3b22bfad38b9d7dcb685a9df58fe8f515a3a0b2fe1af903bb</checksum>
+    <open-checksum type="sha256">2a15e618f049a883d360ccbf3e764b30640255f47dc526c633b1722fe23cbcbc</open-checksum>
+    <location href="repodata/08941fae6bdb14f3b22bfad38b9d7dcb685a9df58fe8f515a3a0b2fe1af903bb-primary.xml.gz"/>
+    <timestamp>1612479075</timestamp>
+    <size>1240</size>
+    <open-size>3888</open-size>
+  </data>
+  <data type="filelists">
+    <checksum type="sha256">e37a0b4a63b2b245dca1727195300cd3961f80aebc82ae7b9849dbf7482f5d0f</checksum>
+    <open-checksum type="sha256">b1782bc4207a5b7c3e64115d5a1d001802e8d363f022ea165df7cdab6f14651c</open-checksum>
+    <location href="repodata/e37a0b4a63b2b245dca1727195300cd3961f80aebc82ae7b9849dbf7482f5d0f-filelists.xml.gz"/>
+    <timestamp>1612479075</timestamp>
+    <size>439</size>
+    <open-size>1295</open-size>
+  </data>
+  <data type="other">
+    <checksum type="sha256">92992176bce71dcde9e4b6ad1442e7b5c7f3de9b7f019a2cd27d042ab38ea2b1</checksum>
+    <open-checksum type="sha256">3b847919691ad32279b13463de6c08f1f8b32f51e87b7d8d7e95a3ec2f46ef51</open-checksum>
+    <location href="repodata/92992176bce71dcde9e4b6ad1442e7b5c7f3de9b7f019a2cd27d042ab38ea2b1-other.xml.gz"/>
+    <timestamp>1612479075</timestamp>
+    <size>630</size>
+    <open-size>1911</open-size>
+  </data>
+  <data type="modules">
+    <checksum type="sha256">e7a671401f8e207e4cd3b90b4ac92d621f84a34dc9026f57c3f427fbed444c57</checksum>
+    <open-checksum type="sha256">d59fee86c18018cc18bb7325aa74aa0abf923c64d29a4ec45e08dcd01a0c3966</open-checksum>
+    <location href="repodata/e7a671401f8e207e4cd3b90b4ac92d621f84a34dc9026f57c3f427fbed444c57-modules.yaml.gz"/>
+    <timestamp>1612479075</timestamp>
+    <size>920</size>
+    <open-size>3308</open-size>
+  </data>
+</repomd>

tests/data/test_create_packages_json/test_repo/repodata/repomd.xml

@@ -0,0 +1,55 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<repomd xmlns="http://linux.duke.edu/metadata/repo" xmlns:rpm="http://linux.duke.edu/metadata/rpm">
+  <revision>1666177486</revision>
+  <data type="primary">
+    <checksum type="sha256">89cb9cc1181635c9147864a7076d91fb81072641d481cd202832a2d257453576</checksum>
+    <open-checksum type="sha256">07255d9856f7531b52a6459f6fc7701c6d93c6d6c29d1382d83afcc53f13494a</open-checksum>
+    <location href="repodata/89cb9cc1181635c9147864a7076d91fb81072641d481cd202832a2d257453576-primary.xml.gz"/>
+    <timestamp>1666177486</timestamp>
+    <size>1387</size>
+    <open-size>6528</open-size>
+  </data>
+  <data type="filelists">
+    <checksum type="sha256">f69ca03957574729fd5150335b0d87afddcfb37a97aed5b06272212854f1773d</checksum>
+    <open-checksum type="sha256">c2e1e674d7d48bccaa16cae0a5f70cb55ef4cd7352b4d9d4fdaa619075d07dbc</open-checksum>
+    <location href="repodata/f69ca03957574729fd5150335b0d87afddcfb37a97aed5b06272212854f1773d-filelists.xml.gz"/>
+    <timestamp>1666177486</timestamp>
+    <size>1252</size>
+    <open-size>5594</open-size>
+  </data>
+  <data type="other">
+    <checksum type="sha256">b3827bd6c9ea67ffa3912002515c64e4d9fe5c4dacbf7c46b0d8768b7abbb84f</checksum>
+    <open-checksum type="sha256">9ce24c526239e349d023c577b2ae3872c8b0f1888aed1fb24b9b9aa12063fdf3</open-checksum>
+    <location href="repodata/b3827bd6c9ea67ffa3912002515c64e4d9fe5c4dacbf7c46b0d8768b7abbb84f-other.xml.gz"/>
+    <timestamp>1666177486</timestamp>
+    <size>999</size>
+    <open-size>6320</open-size>
+  </data>
+  <data type="primary_db">
+    <checksum type="sha256">ab8df35061dfa0285069b843f24a7076e31266d9a8abe8282340bcb936aa61d7</checksum>
+    <open-checksum type="sha256">2bce9554ce4496cef34b5cd69f186f7f3143c7cabae8fa384fc5c9eeab326f7f</open-checksum>
+    <location href="repodata/ab8df35061dfa0285069b843f24a7076e31266d9a8abe8282340bcb936aa61d7-primary.sqlite.bz2"/>
+    <timestamp>1666177486</timestamp>
+    <size>3558</size>
+    <open-size>106496</open-size>
+    <database_version>10</database_version>
+  </data>
+  <data type="filelists_db">
+    <checksum type="sha256">8bcf6d40db4e922934ac47e8ac7fb8d15bdacf579af8c819d2134ed54d30550b</checksum>
+    <open-checksum type="sha256">f7001d1df7f5f7e4898919b15710bea8ed9711ce42faf68e22b757e63169b1fb</open-checksum>
+    <location href="repodata/8bcf6d40db4e922934ac47e8ac7fb8d15bdacf579af8c819d2134ed54d30550b-filelists.sqlite.bz2"/>
+    <timestamp>1666177486</timestamp>
+    <size>2360</size>
+    <open-size>28672</open-size>
+    <database_version>10</database_version>
+  </data>
+  <data type="other_db">
+    <checksum type="sha256">01b82e9eb7ee9151f283c6e761ae450de18ed2d64b5e32de88689eaf95216a80</checksum>
+    <open-checksum type="sha256">07f5b9750af1e440d37ca216e719dd288149e79e9132f2fdccb6f73b2e5dd541</open-checksum>
+    <location href="repodata/01b82e9eb7ee9151f283c6e761ae450de18ed2d64b5e32de88689eaf95216a80-other.sqlite.bz2"/>
+    <timestamp>1666177486</timestamp>
+    <size>2196</size>
+    <open-size>32768</open-size>
+    <database_version>10</database_version>
+  </data>
+</repomd>

tests/data/test_create_packages_json/test_repo_2/repodata/repomd.xml

@@ -0,0 +1,55 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<repomd xmlns="http://linux.duke.edu/metadata/repo" xmlns:rpm="http://linux.duke.edu/metadata/rpm">
+  <revision>1666177500</revision>
+  <data type="primary">
+    <checksum type="sha256">a1d342aa7cef3a2034fc3f9d6ee02d63572780bc76e61749a57e50b6b3ca9869</checksum>
+    <open-checksum type="sha256">a9e3eae447dd44282d7d96db5f15f049b757925397adb752f4df982176bab7e0</open-checksum>
+    <location href="repodata/a1d342aa7cef3a2034fc3f9d6ee02d63572780bc76e61749a57e50b6b3ca9869-primary.xml.gz"/>
+    <timestamp>1666177500</timestamp>
+    <size>3501</size>
+    <open-size>37296</open-size>
+  </data>
+  <data type="filelists">
+    <checksum type="sha256">6778922d5853d20f213ae7702699a76f1e87e55d6bfb5e4ac6a117d904d47b3c</checksum>
+    <open-checksum type="sha256">e30b666d9d88a70de69a08f45e6696bcd600c45485d856bd0213395d7da7bd49</open-checksum>
+    <location href="repodata/6778922d5853d20f213ae7702699a76f1e87e55d6bfb5e4ac6a117d904d47b3c-filelists.xml.gz"/>
+    <timestamp>1666177500</timestamp>
+    <size>27624</size>
+    <open-size>318187</open-size>
+  </data>
+  <data type="other">
+    <checksum type="sha256">5a60d79d8bce6a805f4fdb22fd891524359dce8ccc665c0b54e7299e79debe84</checksum>
+    <open-checksum type="sha256">b18138f4a3de45714e578fb1f30b7ec54fdcdaf1a22585891625b6af0894388e</open-checksum>
+    <location href="repodata/5a60d79d8bce6a805f4fdb22fd891524359dce8ccc665c0b54e7299e79debe84-other.xml.gz"/>
+    <timestamp>1666177500</timestamp>
+    <size>1876</size>
+    <open-size>28701</open-size>
+  </data>
+  <data type="primary_db">
+    <checksum type="sha256">c27bc2ce947173aba305041552c3c6d8db71442c1a2e5dcaf35ff750fe0469fc</checksum>
+    <open-checksum type="sha256">586e1af8934229925adb9e746ae5ced119859dfd97f4e3237399bb36a7d7f071</open-checksum>
+    <location href="repodata/c27bc2ce947173aba305041552c3c6d8db71442c1a2e5dcaf35ff750fe0469fc-primary.sqlite.bz2"/>
+    <timestamp>1666177500</timestamp>
+    <size>11528</size>
+    <open-size>126976</open-size>
+    <database_version>10</database_version>
+  </data>
+  <data type="filelists_db">
+    <checksum type="sha256">ed350865982e7a1e45b144839b56eac888e5d8f680571dd2cd06b37dc83e0fd8</checksum>
+    <open-checksum type="sha256">697903989d0f77de2d44a2b603e75c9b4ca23b3795eb136d175caf5666ce6459</open-checksum>
+    <location href="repodata/ed350865982e7a1e45b144839b56eac888e5d8f680571dd2cd06b37dc83e0fd8-filelists.sqlite.bz2"/>
+    <timestamp>1666177500</timestamp>
+    <size>20440</size>
+    <open-size>163840</open-size>
+    <database_version>10</database_version>
+  </data>
+  <data type="other_db">
+    <checksum type="sha256">35eff699131e0976429144c6f4514d21568177dc64bb4091c3ff62f76b293725</checksum>
+    <open-checksum type="sha256">3bd999a1bdf300df836a4607b7b75f845d8e1432e3e4e1ab6f0c7cc8a853db39</open-checksum>
+    <location href="repodata/35eff699131e0976429144c6f4514d21568177dc64bb4091c3ff62f76b293725-other.sqlite.bz2"/>
+    <timestamp>1666177500</timestamp>
+    <size>4471</size>
+    <open-size>49152</open-size>
+    <database_version>10</database_version>
+  </data>
+</repomd>

tests/fixtures/extraiso-tweaked-expected.treeinfo

@@ -0,0 +1,58 @@
+[checksums]
+images/boot.iso = sha256:fc8a4be604b6425746f12fa706116eb940f93358f036b8fbbe518b516cb6870c
+
+[general]
+; WARNING.0 = This section provides compatibility with pre-productmd treeinfos.
+; WARNING.1 = Read productmd documentation for details about new format.
+arch = x86_64
+family = Test
+name = Test 1.0
+packagedir = Packages
+platforms = x86_64,xen
+repository = .
+timestamp = 1531881582
+variant = Server
+variants = Client,Server
+version = 1.0
+
+[header]
+type = productmd.treeinfo
+version = 1.2
+
+[images-x86_64]
+boot.iso = images/boot.iso
+
+[images-xen]
+initrd = images/pxeboot/initrd.img
+kernel = images/pxeboot/vmlinuz
+
+[release]
+name = Test
+short = T
+version = 1.0
+
+[stage2]
+mainimage = images/install.img
+
+[tree]
+arch = x86_64
+build_timestamp = 1531881582
+platforms = x86_64,xen
+variants = Client,Server
+
+[variant-Client]
+id = Client
+name = Client
+packages = ../../../Client/x86_64/os/Packages
+repository = ../../../Client/x86_64/os
+type = variant
+uid = Client
+
+[variant-Server]
+id = Server
+name = Server
+packages = Packages
+repository = .
+type = variant
+uid = Server
+

tests/fixtures/mmds/modules/x86_64/module:master-20190318-abcdef

@@ -0,0 +1,20 @@
+---
+document: modulemd
+version: 2
+data:
+  name: module
+  stream: master
+  version: 20190318
+  context: abcdef
+  arch: x86_64
+  summary: Dummy module
+  description: Dummy module
+  license:
+    module:
+    - Beerware
+    content:
+    - Beerware
+  artifacts:
+    rpms:
+      - foobar-0:1.0-1.noarch
+...

tests/fixtures/mmds/modules/x86_64/modulemd.x86_64.txt

@@ -0,0 +1,20 @@
+---
+document: modulemd
+version: 2
+data:
+  name: module
+  stream: master
+  version: 20190318
+  context: abcdef
+  arch: x86_64
+  summary: Dummy module
+  description: Dummy module
+  license:
+    module:
+    - Beerware
+    content:
+    - Beerware
+  artifacts:
+    rpms:
+      - foobar-0:1.0-1.noarch
+...

tests/fixtures/mmds/modules/x86_64/scratch-module.x86_64.txt

@@ -0,0 +1,20 @@
+---
+document: modulemd
+version: 2
+data:
+  name: scratch-module
+  stream: master
+  version: 20200710
+  context: abcdef
+  arch: x86_64
+  summary: Dummy module
+  description: Dummy module
+  license:
+    module:
+    - Beerware
+    content:
+    - Beerware
+  artifacts:
+    rpms:
+      - foobar-0:1.0-1.noarch
+...

tests/fixtures/mmds/modules/x86_64/scratch-module:master-20190318-abcdef

@@ -0,0 +1,20 @@
+---
+document: modulemd
+version: 2
+data:
+  name: scratch-module
+  stream: master
+  version: 20200710
+  context: abcdef
+  arch: x86_64
+  summary: Dummy module
+  description: Dummy module
+  license:
+    module:
+    - Beerware
+    content:
+    - Beerware
+  artifacts:
+    rpms:
+      - foobar-0:1.0-1.noarch
+...

tests/helpers.py

@@ -2,12 +2,13 @@
 
 import difflib
 import errno
+import hashlib
 import os
 import shutil
 import tempfile
 from collections import defaultdict
 
-import mock
+from unittest import mock
 import six
 from kobo.rpmlib import parse_nvr
 
@@ -21,6 +22,15 @@ from pungi import paths, checks
 from pungi.module_util import Modulemd
 
 
+GIT_WITH_CREDS = [
+    "git",
+    "-c",
+    "credential.useHttpPath=true",
+    "-c",
+    "credential.helper=!ch",
+]
+
+
 class BaseTestCase(unittest.TestCase):
     def assertFilesEqual(self, fn1, fn2):
         with open(fn1, "rb") as f1:
@@ -158,6 +168,20 @@ class IterableMock(mock.Mock):
         return iter([])
 
 
+class FSKojiDownloader(object):
+    """Mock for KojiDownloadProxy that checks provided path."""
+
+    def get_file(self, path, validator=None):
+        return path if os.path.isfile(path) else None
+
+
+class DummyKojiDownloader(object):
+    """Mock for KojiDownloadProxy that always finds the file in original location."""
+
+    def get_file(self, path, validator=None):
+        return path
+
+
 class DummyCompose(object):
     def __init__(self, topdir, config):
         self.supported = True
@@ -232,6 +256,8 @@ class DummyCompose(object):
         self.cache_region = None
         self.containers_metadata = {}
         self.load_old_compose_config = mock.Mock(return_value=None)
+        self.koji_downloader = DummyKojiDownloader()
+        self.koji_downloader.path_prefix = "/prefix"
 
     def setup_optional(self):
         self.all_variants["Server-optional"] = MockVariant(
@@ -336,3 +362,9 @@ def fake_run_in_threads(func, params, threads=None):
     """Like run_in_threads from Kobo, but actually runs tasks serially."""
     for num, param in enumerate(params):
         func(None, param, num)
+
+
+def hash_string(alg, s):
+    m = hashlib.new(alg)
+    m.update(s.encode("utf-8"))
+    return m.hexdigest()

tests/test_arch.py

@@ -1,6 +1,6 @@
 # -*- coding: utf-8 -*-
 
-import mock
+from unittest import mock
 import unittest
 
 from pungi.arch import (

tests/test_arguments.py

@@ -1,4 +1,4 @@
-import mock
+from unittest import mock
 
 try:
     import unittest2 as unittest

tests/test_buildinstall.py

@@ -1,15 +1,15 @@
 # -*- coding: utf-8 -*-
 
-
 try:
     import unittest2 as unittest
 except ImportError:
     import unittest
 
-import mock
+from unittest import mock
 import six
 from copy import copy
 from six.moves import StringIO
+from ddt import ddt, data
 
 import os
 
@@ -254,6 +254,7 @@ class TestBuildinstallPhase(PungiTestCase):
     def test_starts_threads_for_each_cmd_with_lorax_koji_plugin(
         self, get_volid, poolCls
     ):
+        topurl = "https://example.com/composes/"
         compose = BuildInstallCompose(
             self.topdir,
             {
@@ -264,6 +265,7 @@ class TestBuildinstallPhase(PungiTestCase):
                 "buildinstall_method": "lorax",
                 "lorax_use_koji_plugin": True,
                 "disc_types": {"dvd": "DVD"},
+                "translate_paths": [(self.topdir, topurl)],
             },
         )
 
@@ -280,9 +282,9 @@ class TestBuildinstallPhase(PungiTestCase):
                 "version": "1",
                 "release": "1",
                 "sources": [
-                    self.topdir + "/work/amd64/repo/p1",
-                    self.topdir + "/work/amd64/repo/p2",
-                    self.topdir + "/work/amd64/comps_repo_Server",
+                    topurl + "work/amd64/repo/p1",
+                    topurl + "work/amd64/repo/p2",
+                    topurl + "work/amd64/comps_repo_Server",
                 ],
                 "variant": "Server",
                 "installpkgs": ["bash", "vim"],
@@ -299,7 +301,6 @@ class TestBuildinstallPhase(PungiTestCase):
                 "rootfs-size": None,
                 "dracut-args": [],
                 "skip_branding": False,
-                "outputdir": self.topdir + "/work/amd64/buildinstall/Server",
                 "squashfs_only": False,
                 "configuration_file": None,
             },
@@ -308,9 +309,9 @@ class TestBuildinstallPhase(PungiTestCase):
                 "version": "1",
                 "release": "1",
                 "sources": [
-                    self.topdir + "/work/amd64/repo/p1",
-                    self.topdir + "/work/amd64/repo/p2",
-                    self.topdir + "/work/amd64/comps_repo_Client",
+                    topurl + "work/amd64/repo/p1",
+                    topurl + "work/amd64/repo/p2",
+                    topurl + "work/amd64/comps_repo_Client",
                 ],
                 "variant": "Client",
                 "installpkgs": [],
@@ -327,7 +328,6 @@ class TestBuildinstallPhase(PungiTestCase):
                 "rootfs-size": None,
                 "dracut-args": [],
                 "skip_branding": False,
-                "outputdir": self.topdir + "/work/amd64/buildinstall/Client",
                 "squashfs_only": False,
                 "configuration_file": None,
             },
@@ -336,9 +336,9 @@ class TestBuildinstallPhase(PungiTestCase):
                 "version": "1",
                 "release": "1",
                 "sources": [
-                    self.topdir + "/work/x86_64/repo/p1",
-                    self.topdir + "/work/x86_64/repo/p2",
-                    self.topdir + "/work/x86_64/comps_repo_Server",
+                    topurl + "work/x86_64/repo/p1",
+                    topurl + "work/x86_64/repo/p2",
+                    topurl + "work/x86_64/comps_repo_Server",
                 ],
                 "variant": "Server",
                 "installpkgs": ["bash", "vim"],
@@ -355,7 +355,6 @@ class TestBuildinstallPhase(PungiTestCase):
                 "rootfs-size": None,
                 "dracut-args": [],
                 "skip_branding": False,
-                "outputdir": self.topdir + "/work/x86_64/buildinstall/Server",
                 "squashfs_only": False,
                 "configuration_file": None,
             },
@@ -473,70 +472,6 @@ class TestBuildinstallPhase(PungiTestCase):
             ],
         )
 
-    @mock.patch("pungi.phases.buildinstall.ThreadPool")
-    @mock.patch("pungi.phases.buildinstall.LoraxWrapper")
-    @mock.patch("pungi.phases.buildinstall.get_volid")
-    def test_starts_threads_for_each_cmd_with_buildinstall(
-        self, get_volid, loraxCls, poolCls
-    ):
-        compose = BuildInstallCompose(
-            self.topdir,
-            {
-                "bootable": True,
-                "release_name": "Test",
-                "release_short": "t",
-                "release_version": "1",
-                "buildinstall_method": "buildinstall",
-                "disc_types": {"dvd": "DVD"},
-            },
-        )
-
-        get_volid.return_value = "vol_id"
-
-        phase = BuildinstallPhase(compose, self._make_pkgset_phase(["p1"]))
-
-        phase.run()
-
-        # Two items added for processing in total.
-        pool = poolCls.return_value
-        self.assertEqual(2, len(pool.queue_put.mock_calls))
-
-        # Obtained correct lorax commands.
-        six.assertCountEqual(
-            self,
-            loraxCls.return_value.get_buildinstall_cmd.mock_calls,
-            [
-                mock.call(
-                    "Test",
-                    "1",
-                    "1",
-                    [self.topdir + "/work/x86_64/repo/p1"],
-                    self.topdir + "/work/x86_64/buildinstall",
-                    buildarch="x86_64",
-                    is_final=True,
-                    volid="vol_id",
-                ),
-                mock.call(
-                    "Test",
-                    "1",
-                    "1",
-                    [self.topdir + "/work/amd64/repo/p1"],
-                    self.topdir + "/work/amd64/buildinstall",
-                    buildarch="amd64",
-                    is_final=True,
-                    volid="vol_id",
-                ),
-            ],
-        )
-        six.assertCountEqual(
-            self,
-            get_volid.mock_calls,
-            [
-                mock.call(compose, "x86_64", disc_type="DVD"),
-                mock.call(compose, "amd64", disc_type="DVD"),
-            ],
-        )
-
     @mock.patch("pungi.phases.buildinstall.get_file")
     @mock.patch("pungi.phases.buildinstall.ThreadPool")
     @mock.patch("pungi.phases.buildinstall.LoraxWrapper")
@@ -1209,6 +1144,7 @@ class BuildinstallThreadTestCase(PungiTestCase):
                 "buildinstall_method": "lorax",
                 "runroot_tag": "rrt",
                 "koji_profile": "koji",
+                "koji_cache": "/tmp",
                 "runroot_weights": {"buildinstall": 123},
             },
         )
@@ -1297,9 +1233,9 @@ class BuildinstallThreadTestCase(PungiTestCase):
     @mock.patch("pungi.wrappers.kojiwrapper.KojiWrapper")
     @mock.patch("pungi.wrappers.kojiwrapper.get_buildroot_rpms")
     @mock.patch("pungi.phases.buildinstall.run")
-    @mock.patch("pungi.phases.buildinstall.move_all")
+    @mock.patch("pungi.phases.buildinstall.download_and_extract_archive")
     def test_buildinstall_thread_with_lorax_using_koji_plugin(
-        self, move_all, run, get_buildroot_rpms, KojiWrapperMock, mock_tweak, mock_link
+        self, download, run, get_buildroot_rpms, KojiWrapperMock, mock_tweak, mock_link
     ):
         compose = BuildInstallCompose(
             self.topdir,
@@ -1308,6 +1244,7 @@ class BuildinstallThreadTestCase(PungiTestCase):
                 "lorax_use_koji_plugin": True,
                 "runroot_tag": "rrt",
                 "koji_profile": "koji",
+                "koji_cache": "/tmp",
                 "runroot_weights": {"buildinstall": 123},
             },
         )
@@ -1344,9 +1281,8 @@ class BuildinstallThreadTestCase(PungiTestCase):
                     self.cmd,
                     channel=None,
                     packages=["lorax"],
-                    mounts=[self.topdir],
                     weight=123,
-                    chown_uid=os.getuid(),
+                    chown_uid=None,
                 )
             ],
         )
@@ -1387,150 +1323,17 @@ class BuildinstallThreadTestCase(PungiTestCase):
             [mock.call(compose, "x86_64", compose.variants["Server"], False)],
         )
         self.assertEqual(
-            move_all.call_args_list,
-            [
-                mock.call(os.path.join(destdir, "results"), destdir, rm_src_dir=True),
-                mock.call(
-                    os.path.join(destdir, "logs"),
-                    os.path.join(self.topdir, "logs/x86_64/buildinstall-Server-logs"),
-                    rm_src_dir=True,
-                ),
-            ],
-        )
-
-    @mock.patch("pungi.phases.buildinstall.link_boot_iso")
-    @mock.patch("pungi.phases.buildinstall.tweak_buildinstall")
-    @mock.patch("pungi.wrappers.kojiwrapper.KojiWrapper")
-    @mock.patch("pungi.wrappers.kojiwrapper.get_buildroot_rpms")
-    @mock.patch("pungi.phases.buildinstall.run")
-    def test_buildinstall_thread_with_buildinstall_in_runroot(
-        self, run, get_buildroot_rpms, KojiWrapperMock, mock_tweak, mock_link
-    ):
-        compose = BuildInstallCompose(
-            self.topdir,
-            {
-                "buildinstall_method": "buildinstall",
-                "runroot_tag": "rrt",
-                "koji_profile": "koji",
-            },
-        )
-
-        get_buildroot_rpms.return_value = ["bash", "zsh"]
-
-        get_runroot_cmd = KojiWrapperMock.return_value.get_runroot_cmd
-
-        run_runroot_cmd = KojiWrapperMock.return_value.run_runroot_cmd
-        run_runroot_cmd.return_value = {
-            "output": "Foo bar baz",
-            "retcode": 0,
-            "task_id": 1234,
-        }
-
-        t = BuildinstallThread(self.pool)
-
-        with mock.patch("time.sleep"):
-            pkgset_phase = self._make_pkgset_phase(["p1"])
-            t.process((compose, "amd64", None, self.cmd, pkgset_phase), 0)
-
-        destdir = os.path.join(self.topdir, "work/amd64/buildinstall")
-        self.assertEqual(
-            get_runroot_cmd.mock_calls,
-            [
-                mock.call(
-                    "rrt",
-                    "amd64",
-                    self.cmd,
-                    channel=None,
-                    use_shell=True,
-                    packages=["anaconda"],
-                    mounts=[self.topdir],
-                    weight=None,
-                    chown_paths=[destdir],
-                )
-            ],
-        )
-        self.assertEqual(
-            run_runroot_cmd.mock_calls,
-            [
-                mock.call(
-                    get_runroot_cmd.return_value,
-                    log_file=self.topdir + "/logs/amd64/buildinstall.amd64.log",
-                )
-            ],
-        )
-        with open(self.topdir + "/logs/amd64/buildinstall-RPMs.amd64.log") as f:
-            rpms = f.read().strip().split("\n")
-        six.assertCountEqual(self, rpms, ["bash", "zsh"])
-        six.assertCountEqual(self, self.pool.finished_tasks, [(None, "amd64")])
-        six.assertCountEqual(
-            self,
-            mock_tweak.call_args_list,
+            download.call_args_list,
             [
+                mock.call(compose, 1234, "results.tar.gz", destdir),
                 mock.call(
                     compose,
-                    destdir,
-                    os.path.join(self.topdir, "compose", var, "amd64/os"),
-                    "amd64",
-                    var,
-                    "",
-                    "dummy-volid",
-                    self.pool.kickstart_file,
-                )
-                for var in ["Client", "Server"]
+                    1234,
+                    "logs.tar.gz",
+                    os.path.join(self.topdir, "logs/x86_64/buildinstall-Server-logs"),
+                ),
             ],
         )
-        six.assertCountEqual(
-            self,
-            mock_link.call_args_list,
-            [
-                mock.call(compose, "amd64", compose.variants["Client"], False),
-                mock.call(compose, "amd64", compose.variants["Server"], False),
-            ],
-        )
-
-    @mock.patch("pungi.wrappers.kojiwrapper.KojiWrapper")
-    @mock.patch("pungi.wrappers.kojiwrapper.get_buildroot_rpms")
-    @mock.patch("pungi.phases.buildinstall.run")
-    def test_buildinstall_fail_exit_code(
-        self, run, get_buildroot_rpms, KojiWrapperMock
-    ):
-        compose = BuildInstallCompose(
-            self.topdir,
-            {
-                "buildinstall_method": "buildinstall",
-                "runroot_tag": "rrt",
-                "koji_profile": "koji",
-                "failable_deliverables": [("^.+$", {"*": ["buildinstall"]})],
-            },
-        )
-
-        get_buildroot_rpms.return_value = ["bash", "zsh"]
-
-        run_runroot_cmd = KojiWrapperMock.return_value.run_runroot_cmd
-        run_runroot_cmd.return_value = {
-            "output": "Foo bar baz",
-            "retcode": 1,
-            "task_id": 1234,
-        }
-
-        t = BuildinstallThread(self.pool)
-
-        with mock.patch("time.sleep"):
-            pkgset_phase = self._make_pkgset_phase(["p1"])
-            t.process((compose, "x86_64", None, self.cmd, pkgset_phase), 0)
-
-        compose._logger.error.assert_has_calls(
-            [
-                mock.call(
-                    "[FAIL] Buildinstall (variant None, arch x86_64) failed, but going on anyway."  # noqa: E501
-                ),
-                mock.call(
-                    "Runroot task failed: 1234. See %s/logs/x86_64/buildinstall.x86_64.log for more details."  # noqa: E501
-                    % self.topdir
-                ),
-            ]
-        )
-        self.assertEqual(self.pool.finished_tasks, set())
 
     @mock.patch("pungi.wrappers.kojiwrapper.KojiWrapper")
     @mock.patch("pungi.wrappers.kojiwrapper.get_buildroot_rpms")
@@ -1542,6 +1345,7 @@ class BuildinstallThreadTestCase(PungiTestCase):
                 "buildinstall_method": "lorax",
                 "runroot_tag": "rrt",
                 "koji_profile": "koji",
+                "koji_cache": "/tmp",
                 "failable_deliverables": [("^.+$", {"*": ["buildinstall"]})],
             },
         )
@@ -1591,6 +1395,7 @@ class BuildinstallThreadTestCase(PungiTestCase):
                 "buildinstall_method": "lorax",
                 "runroot_tag": "rrt",
                 "koji_profile": "koji",
+                "koji_cache": "/tmp",
                 "failable_deliverables": [("^.+$", {"*": ["buildinstall"]})],
             },
         )
@@ -1663,6 +1468,7 @@ class BuildinstallThreadTestCase(PungiTestCase):
                 "buildinstall_method": "lorax",
                 "runroot_tag": "rrt",
                 "koji_profile": "koji",
+                "koji_cache": "/tmp",
                 "failable_deliverables": [("^.+$", {"*": ["buildinstall"]})],
             },
         )
@@ -1701,6 +1507,7 @@ class BuildinstallThreadTestCase(PungiTestCase):
                 "buildinstall_method": "lorax",
                 "runroot_tag": "rrt",
                 "koji_profile": "koji",
+                "koji_cache": "/tmp",
                 "runroot_weights": {"buildinstall": 123},
                 "buildinstall_topdir": "/buildinstall_topdir",
             },
@@ -1810,6 +1617,7 @@ class BuildinstallThreadTestCase(PungiTestCase):
                 "buildinstall_method": "lorax",
                 "runroot_tag": "rrt",
                 "koji_profile": "koji",
+                "koji_cache": "/tmp",
             },
         )
 
@@ -2005,6 +1813,7 @@ class BuildinstallThreadTestCase(PungiTestCase):
         self.assertEqual(ret, None)
 
 
+@ddt
 class TestSymlinkIso(PungiTestCase):
     def setUp(self):
         super(TestSymlinkIso, self).setUp()
@@ -2020,8 +1829,13 @@ class TestSymlinkIso(PungiTestCase):
     @mock.patch("pungi.phases.buildinstall.get_file_size")
     @mock.patch("pungi.phases.buildinstall.iso")
     @mock.patch("pungi.phases.buildinstall.run")
-    def test_hardlink(self, run, iso, get_file_size, get_mtime, ImageCls):
-        self.compose.conf = {"buildinstall_symlink": False, "disc_types": {}}
+    @data(['Server'], ['BaseOS'])
+    def test_hardlink(self, netinstall_variants, run, iso, get_file_size, get_mtime, ImageCls):
+        self.compose.conf = {
+            "buildinstall_symlink": False,
+            "disc_types": {},
+            "netinstall_variants": netinstall_variants,
+        }
         get_file_size.return_value = 1024
         get_mtime.return_value = 13579
 
@@ -2048,7 +1862,7 @@ class TestSymlinkIso(PungiTestCase):
         )
         self.assertEqual(iso.get_implanted_md5.mock_calls, [mock.call(tgt)])
         self.assertEqual(iso.get_manifest_cmd.mock_calls, [mock.call("image-name")])
-        self.assertEqual(iso.get_volume_id.mock_calls, [mock.call(tgt)])
+        self.assertEqual(iso.get_volume_id.mock_calls, [mock.call(tgt, None)])
         self.assertEqual(
             run.mock_calls,
             [
@@ -2071,9 +1885,14 @@ class TestSymlinkIso(PungiTestCase):
         self.assertEqual(image.bootable, True)
         self.assertEqual(image.implant_md5, iso.get_implanted_md5.return_value)
         self.assertEqual(image.can_fail, False)
-        self.assertEqual(
-            self.compose.im.add.mock_calls, [mock.call("Server", "x86_64", image)]
-        )
+        if 'Server' in netinstall_variants:
+            self.assertEqual(
+                self.compose.im.add.mock_calls, [mock.call("Server", "x86_64", image)]
+            )
+        else:
+            self.assertEqual(
+                self.compose.im.add.mock_calls, []
+            )
 
     @mock.patch("pungi.phases.buildinstall.Image")
     @mock.patch("pungi.phases.buildinstall.get_mtime")
@@ -2086,6 +1905,7 @@ class TestSymlinkIso(PungiTestCase):
         self.compose.conf = {
             "buildinstall_symlink": False,
             "disc_types": {"boot": "netinst"},
+            "netinstall_variants": ['Server'],
         }
         get_file_size.return_value = 1024
         get_mtime.return_value = 13579
@@ -2113,7 +1933,7 @@ class TestSymlinkIso(PungiTestCase):
         )
         self.assertEqual(iso.get_implanted_md5.mock_calls, [mock.call(tgt)])
         self.assertEqual(iso.get_manifest_cmd.mock_calls, [mock.call("image-name")])
-        self.assertEqual(iso.get_volume_id.mock_calls, [mock.call(tgt)])
+        self.assertEqual(iso.get_volume_id.mock_calls, [mock.call(tgt, None)])
         self.assertEqual(
             run.mock_calls,
             [

tests/test_checks.py

@@ -1,6 +1,6 @@
 # -*- coding: utf-8 -*-
 
-import mock
+from unittest import mock
 
 try:
     import unittest2 as unittest

tests/test_compose.py

@@ -1,7 +1,7 @@
 # -*- coding: utf-8 -*-
 
 import logging
-import mock
+from unittest import mock
 
 try:
     import unittest2 as unittest
@@ -656,6 +656,7 @@ class ComposeTestCase(unittest.TestCase):
             mocked_requests.post.assert_called_once_with(
                 "https://cts.localhost.tld/api/1/composes/",
                 auth=mock.ANY,
+                data=None,
                 json=expected_json,
             )
 
@@ -794,12 +795,16 @@ class TracebackTest(unittest.TestCase):
         shutil.rmtree(self.tmp_dir)
         self.patcher.stop()
 
-    def assertTraceback(self, filename):
+    def assertTraceback(self, filename, show_locals=True):
         self.assertTrue(
             os.path.isfile("%s/logs/global/%s.global.log" % (self.tmp_dir, filename))
         )
         self.assertEqual(
-            self.Traceback.mock_calls, [mock.call(), mock.call().get_traceback()]
+            self.Traceback.mock_calls,
+            [
+                mock.call(show_locals=show_locals),
+                mock.call(show_locals=show_locals).get_traceback(),
+            ],
         )
 
     def test_traceback_default(self):
@@ -824,8 +829,8 @@ class RetryRequestTest(unittest.TestCase):
         self.assertEqual(
             mocked_requests.mock_calls,
             [
-                mock.call.post(url, json=None, auth=None),
-                mock.call.post(url, json=None, auth=None),
+                mock.call.post(url, data=None, json=None, auth=None),
+                mock.call.post(url, data=None, json=None, auth=None),
             ],
         )
         self.assertEqual(rv.status_code, 200)
@@ -841,5 +846,5 @@ class RetryRequestTest(unittest.TestCase):
 
         self.assertEqual(
             mocked_requests.mock_calls,
-            [mock.call.post(url, json=None, auth=None)],
+            [mock.call.post(url, data=None, json=None, auth=None)],
         )

tests/test_config.py

@@ -7,7 +7,7 @@ except ImportError:
     import unittest
 
 import six
-import mock
+from unittest import mock
 
 from pungi import checks
 from tests.helpers import load_config, PKGSET_REPOS
@@ -223,22 +223,6 @@ class BuildinstallConfigTestCase(ConfigTestCase):
 
         self.assertValidation(cfg, [])
 
-    def test_buildinstall_with_lorax_options(self):
-        cfg = load_config(
-            PKGSET_REPOS,
-            buildinstall_method="buildinstall",
-            lorax_options=[("^Server$", {})],
-        )
-
-        self.assertValidation(
-            cfg,
-            [
-                checks.CONFLICTS.format(
-                    "buildinstall_method", "buildinstall", "lorax_options"
-                )
-            ],
-        )
-
     def test_lorax_with_lorax_options(self):
         cfg = load_config(PKGSET_REPOS, buildinstall_method="lorax", lorax_options=[])
 
@@ -440,7 +424,7 @@ class LiveMediaConfigTestCase(ConfigTestCase):
             live_media_version="Rawhide",
         )
 
-        resolve_git_url.side_effect = lambda x: x.replace("HEAD", "CAFE")
+        resolve_git_url.side_effect = lambda x, _helper: x.replace("HEAD", "CAFE")
 
         self.assertValidation(cfg)
         self.assertEqual(cfg["live_media_ksurl"], "git://example.com/repo.git#CAFE")

tests/test_config_validate_script.py

@@ -1,7 +1,7 @@
 # -*- coding: utf-8 -*-
 
 
-import mock
+from unittest import mock
 import os
 import six
 

tests/test_create_extra_repo.py

@@ -0,0 +1,228 @@
+# coding=utf-8
+
+import os
+from unittest import TestCase, mock, main
+
+import yaml
+
+from pungi.scripts.create_extra_repo import CreateExtraRepo, ExtraVariantInfo, RepoInfo
+
+FOLDER_WITH_TEST_DATA = os.path.join(
+    os.path.dirname(
+        os.path.abspath(__file__)
+    ),
+    'data/test_create_extra_repo/',
+)
+
+TEST_MODULE_INFO = yaml.load("""
+---
+document: modulemd
+version: 2
+data:
+  name: perl-App-cpanminus
+  stream: 1.7044
+  version: 8030020210126085450
+  context: 3a33b840
+  arch: x86_64
+  summary: Get, unpack, build and install CPAN modules
+  description: >
+    This is a CPAN client that requires zero configuration, and stands alone but it's
+    maintainable and extensible with plug-ins and friendly to shell scripting.
+  license:
+    module:
+    - MIT
+    content:
+    - (GPL+ or Artistic) and GPLv2+
+    - ASL 2.0
+    - GPL+ or Artistic
+  dependencies:
+  - buildrequires:
+      perl: [5.30]
+      platform: [el8.3.0]
+    requires:
+      perl: [5.30]
+      perl-YAML: []
+      platform: [el8]
+  references:
+    community: https://metacpan.org/release/App-cpanminus
+  profiles:
+    common:
+      description: App-cpanminus distribution
+      rpms:
+      - perl-App-cpanminus
+  api:
+    rpms:
+    - perl-App-cpanminus
+  filter:
+    rpms:
+    - perl-CPAN-DistnameInfo-dummy
+    - perl-Test-Deep
+  buildopts:
+    rpms:
+      macros: >
+        %_without_perl_CPAN_Meta_Check_enables_extra_test 1
+  components:
+    rpms:
+      perl-App-cpanminus:
+        rationale: The API.
+        ref: perl-App-cpanminus-1.7044-5.module+el8.2.0+4278+abcfa81a.src.rpm
+        buildorder: 1
+        arches: [i686, x86_64]
+      perl-CPAN-DistnameInfo:
+        rationale: Run-time dependency.
+        ref: stream-0.12-rhel-8.3.0
+        arches: [i686, x86_64]
+      perl-CPAN-Meta-Check:
+        rationale: Run-time dependency.
+        ref: perl-CPAN-Meta-Check-0.014-6.module+el8.2.0+4278+abcfa81a.src.rpm
+        buildorder: 1
+        arches: [i686, x86_64]
+      perl-File-pushd:
+        rationale: Run-time dependency.
+        ref: perl-File-pushd-1.014-6.module+el8.2.0+4278+abcfa81a.src.rpm
+        arches: [i686, x86_64]
+      perl-Module-CPANfile:
+        rationale: Run-time dependency.
+        ref: perl-Module-CPANfile-1.1002-7.module+el8.2.0+4278+abcfa81a.src.rpm
+        arches: [i686, x86_64]
+      perl-Parse-PMFile:
+        rationale: Run-time dependency.
+        ref: perl-Parse-PMFile-0.41-7.module+el8.2.0+4278+abcfa81a.src.rpm
+        arches: [i686, x86_64]
+      perl-String-ShellQuote:
+        rationale: Run-time dependency.
+        ref: perl-String-ShellQuote-1.04-24.module+el8.2.0+4278+abcfa81a.src.rpm
+        arches: [i686, x86_64]
+      perl-Test-Deep:
+        rationale: Build-time dependency.
+        ref: stream-1.127-rhel-8.3.0
+        arches: [i686, x86_64]
+  artifacts:
+    rpms:
+    - perl-App-cpanminus-0:1.7044-5.module_el8.3.0+2027+c8990d1d.noarch
+    - perl-App-cpanminus-0:1.7044-5.module_el8.3.0+2027+c8990d1d.src
+    - perl-CPAN-Meta-Check-0:0.014-6.module_el8.3.0+2027+c8990d1d.noarch
+    - perl-CPAN-Meta-Check-0:0.014-6.module_el8.3.0+2027+c8990d1d.src
+    - perl-File-pushd-0:1.014-6.module_el8.3.0+2027+c8990d1d.noarch
+    - perl-File-pushd-0:1.014-6.module_el8.3.0+2027+c8990d1d.src
+    - perl-Module-CPANfile-0:1.1002-7.module_el8.3.0+2027+c8990d1d.noarch
+    - perl-Module-CPANfile-0:1.1002-7.module_el8.3.0+2027+c8990d1d.src
+    - perl-Parse-PMFile-0:0.41-7.module_el8.3.0+2027+c8990d1d.noarch
+    - perl-Parse-PMFile-0:0.41-7.module_el8.3.0+2027+c8990d1d.src
+    - perl-String-ShellQuote-0:1.04-24.module_el8.3.0+2027+c8990d1d.noarch
+    - perl-String-ShellQuote-0:1.04-24.module_el8.3.0+2027+c8990d1d.src
+...
+""", Loader=yaml.BaseLoader)
+
+TEST_REPO_INFO = RepoInfo(
+    path=FOLDER_WITH_TEST_DATA,
+    folder='test_repo',
+    is_remote=False,
+)
+TEST_VARIANT_INFO = ExtraVariantInfo(
+    name='TestRepo',
+    arch='x86_64',
+    packages=[],
+    modules=[],
+    repos=[TEST_REPO_INFO]
+)
+
+BS_BUILD_INFO = {
+    'build_platforms': [
+        {
+            'architectures': ['non_fake_arch', 'fake_arch'],
+            'name': 'fake_platform'
+        }
+    ]
+}
+
+
+class TestCreteExtraRepo(TestCase):
+
+    maxDiff = None
+
+    def test_01_get_repo_info_from_bs_repo(self):
+        auth_token = 'fake_auth_token'
+        build_id = 'fake_build_id'
+        arch = 'fake_arch'
+        packages = ['fake_package1', 'fake_package2']
+        modules = ['fake_module1', 'fake_module2']
+
+        request_object = mock.Mock()
+        request_object.raise_for_status = lambda: True
+        request_object.json = lambda: BS_BUILD_INFO
+        with mock.patch(
+            'pungi.scripts.create_extra_repo.requests.get',
+            return_value=request_object,
+        ) as mock_request_get:
+            repos_info = CreateExtraRepo.get_repo_info_from_bs_repo(
+                auth_token=auth_token,
+                build_id=build_id,
+                arch=arch,
+                packages=packages,
+                modules=modules,
+            )
+            self.assertEqual(
+                [
+                    ExtraVariantInfo(
+                        name=f'{build_id}-fake_platform-{arch}',
+                        arch=arch,
+                        packages=packages,
+                        modules=modules,
+                        repos=[
+                            RepoInfo(
+                                path='https://build.cloudlinux.com/'
+                                     f'build_repos/{build_id}/fake_platform',
+                                folder=arch,
+                                is_remote=True,
+                            )
+                        ]
+                    )
+                ],
+                repos_info,
+            )
+            mock_request_get.assert_called_once_with(
+                url=f'https://build.cloudlinux.com/api/v1/builds/{build_id}',
+                headers={
+                    'Authorization': f'Bearer {auth_token}',
+                }
+            )
+
+    def test_02_create_extra_repo(self):
+        with mock.patch(
+            'pungi.scripts.create_extra_repo.'
+            'CreateExtraRepo._read_local_modules_yaml',
+            return_value=[],
+        ) as mock__read_local_modules_yaml, mock.patch(
+            'pungi.scripts.create_extra_repo.'
+            'CreateExtraRepo._download_rpm_to_local_repo',
+        ) as mock__download_rpm_to_local_repo, mock.patch(
+            'pungi.scripts.create_extra_repo.'
+            'CreateExtraRepo._dump_local_modules_yaml'
+        ) as mock__dump_local_modules_yaml, mock.patch(
+            'pungi.scripts.create_extra_repo.'
+            'CreateExtraRepo._create_local_extra_repo'
+        ) as mock__create_local_extra_repo:
+            cer = CreateExtraRepo(
+                variants=[TEST_VARIANT_INFO],
+                bs_auth_token='fake_auth_token',
+                local_repository_path='/path/to/local/repo',
+                clear_target_repo=False,
+            )
+            mock__read_local_modules_yaml.assert_called_once_with()
+            cer.create_extra_repo()
+            mock__download_rpm_to_local_repo.assert_called_once_with(
+                package_location='perl-App-cpanminus-1.7044-5.'
+                                 'module_el8.3.0+2027+c8990d1d.noarch.rpm',
+                repo_info=TEST_REPO_INFO,
+            )
+            mock__dump_local_modules_yaml.assert_called_once_with()
+            mock__create_local_extra_repo.assert_called_once_with()
+            self.assertEqual(
+                [TEST_MODULE_INFO],
+                cer.local_modules_data,
+            )
+
+
+if __name__ == '__main__':
+    main()

tests/test_create_packages_json.py

@@ -0,0 +1,112 @@
+# coding=utf-8
+
+import os
+from collections import defaultdict
+from unittest import TestCase, mock, main
+
+from pungi.scripts.create_packages_json import (
+    PackagesGenerator,
+    RepoInfo,
+    VariantInfo,
+)
+
+FOLDER_WITH_TEST_DATA = os.path.join(
+    os.path.dirname(
+        os.path.abspath(__file__)
+    ),
+    'data/test_create_packages_json/',
+)
+
+test_repo_info = RepoInfo(
+    path=FOLDER_WITH_TEST_DATA,
+    folder='test_repo',
+    is_remote=False,
+    is_reference=True,
+)
+
+test_repo_info_2 = RepoInfo(
+    path=FOLDER_WITH_TEST_DATA,
+    folder='test_repo_2',
+    is_remote=False,
+    is_reference=True,
+)
+variant_info_1 = VariantInfo(
+    name='TestRepo',
+    arch='x86_64',
+    repos=[test_repo_info]
+)
+variant_info_2 = VariantInfo(
+    name='TestRepo2',
+    arch='x86_64',
+    repos=[test_repo_info_2]
+)
+
+
+class TestPackagesJson(TestCase):
+    def test_01_get_remote_file_content(self):
+        """
+        Test the getting of content from a remote file
+        """
+        request_object = mock.Mock()
+        request_object.raise_for_status = lambda: True
+        request_object.content = b'TestContent'
+        with mock.patch(
+            'pungi.scripts.create_packages_json.requests.get',
+            return_value=request_object,
+        ) as mock_requests_get, mock.patch(
+            'pungi.scripts.create_packages_json.tempfile.NamedTemporaryFile',
+        ) as mock_tempfile:
+            mock_tempfile.return_value.__enter__.return_value.name = 'tmpfile'
+            packages_generator = PackagesGenerator(
+                variants=[],
+                excluded_packages=[],
+                included_packages=[],
+            )
+            file_name = packages_generator.get_remote_file_content(
+                file_url='fakeurl')
+            mock_requests_get.assert_called_once_with(url='fakeurl')
+            mock_tempfile.assert_called_once_with(delete=False)
+            mock_tempfile.return_value.__enter__().\
+                write.assert_called_once_with(b'TestContent')
+            self.assertEqual(
+                file_name,
+                'tmpfile',
+            )
+
+    def test_02_generate_additional_packages(self):
+        pg = PackagesGenerator(
+            variants=[
+                variant_info_1,
+                variant_info_2,
+            ],
+            excluded_packages=['zziplib-utils'],
+            included_packages=['vim-file*'],
+        )
+        test_packages = defaultdict(
+            lambda: defaultdict(
+                lambda: defaultdict(
+                    list,
+                )
+            )
+        )
+        test_packages['TestRepo']['x86_64']['zziplib'] = \
+            [
+                'zziplib.i686',
+                'zziplib.x86_64',
+            ]
+        test_packages['TestRepo2']['x86_64']['vim'] = \
+            [
+                'vim-X11.i686',
+                'vim-common.i686',
+                'vim-enhanced.i686',
+                'vim-filesystem.noarch',
+            ]
+        result = pg.generate_packages_json()
+        self.assertEqual(
+            test_packages,
+            result,
+        )
+
+
+if __name__ == '__main__':
+    main()

tests/test_createiso_phase.py

@@ -2,9 +2,11 @@
 
 
 import logging
-import mock
+from unittest import mock
+import contextlib
 import six
 
+import productmd
 import os
 
 from tests import helpers
@@ -552,6 +554,7 @@ class CreateisoThreadTest(helpers.PungiTestCase):
                 "release_version": "1.0",
                 "runroot_tag": "f25-build",
                 "koji_profile": "koji",
+                "koji_cache": "/tmp",
             },
         )
         cmd = {
@@ -608,7 +611,9 @@ class CreateisoThreadTest(helpers.PungiTestCase):
             iso.get_implanted_md5.call_args_list,
             [mock.call(cmd["iso_path"], logger=compose._logger)],
         )
-        self.assertEqual(iso.get_volume_id.call_args_list, [mock.call(cmd["iso_path"])])
+        self.assertEqual(
+            iso.get_volume_id.call_args_list, [mock.call(cmd["iso_path"], False)]
+        )
 
         self.assertEqual(len(compose.im.add.call_args_list), 1)
         args, _ = compose.im.add.call_args_list[0]
@@ -633,6 +638,7 @@ class CreateisoThreadTest(helpers.PungiTestCase):
                 "release_version": "1.0",
                 "runroot_tag": "f25-build",
                 "koji_profile": "koji",
+                "koji_cache": "/tmp",
                 "create_jigdo": False,
                 "runroot_weights": {"createiso": 123},
             },
@@ -690,7 +696,9 @@ class CreateisoThreadTest(helpers.PungiTestCase):
             iso.get_implanted_md5.call_args_list,
             [mock.call(cmd["iso_path"], logger=compose._logger)],
         )
-        self.assertEqual(iso.get_volume_id.call_args_list, [mock.call(cmd["iso_path"])])
+        self.assertEqual(
+            iso.get_volume_id.call_args_list, [mock.call(cmd["iso_path"], False)]
+        )
 
         self.assertEqual(len(compose.im.add.call_args_list), 2)
         for args, _ in compose.im.add.call_args_list:
@@ -717,6 +725,7 @@ class CreateisoThreadTest(helpers.PungiTestCase):
                 "buildinstall_method": "lorax",
                 "runroot_tag": "f25-build",
                 "koji_profile": "koji",
+                "koji_cache": "/tmp",
             },
         )
         cmd = {
@@ -780,7 +789,9 @@ class CreateisoThreadTest(helpers.PungiTestCase):
             iso.get_implanted_md5.call_args_list,
             [mock.call(cmd["iso_path"], logger=compose._logger)],
         )
-        self.assertEqual(iso.get_volume_id.call_args_list, [mock.call(cmd["iso_path"])])
+        self.assertEqual(
+            iso.get_volume_id.call_args_list, [mock.call(cmd["iso_path"], False)]
+        )
 
         self.assertEqual(len(compose.im.add.call_args_list), 1)
         args, _ = compose.im.add.call_args_list[0]
@@ -807,6 +818,7 @@ class CreateisoThreadTest(helpers.PungiTestCase):
                 "release_version": "1.0",
                 "runroot_tag": "f25-build",
                 "koji_profile": "koji",
+                "koji_cache": "/tmp",
             },
         )
         cmd = {
@@ -839,6 +851,7 @@ class CreateisoThreadTest(helpers.PungiTestCase):
                 "release_version": "1.0",
                 "runroot_tag": "f25-build",
                 "koji_profile": "koji",
+                "koji_cache": "/tmp",
                 "failable_deliverables": [("^.*$", {"*": "iso"})],
             },
         )
@@ -881,6 +894,7 @@ class CreateisoThreadTest(helpers.PungiTestCase):
                 "release_version": "1.0",
                 "runroot_tag": "f25-build",
                 "koji_profile": "koji",
+                "koji_cache": "/tmp",
                 "failable_deliverables": [("^.*$", {"*": "iso"})],
             },
         )
@@ -958,7 +972,9 @@ class CreateisoThreadTest(helpers.PungiTestCase):
             iso.get_implanted_md5.call_args_list,
             [mock.call(cmd["iso_path"], logger=compose._logger)],
         )
-        self.assertEqual(iso.get_volume_id.call_args_list, [mock.call(cmd["iso_path"])])
+        self.assertEqual(
+            iso.get_volume_id.call_args_list, [mock.call(cmd["iso_path"], False)]
+        )
 
         self.assertEqual(len(compose.im.add.call_args_list), 1)
         args, _ = compose.im.add.call_args_list[0]
@@ -1370,7 +1386,9 @@ class CreateisoTryReusePhaseTest(helpers.PungiTestCase):
         )
 
     def test_old_config_changed(self):
-        compose = helpers.DummyCompose(self.topdir, {"createiso_allow_reuse": True})
+        compose = helpers.DummyCompose(
+            self.topdir, {"createiso_allow_reuse": True, "sigkeys": ["abcdef"]}
+        )
         old_config = compose.conf.copy()
         old_config["release_version"] = "2"
         compose.load_old_compose_config.return_value = old_config
@@ -1383,12 +1401,30 @@ class CreateisoTryReusePhaseTest(helpers.PungiTestCase):
             phase.try_reuse(cmd, compose.variants["Server"], "x86_64", opts)
         )
 
-    def test_no_old_metadata(self):
+    @mock.patch("pungi.phases.createiso.read_json_file")
+    def test_unsigned_packages_allowed(self, read_json_file):
         compose = helpers.DummyCompose(self.topdir, {"createiso_allow_reuse": True})
         compose.load_old_compose_config.return_value = compose.conf.copy()
         phase = createiso.CreateisoPhase(compose, mock.Mock())
         phase.logger = self.logger
         cmd = {"disc_num": 1, "disc_count": 1}
+
+        opts = CreateIsoOpts(volid="new-volid")
+
+        read_json_file.return_value = {"opts": {"volid": "old-volid"}}
+
+        self.assertFalse(
+            phase.try_reuse(cmd, compose.variants["Server"], "x86_64", opts)
+        )
+
+    def test_no_old_metadata(self):
+        compose = helpers.DummyCompose(
+            self.topdir, {"createiso_allow_reuse": True, "sigkeys": ["abcdef"]}
+        )
+        compose.load_old_compose_config.return_value = compose.conf.copy()
+        phase = createiso.CreateisoPhase(compose, mock.Mock())
+        phase.logger = self.logger
+        cmd = {"disc_num": 1, "disc_count": 1}
         opts = CreateIsoOpts()
 
         self.assertFalse(
@@ -1397,7 +1433,9 @@ class CreateisoTryReusePhaseTest(helpers.PungiTestCase):
 
     @mock.patch("pungi.phases.createiso.read_json_file")
     def test_volume_id_differs(self, read_json_file):
-        compose = helpers.DummyCompose(self.topdir, {"createiso_allow_reuse": True})
+        compose = helpers.DummyCompose(
+            self.topdir, {"createiso_allow_reuse": True, "sigkeys": ["abcdef"]}
+        )
         compose.load_old_compose_config.return_value = compose.conf.copy()
         phase = createiso.CreateisoPhase(compose, mock.Mock())
         phase.logger = self.logger
@@ -1413,7 +1451,9 @@ class CreateisoTryReusePhaseTest(helpers.PungiTestCase):
 
     @mock.patch("pungi.phases.createiso.read_json_file")
     def test_packages_differ(self, read_json_file):
-        compose = helpers.DummyCompose(self.topdir, {"createiso_allow_reuse": True})
+        compose = helpers.DummyCompose(
+            self.topdir, {"createiso_allow_reuse": True, "sigkeys": ["abcdef"]}
+        )
         compose.load_old_compose_config.return_value = compose.conf.copy()
         phase = createiso.CreateisoPhase(compose, mock.Mock())
         phase.logger = self.logger
@@ -1435,7 +1475,9 @@ class CreateisoTryReusePhaseTest(helpers.PungiTestCase):
 
     @mock.patch("pungi.phases.createiso.read_json_file")
     def test_runs_perform_reuse(self, read_json_file):
-        compose = helpers.DummyCompose(self.topdir, {"createiso_allow_reuse": True})
+        compose = helpers.DummyCompose(
+            self.topdir, {"createiso_allow_reuse": True, "sigkeys": ["abcdef"]}
+        )
         compose.load_old_compose_config.return_value = compose.conf.copy()
         phase = createiso.CreateisoPhase(compose, mock.Mock())
         phase.logger = self.logger
@@ -1589,3 +1631,103 @@ class ComposeConfGetIsoLevelTest(helpers.PungiTestCase):
                 compose, compose.variants["Client"], "x86_64"
             ),
         )
+
+
+def mk_mount(topdir, images):
+    @contextlib.contextmanager
+    def dummy_mount(path, logger):
+        treeinfo = [
+            "[general]",
+            "family = Test",
+            "version = 1.0",
+            "arch = x86_64",
+            "variant = Server",
+            "[checksums]",
+        ]
+        for image in images:
+            helpers.touch(os.path.join(topdir, image.path), image.content)
+            treeinfo.append("%s = sha256:%s" % (image.path, image.checksum))
+        helpers.touch(os.path.join(topdir, ".treeinfo"), "\n".join(treeinfo))
+        yield topdir
+
+    return dummy_mount
+
+
+class _MockRun:
+    """This class replaces kobo.shortcuts.run and validates that the correct
+    two commands are called. The assertions can not be done after the tested
+    function finishes because it will clean up the .treeinfo file that needs to
+    be checked.
+    """
+
+    def __init__(self):
+        self.num_calls = 0
+        self.asserts = [self._assert_xorriso, self._assert_implantisomd5]
+
+    def __call__(self, cmd, logfile):
+        self.num_calls += 1
+        self.asserts.pop(0)(cmd)
+
+    def _assert_xorriso(self, cmd):
+        assert cmd[0] == "xorriso"
+        ti = productmd.TreeInfo()
+        input_iso = None
+        for i, arg in enumerate(cmd):
+            if arg == "-map":
+                ti.load(cmd[i + 1])
+            if arg == "-outdev":
+                self.temp_iso = cmd[i + 1]
+            if arg == "-indev":
+                input_iso = cmd[i + 1]
+        assert self.input_iso == input_iso
+        assert ti.checksums.checksums[self.image_relative_path] == self.image_checksum
+
+    def _assert_implantisomd5(self, cmd):
+        assert cmd[0] == "/usr/bin/implantisomd5"
+        assert cmd[-1] == self.temp_iso
+
+
+class DummyImage:
+    def __init__(self, path, content, checksum=None):
+        self.path = path
+        self.content = content
+        self.checksum = checksum or helpers.hash_string("sha256", content)
+
+
+@mock.patch("os.rename")
+@mock.patch("pungi.phases.createiso.run", new_callable=_MockRun)
+class FixChecksumsTest(helpers.PungiTestCase):
+    def test_checksum_matches(self, mock_run, mock_rename):
+        compose = helpers.DummyCompose(self.topdir, {})
+        arch = "x86_64"
+        iso_path = "DUMMY_ISO"
+
+        with mock.patch(
+            "pungi.wrappers.iso.mount",
+            new=mk_mount(self.topdir, [DummyImage("images/eltorito.img", "eltorito")]),
+        ):
+            createiso.fix_treeinfo_checksums(compose, iso_path, arch)
+
+        self.assertEqual(mock_run.num_calls, 0)
+        self.assertEqual(mock_rename.call_args_list, [])
+
+    def test_checksum_fix(self, mock_run, mock_rename):
+        compose = helpers.DummyCompose(self.topdir, {})
+        arch = "x86_64"
+        img = "images/eltorito.img"
+        content = "eltorito"
+        iso_path = "DUMMY_ISO"
+        mock_run.input_iso = iso_path
+        mock_run.image_relative_path = "images/eltorito.img"
+        mock_run.image_checksum = ("sha256", helpers.hash_string("sha256", content))
+
+        with mock.patch(
+            "pungi.wrappers.iso.mount",
+            new=mk_mount(self.topdir, [DummyImage(img, content, "abc")]),
+        ):
+            createiso.fix_treeinfo_checksums(compose, iso_path, arch)
+
+        # The new image was copied over the old one
+        self.assertEqual(
+            mock_rename.call_args_list, [mock.call(mock_run.temp_iso, iso_path)]
+        )

tests/test_createiso_script.py

@@ -1,6 +1,6 @@
 # -*- coding: utf-8 -*-
 
-import mock
+from unittest import mock
 from parameterized import parameterized
 
 import os
@@ -267,58 +267,6 @@ class CreateIsoScriptTest(helpers.PungiTestCase):
             ]
         )
 
-    def test_bootable_run_buildinstall(self):
-        createiso.write_script(
-            createiso.CreateIsoOpts(
-                output_dir=self.outdir,
-                iso_name="DP-1.0-20160405.t.3-ppc64.iso",
-                volid="DP-1.0-20160405.t.3",
-                graft_points="graft-list",
-                arch="ppc64",
-                buildinstall_method="buildinstall",
-            ),
-            self.out,
-        )
-
-        self.assertScript(
-            [
-                " ".join(
-                    [
-                        "/usr/bin/genisoimage",
-                        "-untranslated-filenames",
-                        "-volid",
-                        "DP-1.0-20160405.t.3",
-                        "-J",
-                        "-joliet-long",
-                        "-rational-rock",
-                        "-translation-table",
-                        "-x",
-                        "./lost+found",
-                        "-part",
-                        "-hfs",
-                        "-r",
-                        "-l",
-                        "-sysid",
-                        "PPC",
-                        "-no-desktop",
-                        "-allow-multidot",
-                        "-chrp-boot",
-                        "-map",
-                        "/usr/lib/anaconda-runtime/boot/mapping",
-                        "-hfs-bless",
-                        "/ppc/mac",
-                        "-o",
-                        "DP-1.0-20160405.t.3-ppc64.iso",
-                        "-graft-points",
-                        "-path-list",
-                        "graft-list",
-                    ]
-                ),
-                " ".join(["/usr/bin/implantisomd5", "DP-1.0-20160405.t.3-ppc64.iso"]),
-                "isoinfo -R -f -i DP-1.0-20160405.t.3-ppc64.iso | grep -v '/TRANS.TBL$' | sort >> DP-1.0-20160405.t.3-ppc64.iso.manifest",  # noqa: E501
-            ]
-        )
-
     @mock.patch("sys.stderr")
     @mock.patch("kobo.shortcuts.run")
     def test_run_with_jigdo_bad_args(self, run, stderr):

tests/test_createrepophase.py

@@ -8,7 +8,7 @@ except ImportError:
 import glob
 import os
 
-import mock
+from unittest import mock
 import six
 
 from pungi.module_util import Modulemd

tests/test_extra_files_phase.py

@@ -1,6 +1,6 @@
 # -*- coding: utf-8 -*-
 
-import mock
+from unittest import mock
 import os
 
 from productmd.extra_files import ExtraFiles

tests/test_extra_isos_phase.py

@@ -1,8 +1,8 @@
 # -*- coding: utf-8 -*-
-
-import logging
-import mock
+from typing import AnyStr, List
+from unittest import mock
 import six
+import logging
 
 import os
 
@@ -130,6 +130,7 @@ class ExtraIsosThreadTest(helpers.PungiTestCase):
                 ),
             ],
         )
+        iso_path = os.path.join(self.topdir, "compose/Server/x86_64/iso/my.iso")
         self.assertEqual(
             rcc.call_args_list,
             [
@@ -148,6 +149,7 @@ class ExtraIsosThreadTest(helpers.PungiTestCase):
                     log_file=os.path.join(
                         self.topdir, "logs/x86_64/extraiso-my.iso.x86_64.log"
                     ),
+                    iso_path=iso_path,
                 )
             ],
         )
@@ -158,7 +160,7 @@ class ExtraIsosThreadTest(helpers.PungiTestCase):
                     compose,
                     server,
                     "x86_64",
-                    os.path.join(self.topdir, "compose/Server/x86_64/iso/my.iso"),
+                    iso_path,
                     True,
                     additional_variants=["Client"],
                 )
@@ -205,6 +207,7 @@ class ExtraIsosThreadTest(helpers.PungiTestCase):
                 ),
             ],
         )
+        iso_path = os.path.join(self.topdir, "compose/Server/x86_64/iso/my.iso")
         self.assertEqual(
             rcc.call_args_list,
             [
@@ -223,6 +226,7 @@ class ExtraIsosThreadTest(helpers.PungiTestCase):
                     log_file=os.path.join(
                         self.topdir, "logs/x86_64/extraiso-my.iso.x86_64.log"
                     ),
+                    iso_path=iso_path,
                 )
             ],
         )
@@ -233,7 +237,7 @@ class ExtraIsosThreadTest(helpers.PungiTestCase):
                     compose,
                     server,
                     "x86_64",
-                    os.path.join(self.topdir, "compose/Server/x86_64/iso/my.iso"),
+                    iso_path,
                     True,
                     additional_variants=["Client"],
                 )
@@ -278,6 +282,7 @@ class ExtraIsosThreadTest(helpers.PungiTestCase):
                 ),
             ],
         )
+        iso_path = os.path.join(self.topdir, "compose/Server/x86_64/iso/my.iso")
         self.assertEqual(
             rcc.call_args_list,
             [
@@ -296,6 +301,7 @@ class ExtraIsosThreadTest(helpers.PungiTestCase):
                     log_file=os.path.join(
                         self.topdir, "logs/x86_64/extraiso-my.iso.x86_64.log"
                     ),
+                    iso_path=iso_path,
                 )
             ],
         )
@@ -306,7 +312,7 @@ class ExtraIsosThreadTest(helpers.PungiTestCase):
                     compose,
                     server,
                     "x86_64",
-                    os.path.join(self.topdir, "compose/Server/x86_64/iso/my.iso"),
+                    iso_path,
                     True,
                     additional_variants=["Client"],
                 )
@@ -353,6 +359,7 @@ class ExtraIsosThreadTest(helpers.PungiTestCase):
                 ),
             ],
         )
+        iso_path = os.path.join(self.topdir, "compose/Server/x86_64/iso/my.iso")
         self.assertEqual(
             rcc.call_args_list,
             [
@@ -371,6 +378,7 @@ class ExtraIsosThreadTest(helpers.PungiTestCase):
                     log_file=os.path.join(
                         self.topdir, "logs/x86_64/extraiso-my.iso.x86_64.log"
                     ),
+                    iso_path=iso_path,
                 )
             ],
         )
@@ -381,7 +389,7 @@ class ExtraIsosThreadTest(helpers.PungiTestCase):
                     compose,
                     server,
                     "x86_64",
-                    os.path.join(self.topdir, "compose/Server/x86_64/iso/my.iso"),
+                    iso_path,
                     False,
                     additional_variants=["Client"],
                 )
@@ -423,6 +431,7 @@ class ExtraIsosThreadTest(helpers.PungiTestCase):
                 ),
             ],
         )
+        iso_path = os.path.join(self.topdir, "compose/Server/source/iso/my.iso")
         self.assertEqual(
             rcc.call_args_list,
             [
@@ -441,6 +450,7 @@ class ExtraIsosThreadTest(helpers.PungiTestCase):
                     log_file=os.path.join(
                         self.topdir, "logs/src/extraiso-my.iso.src.log"
                     ),
+                    iso_path=iso_path,
                 )
             ],
         )
@@ -451,7 +461,7 @@ class ExtraIsosThreadTest(helpers.PungiTestCase):
                     compose,
                     server,
                     "src",
-                    os.path.join(self.topdir, "compose/Server/source/iso/my.iso"),
+                    iso_path,
                     False,
                     additional_variants=["Client"],
                 )
@@ -614,6 +624,7 @@ class GetExtraFilesTest(helpers.PungiTestCase):
         )
 
 
+@mock.patch("pungi.phases.extra_isos.tweak_repo_treeinfo")
 @mock.patch("pungi.phases.extra_isos.tweak_treeinfo")
 @mock.patch("pungi.wrappers.iso.write_graft_points")
 @mock.patch("pungi.wrappers.iso.get_graft_points")
@@ -623,7 +634,7 @@ class GetIsoContentsTest(helpers.PungiTestCase):
         self.compose = helpers.DummyCompose(self.topdir, {})
         self.variant = self.compose.variants["Server"]
 
-    def test_non_bootable_binary(self, ggp, wgp, tt):
+    def test_non_bootable_binary(self, ggp, wgp, tt, trt):
         gp = {
             "compose/Client/x86_64/os/Packages": {"f/foo.rpm": "/mnt/f/foo.rpm"},
             "compose/Client/x86_64/os/repodata": {
@@ -693,7 +704,15 @@ class GetIsoContentsTest(helpers.PungiTestCase):
             ],
         )
 
-    def test_inherit_extra_files(self, ggp, wgp, tt):
+        # Check correct call to tweak_repo_treeinfo
+        self._tweak_repo_treeinfo_call_list_checker(
+            trt_mock=trt,
+            main_variant='Server',
+            addon_variants=['Client'],
+            sub_path='x86_64/os',
+        )
+
+    def test_inherit_extra_files(self, ggp, wgp, tt, trt):
         gp = {
             "compose/Client/x86_64/os/Packages": {"f/foo.rpm": "/mnt/f/foo.rpm"},
             "compose/Client/x86_64/os/repodata": {
@@ -767,7 +786,15 @@ class GetIsoContentsTest(helpers.PungiTestCase):
             ],
         )
 
-    def test_source(self, ggp, wgp, tt):
+        # Check correct call to tweak_repo_treeinfo
+        self._tweak_repo_treeinfo_call_list_checker(
+            trt_mock=trt,
+            main_variant='Server',
+            addon_variants=['Client'],
+            sub_path='x86_64/os',
+        )
+
+    def test_source(self, ggp, wgp, tt, trt):
         gp = {
             "compose/Client/source/tree/Packages": {"f/foo.rpm": "/mnt/f/foo.rpm"},
             "compose/Client/source/tree/repodata": {
@@ -837,7 +864,15 @@ class GetIsoContentsTest(helpers.PungiTestCase):
             ],
         )
 
-    def test_bootable(self, ggp, wgp, tt):
+        # Check correct call to tweak_repo_treeinfo
+        self._tweak_repo_treeinfo_call_list_checker(
+            trt_mock=trt,
+            main_variant='Server',
+            addon_variants=['Client'],
+            sub_path='source/tree',
+        )
+
+    def test_bootable(self, ggp, wgp, tt, trt):
         self.compose.conf["buildinstall_method"] = "lorax"
 
         bi_dir = os.path.join(self.topdir, "work/x86_64/buildinstall/Server")
@@ -863,10 +898,8 @@ class GetIsoContentsTest(helpers.PungiTestCase):
             "images/efiboot.img": os.path.join(iso_dir, "images/efiboot.img"),
         }
 
-        ggp.side_effect = (
-            lambda compose, x: gp[x[0][len(self.topdir) + 1 :]]
-            if len(x) == 1
-            else bi_gp
+        ggp.side_effect = lambda compose, x: (
+            gp[x[0][len(self.topdir) + 1 :]] if len(x) == 1 else bi_gp
         )
         gp_file = os.path.join(self.topdir, "work/x86_64/iso/my.iso-graft-points")
 
@@ -939,6 +972,42 @@ class GetIsoContentsTest(helpers.PungiTestCase):
             ],
         )
 
+        # Check correct call to tweak_repo_treeinfo
+        self._tweak_repo_treeinfo_call_list_checker(
+            trt_mock=trt,
+            main_variant='Server',
+            addon_variants=['Client'],
+            sub_path='x86_64/os',
+        )
+
+    def _tweak_repo_treeinfo_call_list_checker(
+            self,
+            trt_mock: mock.Mock,
+            main_variant: AnyStr,
+            addon_variants: List[AnyStr],
+            sub_path: AnyStr) -> None:
+        """
+        Check correct call to tweak_repo_treeinfo
+        """
+        path_to_treeinfo = os.path.join(
+            self.topdir,
+            'compose',
+            main_variant,
+            sub_path,
+            '.treeinfo',
+        )
+        self.assertEqual(
+            trt_mock.call_args_list,
+            [
+                mock.call(
+                    self.compose,
+                    addon_variants,
+                    path_to_treeinfo,
+                    path_to_treeinfo,
+                )
+            ]
+        )
+
 
 class GetFilenameTest(helpers.PungiTestCase):
     def test_use_original_name(self):
@@ -1016,6 +1085,15 @@ class TweakTreeinfoTest(helpers.PungiTestCase):
 
         self.assertFilesEqual(output, expected)
 
+    def test_repo_tweak(self):
+        compose = helpers.DummyCompose(self.topdir, {})
+        input = os.path.join(helpers.FIXTURE_DIR, "extraiso.treeinfo")
+        output = os.path.join(self.topdir, "actual-treeinfo")
+        expected = os.path.join(helpers.FIXTURE_DIR, "extraiso-tweaked-expected.treeinfo")
+        extra_isos.tweak_repo_treeinfo(compose, ["Client"], input, output)
+
+        self.assertFilesEqual(output, expected)
+
 
 class PrepareMetadataTest(helpers.PungiTestCase):
     @mock.patch("pungi.metadata.create_media_repo")
@@ -1078,7 +1156,9 @@ class ExtraisoTryReusePhaseTest(helpers.PungiTestCase):
         )
 
     def test_buildinstall_changed(self):
-        compose = helpers.DummyCompose(self.topdir, {"extraiso_allow_reuse": True})
+        compose = helpers.DummyCompose(
+            self.topdir, {"extraiso_allow_reuse": True, "sigkeys": ["abcdef"]}
+        )
         thread = extra_isos.ExtraIsosThread(compose, mock.Mock())
         thread.logger = self.logger
         thread.bi = mock.Mock()
@@ -1092,7 +1172,9 @@ class ExtraisoTryReusePhaseTest(helpers.PungiTestCase):
         )
 
     def test_no_old_config(self):
-        compose = helpers.DummyCompose(self.topdir, {"extraiso_allow_reuse": True})
+        compose = helpers.DummyCompose(
+            self.topdir, {"extraiso_allow_reuse": True, "sigkeys": ["abcdef"]}
+        )
         thread = extra_isos.ExtraIsosThread(compose, mock.Mock())
         thread.logger = self.logger
         opts = CreateIsoOpts()
@@ -1104,7 +1186,9 @@ class ExtraisoTryReusePhaseTest(helpers.PungiTestCase):
         )
 
     def test_old_config_changed(self):
-        compose = helpers.DummyCompose(self.topdir, {"extraiso_allow_reuse": True})
+        compose = helpers.DummyCompose(
+            self.topdir, {"extraiso_allow_reuse": True, "sigkeys": ["abcdef"]}
+        )
         old_config = compose.conf.copy()
         old_config["release_version"] = "2"
         compose.load_old_compose_config.return_value = old_config
@@ -1119,7 +1203,9 @@ class ExtraisoTryReusePhaseTest(helpers.PungiTestCase):
         )
 
     def test_no_old_metadata(self):
-        compose = helpers.DummyCompose(self.topdir, {"extraiso_allow_reuse": True})
+        compose = helpers.DummyCompose(
+            self.topdir, {"extraiso_allow_reuse": True, "sigkeys": ["abcdef"]}
+        )
         compose.load_old_compose_config.return_value = compose.conf.copy()
         thread = extra_isos.ExtraIsosThread(compose, mock.Mock())
         thread.logger = self.logger
@@ -1133,7 +1219,9 @@ class ExtraisoTryReusePhaseTest(helpers.PungiTestCase):
 
     @mock.patch("pungi.phases.extra_isos.read_json_file")
     def test_volume_id_differs(self, read_json_file):
-        compose = helpers.DummyCompose(self.topdir, {"extraiso_allow_reuse": True})
+        compose = helpers.DummyCompose(
+            self.topdir, {"extraiso_allow_reuse": True, "sigkeys": ["abcdef"]}
+        )
         compose.load_old_compose_config.return_value = compose.conf.copy()
         thread = extra_isos.ExtraIsosThread(compose, mock.Mock())
         thread.logger = self.logger
@@ -1150,7 +1238,9 @@ class ExtraisoTryReusePhaseTest(helpers.PungiTestCase):
 
     @mock.patch("pungi.phases.extra_isos.read_json_file")
     def test_packages_differ(self, read_json_file):
-        compose = helpers.DummyCompose(self.topdir, {"extraiso_allow_reuse": True})
+        compose = helpers.DummyCompose(
+            self.topdir, {"extraiso_allow_reuse": True, "sigkeys": ["abcdef"]}
+        )
         compose.load_old_compose_config.return_value = compose.conf.copy()
         thread = extra_isos.ExtraIsosThread(compose, mock.Mock())
         thread.logger = self.logger
@@ -1172,7 +1262,7 @@ class ExtraisoTryReusePhaseTest(helpers.PungiTestCase):
         )
 
     @mock.patch("pungi.phases.extra_isos.read_json_file")
-    def test_runs_perform_reuse(self, read_json_file):
+    def test_unsigned_packages(self, read_json_file):
         compose = helpers.DummyCompose(self.topdir, {"extraiso_allow_reuse": True})
         compose.load_old_compose_config.return_value = compose.conf.copy()
         thread = extra_isos.ExtraIsosThread(compose, mock.Mock())
@@ -1195,6 +1285,38 @@ class ExtraisoTryReusePhaseTest(helpers.PungiTestCase):
             },
         }
 
+        self.assertFalse(
+            thread.try_reuse(
+                compose, compose.variants["Server"], "x86_64", "abcdef", opts
+            )
+        )
+
+    @mock.patch("pungi.phases.extra_isos.read_json_file")
+    def test_runs_perform_reuse(self, read_json_file):
+        compose = helpers.DummyCompose(
+            self.topdir, {"extraiso_allow_reuse": True, "sigkeys": ["abcdef"]}
+        )
+        compose.load_old_compose_config.return_value = compose.conf.copy()
+        thread = extra_isos.ExtraIsosThread(compose, mock.Mock())
+        thread.logger = self.logger
+        thread.perform_reuse = mock.Mock()
+
+        new_graft_points = os.path.join(self.topdir, "new_graft_points")
+        helpers.touch(new_graft_points)
+        opts = CreateIsoOpts(graft_points=new_graft_points, volid="volid")
+
+        old_graft_points = os.path.join(self.topdir, "old_graft_points")
+        helpers.touch(old_graft_points)
+        dummy_iso_path = "dummy-iso-path/dummy.iso"
+        read_json_file.return_value = {
+            "opts": {
+                "graft_points": old_graft_points,
+                "volid": "volid",
+                "output_dir": os.path.dirname(dummy_iso_path),
+                "iso_name": os.path.basename(dummy_iso_path),
+            },
+        }
+
         self.assertTrue(
             thread.try_reuse(
                 compose, compose.variants["Server"], "x86_64", "abcdef", opts