- Changelog date order

- Typo

.gitignore

@@ -0,0 +1,19 @@
+*.py[co]
+*~
+*.egg-info
+MANIFEST
+build/*
+dist/*
+doc/_build
+noarch/*
+tests/data/repo
+tests/data/repo-krb5-lookaside
+tests/_composes
+htmlcov/
+.coverage
+.eggs
+.idea/
+.tox
+.venv
+.kdev4/
+pungi.kdev4

.hgignore

@@ -1,4 +0,0 @@
-syntax: glob
-*.pyc
-*~
-build/*

.hgtags

@@ -1,28 +0,0 @@
-e80b96291cfe23c4c21b2e668d8d80a8998c7cfc pungi--
-f755487fdd539c3a68296c0dc7b6c6dc49dccb98 pungi-0.1.0-1%{?dist}
-d9bda840074f8f5e7b8844007e9951cd55ad9c1d pungi--
-baa55b9774642535467104c3f6b268671cc35e08 pungi-0.1.0-1
-902402e675943d6c3186b924a9cff89d539b06f9 pungi-0.1.0-1
-14a5e625d91034b7dcb1f2b26486827929e87e24 pungi-0.1.0-1
-b13071d9363851d2766e9efaf80e9e13feec7a0c pungi-0.1.0-1
-00326e01cc7dd77f527d1a70e97fa907f35ce669 pungi-0.1.0-1
-591cf30beec90deb8b01aaef07e042d8878f4f09 pungi-0.1.1-1
-9f954716abd9c8db453b9f1b56f64e0defd8fa1d FC-6
-f0cbd4fbc9e7915fa94588237827e0a1379ec823 pungi--
-c5e81c8e1adc642b15e5aac713ae2e58a386c9b9 pungi-0.2.0-1
-f90b645121cb2f794ceda3c4be050c53d36a7bec pungi-0.2.0-1
-ebfe0e963db6d7b652f63f99aaec121e7ff19074 pungi-0.2.1-1
-769a8e08d77a2234295449282b16ca38ff4d846e pungi-0.2.2-1
-ba049c3454d5dae9326d8050bb0d7d8116982ca4 f7-test1
-780520383876b76dd06fa013e1a41ddd6bf0901e pungi-0.2.3-1
-158bd9a170892b43645caed12bddc3602ef3be4d pungi-0.2.3-1
-6659955ccfdf29ecd6027bd3770f80d815720af0 pungi-0.2.3-1
-9f7b5877c32c533d00ea6baa67057ce424a40a61 pungi-0.2.3-1
-7ea08753383766ce36bb49fef6d4fcf44158ad26 pungi-0.2.3-1
-65596b024b8380bd72c6faec00d37820ada1444d pungi-0.2.4-1
-5e3332cfa2bb723f438507313836c299fcc99cff pungi-0.2.5-1
-61146ab008d70cb4ce294d14a8465c05613e91e5 pungi-0.2.6-1
-6de1d8a07c7b75fc069c72eaa9b3cb4ecaa5ad5a pungi-0.2.7-1
-c150a9d7a125e6c25384fbbf8080d7532191b587 f7-test2
-9c5cdf9e045ab0c804d85a50b24107b108aa2da5 pungi-0.2.8-1
-f1ee949b238b004ee53c6b30915e69352274f583 pungi-0.3.0-1

1715.patch

@@ -0,0 +1,41 @@
+From 432b0bce0401c4bbcd1a958a89305c475a794f26 Mon Sep 17 00:00:00 2001
+From: Adam Williamson <awilliam@redhat.com>
+Date: Jan 19 2024 07:25:09 +0000
+Subject: checks: don't require "repo" in the "ostree" schema
+
+
+Per @siosm in https://pagure.io/pungi-fedora/pull-request/1227
+this option "is deprecated and not needed anymore", so Pungi
+should not be requiring it.
+
+Merges: https://pagure.io/pungi/pull-request/1714
+Signed-off-by: Adam Williamson <awilliam@redhat.com>
+
+---
+
+diff --git a/pungi/checks.py b/pungi/checks.py
+index a340f93..db8b297 100644
+--- a/pungi/checks.py
++++ b/pungi/checks.py
+@@ -1066,7 +1066,6 @@ def make_schema():
+                                     "required": [
+                                         "treefile",
+                                         "config_url",
+-                                        "repo",
+                                         "ostree_repo",
+                                     ],
+                                     "additionalProperties": False,
+diff --git a/pungi/phases/ostree.py b/pungi/phases/ostree.py
+index 90578ae..2649cdb 100644
+--- a/pungi/phases/ostree.py
++++ b/pungi/phases/ostree.py
+@@ -85,7 +85,7 @@ class OSTreeThread(WorkerThread):
+         comps_repo = compose.paths.work.comps_repo(
+             "$basearch", variant=variant, create_dir=False
+         )
+-        repos = shortcuts.force_list(config["repo"]) + self.repos
++        repos = shortcuts.force_list(config.get("repo", [])) + self.repos
+         if compose.has_comps:
+             repos.append(translate_path(compose, comps_repo))
+         repos = get_repo_dicts(repos, logger=self.pool)
+

AUTHORS

@@ -1,6 +1,8 @@
 Authors:
 
 Jesse Keating <jkeating at redhat dot com>
+Dennis Gilmore <dennis at ausil dot us>
+Daniel Mach <dmach at redhat dot com>
 
 Contributors:
 
@@ -9,3 +11,6 @@ Essien Ita Essien <essien at wazobialinux dot com>
 James Bowes <jbowes at redhat dot com>
 Tom Callaway <tcallawa at redhat dot com>
 Joel Andres Granados <jgranado at redhat dot com>
+<proski at fedoraproject dot org>
+Mark McLoughlin <markmc at redhat dot com>
+Jeremy Cline <jcline at redhat dot com>

COPYING

@@ -1,5 +1,5 @@
-    Pungi - a Fedora release compose tool
+    Pungi - Distribution compose tool
-    Copyright (C) 2006 Jesse Keating
+    Copyright (C) 2006-2015  Red Hat, Inc.
 
     This program is free software; you can redistribute it and/or modify
     it under the terms of the GNU General Public License as published by
@@ -11,6 +11,4 @@
     GNU General Public License for more details.
 
     You should have received a copy of the GNU General Public License
-    along with this program; if not, write to the Free Software
+    along with this program; if not, see <https://gnu.org/licenses/>.
-    Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
-

Changelog

@@ -1,185 +0,0 @@
-* Tue May 15 2007 Jesse Keating <jkeating@redhat.com>
-- Don't quote ISO label, not running mkisofs in shell
-- Apply sparc patches (spot)
-- Fix cached downloads comparing correctly
-- Shorten 'development' to 'devel' in default config, more space for mkisofs
-- Handle config file missing better (jgranado)
-
-* Fri Apr 06 2007 Jesse Keating <jkeating@redhat.com>
-- Fix comments in config file
-
-* Mon Apr 02 2007 Jesse Keating <jkeating@redhat.com>
-- Remove incompatible fc6 config files
-- Update default config file with comments / new options
-- Update comps file
-
-* Mon Mar 26 2007 Jesse Keating <jkeating@redhat.com>
-- Enable source iso building again.
-
-* Fri Mar 23 2007 Jesse Keating <jkeating@redhat.com>
-- Don't try a rescue if the script doesn't exist (prarit)
-
-* Thu Mar 22 2007 Jesse Keating <jkeating@redhat.com>
-- Pass flavor off to buildinstall if it is set (wwoods)
-
-* Fri Mar 16 2007 Jesse Keating <jkeating@redhat.com>
-- Fix a logic flaw in the depsolving loop
-
-* Thu Mar 15 2007 Jesse Keating <jkeating@redhat.com>
-- Use yum's built in exclude handling
-- Use yum's built in conditional handling for things from comps
-- Do excludes before group handling.
-- Get all potential matches for deps, let install time figure
-  the best one to use.
-- Work around false positive 'unmatched' packages (globs are fun)
-- Change how depsolving is done
-  - Get all potential matches for a dep, instead of our 'best'
-    our 'best' may not be the same as install time best.
-  - Remove anaconda code, use direct yum functions to get deps
-  - Use a True/False flag to depsolve instead of iterating over
-    a dict.
-  - Log what packages are being added for which reasons.
-
-* Tue Mar 14 2007 Jesse Keating <jkeating@redhat.com>
-- Do things faster/smarter if we've only asked for one disc
-
-* Tue Mar 13 2007 Jesse Keating <jkeating@redhat.com>
-- log the rpm2cpio stuff for release notes
-- correctly capture errors from subprocess
-
-* Wed Mar 07 2007 Jesse Keating <jkeating@redhat.com>
-- Call createrepo ourselves for the tree, not buildinstall's job
-
-* Tue Mar 06 2007 Jesse Keating <jkeating@redhat.com>
-- Convert from commands to subprocess for things we call out
-
-* Fri Mar 02 2007 Jesse Keating <jkeating@redhat.com>
-- Add kickstart %packages syntax support to package manifest
-- Make the list we hand off to yum to search for as unique as we can
-
-* Wed Feb 28 2007 Jesse Keating <jkeating@redhat.com>
-- Update Fedora 7 comps file.
-- Tag for F7 Test2
-
-* Mon Feb 26 2007 Jesse Keating <jkeating@redhat.com>
-- Fix gathering of srpms (thanks skvidal)
-
-* Wed Feb 21 2007 Jesse Keating <jkeating@redhat.com>
-- Don't use TMPDIR with buildinstall for now
-
-* Fri Feb 16 2007 Jesse Keating <jkeating@redhat.com>
-- Make use of anaconda's TMPDIR support
-- Put yum tempdirs in the workdir
-- Add a version option to cli arguments
-- Make cdsize a config option
-
-* Thu Feb 15 2007 Jesse Keating <jkeating@redhat.com>
-- Various logging tweaks
-- Use -d flag in createrepo for sqlite blobs
-- Add pydoc stuff to various functions
-- Support comments in the package manifest
-
-* Wed Feb 14 2007 Jesse Keating <jkeating@redhat.com>
-- Add logging patch from jbowes
-
-* Tue Feb 13 2007 Jesse Keating <jkeating@redhat.com>
-- Fix part of the patch from Essien
-- Add Contributors to the Authors file
-- Adjust the Makefile so that srpm doesn't cause a tag
-- Merged changes from Will Woods
-  - Write out some tree description files
-  - Don't traceback on existing files in download area (not sure this will stay)
-- Style fixed some stuff from Will
-
-* Mon Feb 12 2007 Jesse Keating <jkeating@redhat.com>
-- Add new Make targets (Essien Ita Essien)
-- Add runtime flags for doing specific stages of the compose (Essien Ita Essien)
-- Add ability to define destdir on the cli to override conf file
-- Clean up optionparse stuff, print usage if arg list is too small
-
-* Thu Feb 08 2007 Jesse Keating <jkeating@redhat.com>
-- Add support for globbing in manifest
-
-* Tue Feb 06 2007 Jesse Keating <jkeating@redhat.com>
-- yum bestPackagesFromList takes an arch argument. Fixes ppc64 bug
-- Don't use 'returnSimple' anymore, deprecated in yum api
-- Speed up depsolving a bit by tracking solved deps
-
-* Sat Feb 03 2007 Jesse Keating <jkeating@redhat.com>
-- Be able to opt-out of a bugurl since buildinstall supports this
-- Make isodir an object of pungi (wwoods)
-
-* Tue Jan 30 2007 Jesse Keating <jkeating@redhat.com>
-- implantmd5 _then_ sha1sum.
-
-* Mon Jan 29 2007 Jesse Keating <jkeating@redhat.com>
-- Update the comps file again from F7
-- Fix the ppc boot flags
-- Clean up SRPM-disc junk
-- add bugurl config option for anaconda betanag
-
-* Thu Jan 25 2007 Jesse Keating <jkeating@redhat.com>
-- Update the comps file from F7
-
-* Wed Jan 24 2007 Jesse Keating <jkeating@redhat.com>
-- Add a "flavor" option (such as Desktop)
-- Move packageorder file into workdir
-- Use some anaconda code to depsolve, gets better (and more common) results
-
-* Tue Jan 23 2007 Jesse Keating <jkeating@redhat.com>
-- Now use a manifest to determine what to pull in, not comps itself
-- Add a minimal-manifest for test composes
-- Add current F7 comps file for test composes
-- Bump the iso size to what was used in FC6
-
-* Wed Jan 17 2007 Jesse Keating <jkeating@redhat.com>
-- Move splittree workdirs into work/ at the end of the run
-
-* Tue Jan 16 2007 Jesse Keating <jkeating@redhat.com>
-- Remove our splittree for rawhide
-- Remove old main() sections from pungi.py and gather.py
-- Require yum 3.0.3 or newer
-- Add rescueCD support
-
-* Wed Dec 13 2006 Jesse Keating <jkeating@redhat.com>
-- Fix a bug in DVD repodata
-- Add correct ppc boot args
-- Set ppc arch correctly
-
-* Mon Dec 11 2006 Jesse Keating <jkeating@redhat.com>
-- Now able to get release note files from release note packages.
-- Add a config file for the source run
-
-* Sat Dec  9 2006 Jesse Keating <jkeating@redhat.com>
-- Now able to do srpms, lots of changes
-
-* Fri Nov 17 2006 Jesse Keating <jkeating@redhat.com>
-- First pass at a config file, lots of changes
-
-* Wed Nov  8 2006 Jesse Keating <jkeating@redhat.com>
-- Shuffle things around for dist-utils
-- Add setup.py and such to do installs
-
-* Tue Nov  7 2006 Jesse Keating <jkeating@redhat.com>
-- pungi.py
-    Turn on split repo creation
-    Add iso creation code
-    Add DVD creation code
-- __init.py__
-    Turn on split repo creation
-    Turn on iso creation
-
-* Mon Nov  6 2006 Jesse Keating <jkeating@redhat.com>
-- pungi.py
-    use splittree.py as a module rather than a script
-    use same layout as current fedora trees
-- gather.py
-    use same layout as current fedora trees
-
-* Wed Nov  1 2006 Jesse Keating <jkeating@redhat.com>
-- First changelog entry
-- pungi.py
-    Code up splittree, use our own for now with patches
-    Code up createSplitRepodata
-- Now able to create installable tree and create split CD dirs suitable for 
-  mkisofs

GPL

@@ -1,272 +1,339 @@
+                    GNU GENERAL PUBLIC LICENSE
+                       Version 2, June 1991
 
-      The GNU General Public License (GPL)
+ Copyright (C) 1989, 1991 Free Software Foundation, Inc.,
-      Version 2, June 1991
+ 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
+ Everyone is permitted to copy and distribute verbatim copies
+ of this license document, but changing it is not allowed.
 
-      Copyright (C) 1989, 1991 Free Software Foundation, Inc.
+                            Preamble
-      59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
-      Everyone is permitted to copy and distribute verbatim copies
-      of this license document, but changing it is not allowed.
 
-      Preamble
+  The licenses for most software are designed to take away your
+freedom to share and change it.  By contrast, the GNU General Public
+License is intended to guarantee your freedom to share and change free
+software--to make sure the software is free for all its users.  This
+General Public License applies to most of the Free Software
+Foundation's software and to any other program whose authors commit to
+using it.  (Some other Free Software Foundation software is covered by
+the GNU Lesser General Public License instead.)  You can apply it to
+your programs, too.
 
-      The licenses for most software are designed to take away your freedom to 
+  When we speak of free software, we are referring to freedom, not
-      share and change it. By contrast, the GNU General Public License is 
+price.  Our General Public Licenses are designed to make sure that you
-      intended to guarantee your freedom to share and change free software--to 
+have the freedom to distribute copies of free software (and charge for
-      make sure the software is free for all its users. This General Public 
+this service if you wish), that you receive source code or can get it
-      License applies to most of the Free Software Foundation's software and to 
+if you want it, that you can change the software or use pieces of it
-      any other program whose authors commit to using it. (Some other Free 
+in new free programs; and that you know you can do these things.
-      Software Foundation software is covered by the GNU Library General Public 
-      License instead.) You can apply it to your programs, too.
 
-      When we speak of free software, we are referring to freedom, not price. 
+  To protect your rights, we need to make restrictions that forbid
-      Our General Public Licenses are designed to make sure that you have the 
+anyone to deny you these rights or to ask you to surrender the rights.
-      freedom to distribute copies of free software (and charge for this service 
+These restrictions translate to certain responsibilities for you if you
-      if you wish), that you receive source code or can get it if you want it, 
+distribute copies of the software, or if you modify it.
-      that you can change the software or use pieces of it in new free programs; 
-      and that you know you can do these things.
 
-      To protect your rights, we need to make restrictions that forbid anyone to 
+  For example, if you distribute copies of such a program, whether
-      deny you these rights or to ask you to surrender the rights. These 
+gratis or for a fee, you must give the recipients all the rights that
-      restrictions translate to certain responsibilities for you if you 
+you have.  You must make sure that they, too, receive or can get the
-      distribute copies of the software, or if you modify it.
+source code.  And you must show them these terms so they know their
+rights.
 
-      For example, if you distribute copies of such a program, whether gratis or 
+  We protect your rights with two steps: (1) copyright the software, and
-      for a fee, you must give the recipients all the rights that you have. You 
+(2) offer you this license which gives you legal permission to copy,
-      must make sure that they, too, receive or can get the source code. And you 
+distribute and/or modify the software.
-      must show them these terms so they know their rights.
 
-      We protect your rights with two steps: (1) copyright the software, and (2) 
+  Also, for each author's protection and ours, we want to make certain
-      offer you this license which gives you legal permission to copy, 
+that everyone understands that there is no warranty for this free
-      distribute and/or modify the software.
+software.  If the software is modified by someone else and passed on, we
+want its recipients to know that what they have is not the original, so
+that any problems introduced by others will not reflect on the original
+authors' reputations.
 
-      Also, for each author's protection and ours, we want to make certain that 
+  Finally, any free program is threatened constantly by software
-      everyone understands that there is no warranty for this free software. If 
+patents.  We wish to avoid the danger that redistributors of a free
-      the software is modified by someone else and passed on, we want its 
+program will individually obtain patent licenses, in effect making the
-      recipients to know that what they have is not the original, so that any 
+program proprietary.  To prevent this, we have made it clear that any
-      problems introduced by others will not reflect on the original authors' 
+patent must be licensed for everyone's free use or not licensed at all.
-      reputations.
 
-      Finally, any free program is threatened constantly by software patents. We 
+  The precise terms and conditions for copying, distribution and
-      wish to avoid the danger that redistributors of a free program will 
+modification follow.
-      individually obtain patent licenses, in effect making the program 
-      proprietary. To prevent this, we have made it clear that any patent must 
-      be licensed for everyone's free use or not licensed at all.
 
-      The precise terms and conditions for copying, distribution and 
+                    GNU GENERAL PUBLIC LICENSE
-      modification follow.
+   TERMS AND CONDITIONS FOR COPYING, DISTRIBUTION AND MODIFICATION
 
-      TERMS AND CONDITIONS FOR COPYING, DISTRIBUTION AND MODIFICATION
+  0. This License applies to any program or other work which contains
+a notice placed by the copyright holder saying it may be distributed
+under the terms of this General Public License.  The "Program", below,
+refers to any such program or work, and a "work based on the Program"
+means either the Program or any derivative work under copyright law:
+that is to say, a work containing the Program or a portion of it,
+either verbatim or with modifications and/or translated into another
+language.  (Hereinafter, translation is included without limitation in
+the term "modification".)  Each licensee is addressed as "you".
 
-      0. This License applies to any program or other work which contains a 
+Activities other than copying, distribution and modification are not
-      notice placed by the copyright holder saying it may be distributed under 
+covered by this License; they are outside its scope.  The act of
-      the terms of this General Public License. The "Program", below, refers to 
+running the Program is not restricted, and the output from the Program
-      any such program or work, and a "work based on the Program" means either 
+is covered only if its contents constitute a work based on the
-      the Program or any derivative work under copyright law: that is to say, a 
+Program (independent of having been made by running the Program).
-      work containing the Program or a portion of it, either verbatim or with 
+Whether that is true depends on what the Program does.
-      modifications and/or translated into another language. (Hereinafter, 
-      translation is included without limitation in the term "modification".) 
-      Each licensee is addressed as "you".
 
-      Activities other than copying, distribution and modification are not 
+  1. You may copy and distribute verbatim copies of the Program's
-      covered by this License; they are outside its scope. The act of running 
+source code as you receive it, in any medium, provided that you
-      the Program is not restricted, and the output from the Program is covered 
+conspicuously and appropriately publish on each copy an appropriate
-      only if its contents constitute a work based on the Program (independent 
+copyright notice and disclaimer of warranty; keep intact all the
-      of having been made by running the Program). Whether that is true depends 
+notices that refer to this License and to the absence of any warranty;
-      on what the Program does.
+and give any other recipients of the Program a copy of this License
+along with the Program.
 
-      1. You may copy and distribute verbatim copies of the Program's source 
+You may charge a fee for the physical act of transferring a copy, and
-      code as you receive it, in any medium, provided that you conspicuously and 
+you may at your option offer warranty protection in exchange for a fee.
-      appropriately publish on each copy an appropriate copyright notice and 
-      disclaimer of warranty; keep intact all the notices that refer to this 
-      License and to the absence of any warranty; and give any other recipients 
-      of the Program a copy of this License along with the Program.
 
-      You may charge a fee for the physical act of transferring a copy, and you 
+  2. You may modify your copy or copies of the Program or any portion
-      may at your option offer warranty protection in exchange for a fee.
+of it, thus forming a work based on the Program, and copy and
+distribute such modifications or work under the terms of Section 1
+above, provided that you also meet all of these conditions:
 
-      2. You may modify your copy or copies of the Program or any portion of it, 
+    a) You must cause the modified files to carry prominent notices
-      thus forming a work based on the Program, and copy and distribute such 
+    stating that you changed the files and the date of any change.
-      modifications or work under the terms of Section 1 above, provided that 
-      you also meet all of these conditions:
 
-        a) You must cause the modified files to carry prominent notices stating 
+    b) You must cause any work that you distribute or publish, that in
-        that you changed the files and the date of any change.
+    whole or in part contains or is derived from the Program or any
+    part thereof, to be licensed as a whole at no charge to all third
+    parties under the terms of this License.
 
-        b) You must cause any work that you distribute or publish, that in whole 
+    c) If the modified program normally reads commands interactively
-        or in part contains or is derived from the Program or any part thereof, 
+    when run, you must cause it, when started running for such
-        to be licensed as a whole at no charge to all third parties under the 
+    interactive use in the most ordinary way, to print or display an
-        terms of this License.
+    announcement including an appropriate copyright notice and a
+    notice that there is no warranty (or else, saying that you provide
+    a warranty) and that users may redistribute the program under
+    these conditions, and telling the user how to view a copy of this
+    License.  (Exception: if the Program itself is interactive but
+    does not normally print such an announcement, your work based on
+    the Program is not required to print an announcement.)
 
-        c) If the modified program normally reads commands interactively when 
+These requirements apply to the modified work as a whole.  If
-        run, you must cause it, when started running for such interactive use in 
+identifiable sections of that work are not derived from the Program,
-        the most ordinary way, to print or display an announcement including an 
+and can be reasonably considered independent and separate works in
-        appropriate copyright notice and a notice that there is no warranty (or 
+themselves, then this License, and its terms, do not apply to those
-        else, saying that you provide a warranty) and that users may 
+sections when you distribute them as separate works.  But when you
-        redistribute the program under these conditions, and telling the user 
+distribute the same sections as part of a whole which is a work based
-        how to view a copy of this License. (Exception: if the Program itself is 
+on the Program, the distribution of the whole must be on the terms of
-        interactive but does not normally print such an announcement, your work 
+this License, whose permissions for other licensees extend to the
-        based on the Program is not required to print an announcement.)
+entire whole, and thus to each and every part regardless of who wrote it.
 
-      These requirements apply to the modified work as a whole. If identifiable 
+Thus, it is not the intent of this section to claim rights or contest
-      sections of that work are not derived from the Program, and can be 
+your rights to work written entirely by you; rather, the intent is to
-      reasonably considered independent and separate works in themselves, then 
+exercise the right to control the distribution of derivative or
-      this License, and its terms, do not apply to those sections when you 
+collective works based on the Program.
-      distribute them as separate works. But when you distribute the same 
-      sections as part of a whole which is a work based on the Program, the 
-      distribution of the whole must be on the terms of this License, whose 
-      permissions for other licensees extend to the entire whole, and thus to 
-      each and every part regardless of who wrote it.
 
-      Thus, it is not the intent of this section to claim rights or contest your 
+In addition, mere aggregation of another work not based on the Program
-      rights to work written entirely by you; rather, the intent is to exercise 
+with the Program (or with a work based on the Program) on a volume of
-      the right to control the distribution of derivative or collective works 
+a storage or distribution medium does not bring the other work under
-      based on the Program.
+the scope of this License.
 
-      In addition, mere aggregation of another work not based on the Program 
+  3. You may copy and distribute the Program (or a work based on it,
-      with the Program (or with a work based on the Program) on a volume of a 
+under Section 2) in object code or executable form under the terms of
-      storage or distribution medium does not bring the other work under the 
+Sections 1 and 2 above provided that you also do one of the following:
-      scope of this License.
 
-      3. You may copy and distribute the Program (or a work based on it, under 
+    a) Accompany it with the complete corresponding machine-readable
-      Section 2) in object code or executable form under the terms of Sections 1 
+    source code, which must be distributed under the terms of Sections
-      and 2 above provided that you also do one of the following:
+    1 and 2 above on a medium customarily used for software interchange; or,
 
-        a) Accompany it with the complete corresponding machine-readable source 
+    b) Accompany it with a written offer, valid for at least three
-        code, which must be distributed under the terms of Sections 1 and 2 
+    years, to give any third party, for a charge no more than your
-        above on a medium customarily used for software interchange; or,
+    cost of physically performing source distribution, a complete
+    machine-readable copy of the corresponding source code, to be
+    distributed under the terms of Sections 1 and 2 above on a medium
+    customarily used for software interchange; or,
 
-        b) Accompany it with a written offer, valid for at least three years, to 
+    c) Accompany it with the information you received as to the offer
-        give any third party, for a charge no more than your cost of physically 
+    to distribute corresponding source code.  (This alternative is
-        performing source distribution, a complete machine-readable copy of the 
+    allowed only for noncommercial distribution and only if you
-        corresponding source code, to be distributed under the terms of Sections 
+    received the program in object code or executable form with such
-        1 and 2 above on a medium customarily used for software interchange; or,
+    an offer, in accord with Subsection b above.)
 
-        c) Accompany it with the information you received as to the offer to 
+The source code for a work means the preferred form of the work for
-        distribute corresponding source code. (This alternative is allowed only 
+making modifications to it.  For an executable work, complete source
-        for noncommercial distribution and only if you received the program in 
+code means all the source code for all modules it contains, plus any
-        object code or executable form with such an offer, in accord with 
+associated interface definition files, plus the scripts used to
-        Subsection b above.)
+control compilation and installation of the executable.  However, as a
+special exception, the source code distributed need not include
+anything that is normally distributed (in either source or binary
+form) with the major components (compiler, kernel, and so on) of the
+operating system on which the executable runs, unless that component
+itself accompanies the executable.
 
-      The source code for a work means the preferred form of the work for making 
+If distribution of executable or object code is made by offering
-      modifications to it. For an executable work, complete source code means 
+access to copy from a designated place, then offering equivalent
-      all the source code for all modules it contains, plus any associated 
+access to copy the source code from the same place counts as
-      interface definition files, plus the scripts used to control compilation 
+distribution of the source code, even though third parties are not
-      and installation of the executable. However, as a special exception, the 
+compelled to copy the source along with the object code.
-      source code distributed need not include anything that is normally 
-      distributed (in either source or binary form) with the major components 
-      (compiler, kernel, and so on) of the operating system on which the 
-      executable runs, unless that component itself accompanies the executable.
 
-      If distribution of executable or object code is made by offering access to 
+  4. You may not copy, modify, sublicense, or distribute the Program
-      copy from a designated place, then offering equivalent access to copy the 
+except as expressly provided under this License.  Any attempt
-      source code from the same place counts as distribution of the source code, 
+otherwise to copy, modify, sublicense or distribute the Program is
-      even though third parties are not compelled to copy the source along with 
+void, and will automatically terminate your rights under this License.
-      the object code.
+However, parties who have received copies, or rights, from you under
+this License will not have their licenses terminated so long as such
+parties remain in full compliance.
 
-      4. You may not copy, modify, sublicense, or distribute the Program except 
+  5. You are not required to accept this License, since you have not
-      as expressly provided under this License. Any attempt otherwise to copy, 
+signed it.  However, nothing else grants you permission to modify or
-      modify, sublicense or distribute the Program is void, and will 
+distribute the Program or its derivative works.  These actions are
-      automatically terminate your rights under this License. However, parties 
+prohibited by law if you do not accept this License.  Therefore, by
-      who have received copies, or rights, from you under this License will not 
+modifying or distributing the Program (or any work based on the
-      have their licenses terminated so long as such parties remain in full 
+Program), you indicate your acceptance of this License to do so, and
-      compliance.
+all its terms and conditions for copying, distributing or modifying
+the Program or works based on it.
 
-      5. You are not required to accept this License, since you have not signed 
+  6. Each time you redistribute the Program (or any work based on the
-      it. However, nothing else grants you permission to modify or distribute 
+Program), the recipient automatically receives a license from the
-      the Program or its derivative works. These actions are prohibited by law 
+original licensor to copy, distribute or modify the Program subject to
-      if you do not accept this License. Therefore, by modifying or distributing 
+these terms and conditions.  You may not impose any further
-      the Program (or any work based on the Program), you indicate your 
+restrictions on the recipients' exercise of the rights granted herein.
-      acceptance of this License to do so, and all its terms and conditions for 
+You are not responsible for enforcing compliance by third parties to
-      copying, distributing or modifying the Program or works based on it.
+this License.
 
-      6. Each time you redistribute the Program (or any work based on the 
+  7. If, as a consequence of a court judgment or allegation of patent
-      Program), the recipient automatically receives a license from the original 
+infringement or for any other reason (not limited to patent issues),
-      licensor to copy, distribute or modify the Program subject to these terms 
+conditions are imposed on you (whether by court order, agreement or
-      and conditions. You may not impose any further restrictions on the 
+otherwise) that contradict the conditions of this License, they do not
-      recipients' exercise of the rights granted herein. You are not responsible 
+excuse you from the conditions of this License.  If you cannot
-      for enforcing compliance by third parties to this License.
+distribute so as to satisfy simultaneously your obligations under this
+License and any other pertinent obligations, then as a consequence you
+may not distribute the Program at all.  For example, if a patent
+license would not permit royalty-free redistribution of the Program by
+all those who receive copies directly or indirectly through you, then
+the only way you could satisfy both it and this License would be to
+refrain entirely from distribution of the Program.
 
-      7. If, as a consequence of a court judgment or allegation of patent 
+If any portion of this section is held invalid or unenforceable under
-      infringement or for any other reason (not limited to patent issues), 
+any particular circumstance, the balance of the section is intended to
-      conditions are imposed on you (whether by court order, agreement or 
+apply and the section as a whole is intended to apply in other
-      otherwise) that contradict the conditions of this License, they do not 
+circumstances.
-      excuse you from the conditions of this License. If you cannot distribute 
-      so as to satisfy simultaneously your obligations under this License and 
-      any other pertinent obligations, then as a consequence you may not 
-      distribute the Program at all. For example, if a patent license would not 
-      permit royalty-free redistribution of the Program by all those who receive 
-      copies directly or indirectly through you, then the only way you could 
-      satisfy both it and this License would be to refrain entirely from 
-      distribution of the Program.
 
-      If any portion of this section is held invalid or unenforceable under any 
+It is not the purpose of this section to induce you to infringe any
-      particular circumstance, the balance of the section is intended to apply 
+patents or other property right claims or to contest validity of any
-      and the section as a whole is intended to apply in other circumstances.
+such claims; this section has the sole purpose of protecting the
+integrity of the free software distribution system, which is
+implemented by public license practices.  Many people have made
+generous contributions to the wide range of software distributed
+through that system in reliance on consistent application of that
+system; it is up to the author/donor to decide if he or she is willing
+to distribute software through any other system and a licensee cannot
+impose that choice.
 
-      It is not the purpose of this section to induce you to infringe any 
+This section is intended to make thoroughly clear what is believed to
-      patents or other property right claims or to contest validity of any such 
+be a consequence of the rest of this License.
-      claims; this section has the sole purpose of protecting the integrity of 
-      the free software distribution system, which is implemented by public 
-      license practices. Many people have made generous contributions to the 
-      wide range of software distributed through that system in reliance on 
-      consistent application of that system; it is up to the author/donor to 
-      decide if he or she is willing to distribute software through any other 
-      system and a licensee cannot impose that choice.
 
-      This section is intended to make thoroughly clear what is believed to be a 
+  8. If the distribution and/or use of the Program is restricted in
-      consequence of the rest of this License.
+certain countries either by patents or by copyrighted interfaces, the
+original copyright holder who places the Program under this License
+may add an explicit geographical distribution limitation excluding
+those countries, so that distribution is permitted only in or among
+countries not thus excluded.  In such case, this License incorporates
+the limitation as if written in the body of this License.
 
-      8. If the distribution and/or use of the Program is restricted in certain 
+  9. The Free Software Foundation may publish revised and/or new versions
-      countries either by patents or by copyrighted interfaces, the original 
+of the General Public License from time to time.  Such new versions will
-      copyright holder who places the Program under this License may add an 
+be similar in spirit to the present version, but may differ in detail to
-      explicit geographical distribution limitation excluding those countries, 
+address new problems or concerns.
-      so that distribution is permitted only in or among countries not thus 
-      excluded. In such case, this License incorporates the limitation as if 
-      written in the body of this License.
 
-      9. The Free Software Foundation may publish revised and/or new versions of 
+Each version is given a distinguishing version number.  If the Program
-      the General Public License from time to time. Such new versions will be 
+specifies a version number of this License which applies to it and "any
-      similar in spirit to the present version, but may differ in detail to 
+later version", you have the option of following the terms and conditions
-      address new problems or concerns.
+either of that version or of any later version published by the Free
+Software Foundation.  If the Program does not specify a version number of
+this License, you may choose any version ever published by the Free Software
+Foundation.
 
-      Each version is given a distinguishing version number. If the Program 
+  10. If you wish to incorporate parts of the Program into other free
-      specifies a version number of this License which applies to it and "any 
+programs whose distribution conditions are different, write to the author
-      later version", you have the option of following the terms and conditions 
+to ask for permission.  For software which is copyrighted by the Free
-      either of that version or of any later version published by the Free 
+Software Foundation, write to the Free Software Foundation; we sometimes
-      Software Foundation. If the Program does not specify a version number of 
+make exceptions for this.  Our decision will be guided by the two goals
-      this License, you may choose any version ever published by the Free 
+of preserving the free status of all derivatives of our free software and
-      Software Foundation.
+of promoting the sharing and reuse of software generally.
 
-      10. If you wish to incorporate parts of the Program into other free 
+                            NO WARRANTY
-      programs whose distribution conditions are different, write to the author 
-      to ask for permission. For software which is copyrighted by the Free 
-      Software Foundation, write to the Free Software Foundation; we sometimes 
-      make exceptions for this. Our decision will be guided by the two goals of 
-      preserving the free status of all derivatives of our free software and of 
-      promoting the sharing and reuse of software generally.
 
-      NO WARRANTY
+  11. BECAUSE THE PROGRAM IS LICENSED FREE OF CHARGE, THERE IS NO WARRANTY
+FOR THE PROGRAM, TO THE EXTENT PERMITTED BY APPLICABLE LAW.  EXCEPT WHEN
+OTHERWISE STATED IN WRITING THE COPYRIGHT HOLDERS AND/OR OTHER PARTIES
+PROVIDE THE PROGRAM "AS IS" WITHOUT WARRANTY OF ANY KIND, EITHER EXPRESSED
+OR IMPLIED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF
+MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE.  THE ENTIRE RISK AS
+TO THE QUALITY AND PERFORMANCE OF THE PROGRAM IS WITH YOU.  SHOULD THE
+PROGRAM PROVE DEFECTIVE, YOU ASSUME THE COST OF ALL NECESSARY SERVICING,
+REPAIR OR CORRECTION.
 
-      11. BECAUSE THE PROGRAM IS LICENSED FREE OF CHARGE, THERE IS NO WARRANTY 
+  12. IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN WRITING
-      FOR THE PROGRAM, TO THE EXTENT PERMITTED BY APPLICABLE LAW. EXCEPT WHEN 
+WILL ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MAY MODIFY AND/OR
-      OTHERWISE STATED IN WRITING THE COPYRIGHT HOLDERS AND/OR OTHER PARTIES 
+REDISTRIBUTE THE PROGRAM AS PERMITTED ABOVE, BE LIABLE TO YOU FOR DAMAGES,
-      PROVIDE THE PROGRAM "AS IS" WITHOUT WARRANTY OF ANY KIND, EITHER EXPRESSED 
+INCLUDING ANY GENERAL, SPECIAL, INCIDENTAL OR CONSEQUENTIAL DAMAGES ARISING
-      OR IMPLIED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF 
+OUT OF THE USE OR INABILITY TO USE THE PROGRAM (INCLUDING BUT NOT LIMITED
-      MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE. THE ENTIRE RISK AS 
+TO LOSS OF DATA OR DATA BEING RENDERED INACCURATE OR LOSSES SUSTAINED BY
-      TO THE QUALITY AND PERFORMANCE OF THE PROGRAM IS WITH YOU. SHOULD THE 
+YOU OR THIRD PARTIES OR A FAILURE OF THE PROGRAM TO OPERATE WITH ANY OTHER
-      PROGRAM PROVE DEFECTIVE, YOU ASSUME THE COST OF ALL NECESSARY SERVICING, 
+PROGRAMS), EVEN IF SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE
-      REPAIR OR CORRECTION.
+POSSIBILITY OF SUCH DAMAGES.
 
-      12. IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN WRITING 
+                     END OF TERMS AND CONDITIONS
-      WILL ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MAY MODIFY AND/OR 
-      REDISTRIBUTE THE PROGRAM AS PERMITTED ABOVE, BE LIABLE TO YOU FOR DAMAGES, 
-      INCLUDING ANY GENERAL, SPECIAL, INCIDENTAL OR CONSEQUENTIAL DAMAGES 
-      ARISING OUT OF THE USE OR INABILITY TO USE THE PROGRAM (INCLUDING BUT NOT 
-      LIMITED TO LOSS OF DATA OR DATA BEING RENDERED INACCURATE OR LOSSES 
-      SUSTAINED BY YOU OR THIRD PARTIES OR A FAILURE OF THE PROGRAM TO OPERATE 
-      WITH ANY OTHER PROGRAMS), EVEN IF SUCH HOLDER OR OTHER PARTY HAS BEEN 
-      ADVISED OF THE POSSIBILITY OF SUCH DAMAGES.
 
-      END OF TERMS AND CONDITIONS
+            How to Apply These Terms to Your New Programs
 
+  If you develop a new program, and you want it to be of the greatest
+possible use to the public, the best way to achieve this is to make it
+free software which everyone can redistribute and change under these terms.
 
+  To do so, attach the following notices to the program.  It is safest
+to attach them to the start of each source file to most effectively
+convey the exclusion of warranty; and each file should have at least
+the "copyright" line and a pointer to where the full notice is found.
+
+    <one line to give the program's name and a brief idea of what it does.>
+    Copyright (C) <year>  <name of author>
+
+    This program is free software; you can redistribute it and/or modify
+    it under the terms of the GNU General Public License as published by
+    the Free Software Foundation; either version 2 of the License, or
+    (at your option) any later version.
+
+    This program is distributed in the hope that it will be useful,
+    but WITHOUT ANY WARRANTY; without even the implied warranty of
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+    GNU General Public License for more details.
+
+    You should have received a copy of the GNU General Public License along
+    with this program; if not, write to the Free Software Foundation, Inc.,
+    51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+
+Also add information on how to contact you by electronic and paper mail.
+
+If the program is interactive, make it output a short notice like this
+when it starts in an interactive mode:
+
+    Gnomovision version 69, Copyright (C) year name of author
+    Gnomovision comes with ABSOLUTELY NO WARRANTY; for details type `show w'.
+    This is free software, and you are welcome to redistribute it
+    under certain conditions; type `show c' for details.
+
+The hypothetical commands `show w' and `show c' should show the appropriate
+parts of the General Public License.  Of course, the commands you use may
+be called something other than `show w' and `show c'; they could even be
+mouse-clicks or menu items--whatever suits your program.
+
+You should also get your employer (if you work as a programmer) or your
+school, if any, to sign a "copyright disclaimer" for the program, if
+necessary.  Here is a sample; alter the names:
+
+  Yoyodyne, Inc., hereby disclaims all copyright interest in the program
+  `Gnomovision' (which makes passes at compilers) written by James Hacker.
+
+  <signature of Ty Coon>, 1 April 1989
+  Ty Coon, President of Vice
+
+This General Public License does not permit incorporating your program into
+proprietary programs.  If your program is a subroutine library, you may
+consider it more useful to permit linking proprietary applications with the
+library.  If this is what you want to do, use the GNU Lesser General
+Public License instead of this License.

MANIFEST.in

@@ -1,10 +1,16 @@
-include Authors
+include AUTHORS
-include Changelog
 include COPYING
 include GPL
-include PLAN.*
-include README
-include TESTING
-include ToDo
 include pungi.spec
-include config/*
+include setup.cfg
+include tox.ini
+include share/*
+include share/multilib/*
+include doc/*
+include doc/_static/*
+include tests/*
+include tests/data/*
+include tests/data/specs/*
+recursive-include tests/fixtures *
+global-exclude *.py[co]
+global-exclude *~ *.sw? \#*\#

Makefile

@@ -1,44 +1,113 @@
+.PHONY: all clean doc log test
+
 PKGNAME=pungi
-VERSION=$(shell rpm -q --qf "%{VERSION}\n" --specfile ${PKGNAME}.spec)
+VERSION=$(shell rpm -q --qf "%{VERSION}\n" --specfile ${PKGNAME}.spec | head -n1)
-RELEASE=$(shell rpm -q --qf "%{RELEASE}\n" --specfile ${PKGNAME}.spec)
+RELEASE=$(shell rpm -q --qf "%{RELEASE}\n" --specfile ${PKGNAME}.spec | head -n1)
-HGTAG=${PKGNAME}-$(VERSION)-$(RELEASE)
+GITTAG=${PKGNAME}-$(VERSION)
 PKGRPMFLAGS=--define "_topdir ${PWD}" --define "_specdir ${PWD}" --define "_sourcedir ${PWD}/dist" --define "_srcrpmdir ${PWD}" --define "_rpmdir ${PWD}" --define "_builddir ${PWD}"
 
 RPM="noarch/${PKGNAME}-$(VERSION)-$(RELEASE).noarch.rpm"
 SRPM="${PKGNAME}-$(VERSION)-$(RELEASE).src.rpm"
 
+PYTEST=pytest
 
-default: all
 
-all:
+all: help
-	@echo "Nothing to do"
+
+
+help:
+	@echo "Usage: make <target>"
+	@echo
+	@echo "Available targets are:"
+	@echo " help                    show this text"
+	@echo " clean                   remove python bytecode and temp files"
+	@echo " doc                     build documentation"
+	@echo " install                 install program on current system"
+	@echo " test                    run tests"
+	@echo " test-coverage           run tests and generate a coverage report"
+	@echo " test-compose            run a small teest compose (requires test data)"
+	@echo " test-data               build test data (requirement for running tests)"
+	@echo
+	@echo "Available rel-eng targets are:"
+	@echo " archive                 create source tarball"
+	@echo " log                     display changelog for spec file"
+	@echo " tag                     create a git tag according to version and release from spec file"
+	@echo " rpm                     build rpm"
+	@echo " srpm                    build srpm"
+	@echo " rpminstall              build rpm and install it"
+	@echo " release                 build srpm and create git tag"
+
 
 tag:
-	@hg tag -m "$(HGTAG)" $(HGTAG)
+	@git tag -a -m "Tag as $(GITTAG)" -f $(GITTAG)
-#	@hg push
+	@echo "Tagged as $(GITTAG)"
+
+
+Changelog:
+	(GIT_DIR=.git git log > .changelog.tmp && mv .changelog.tmp Changelog; rm -f .changelog.tmp) || (touch Changelog; echo 'git directory not found: installing possibly empty changelog.' >&2)
+
+
+log:
+	@(LC_ALL=C date +"* %a %b %e %Y `git config --get user.name` <`git config --get user.email`> - VERSION"; git log --pretty="format:- %s (%ae)" | sed -r 's/ \(([^@]+)@[^)]+\)/ (\1)/g' | cat) | less
+
 
 archive:
+	@rm -f Changelog
+	@rm -f MANIFEST
+	@make Changelog
 	@rm -rf ${PKGNAME}-$(VERSION)/
-	@python setup.py sdist > /dev/null
+	@python setup.py sdist --formats=bztar > /dev/null
-	@echo "The archive is in dist/${PKGNAME}-$(VERSION).tar.gz"
+	@echo "The archive is in dist/${PKGNAME}-$(VERSION).tar.bz2"
+
 
 srpm: archive
 	@rm -f $(SRPM)
 	@rpmbuild -bs ${PKGRPMFLAGS} ${PKGNAME}.spec
 	@echo "The srpm is in $(SRPM)"
 
+
 rpm: archive
 	@rpmbuild --clean -bb ${PKGRPMFLAGS} ${PKGNAME}.spec
 	@echo "The rpm is in $(RPM)"
 
+
 rpminstall: rpm
 	@rpm -ivh --force $(RPM)
 
+
 release: tag srpm
 
+
+install:
+	@python setup.py install
+
+
 clean:
-	@rm -f *.rpm 
+	@python setup.py clean
-	@rm -rf noarch
+	@rm -vf *.rpm
-	@rm -f *.tar.gz
+	@rm -vrf noarch
-	@rm -rf dist
+	@rm -vf *.tar.gz
-	@rm -f MANIFEST
+	@rm -vrf dist
+	@rm -vf MANIFEST
+	@rm -vf Changelog
+	@find . -\( -name "*.pyc" -o -name '*.pyo' -o -name "*~" -o -name "__pycache__" -\) -delete
+	@find . -depth -type d -a -name '*.egg-info' -exec rm -rf {} \;
+
+
+test:
+	$(PYTEST) $(PYTEST_OPTS)
+
+test-coverage:
+	$(PYTEST) --cov=pungi --cov-report term --cov-report html --cov-config tox.ini $(PYTEST_OPTS)
+
+test-data:
+	./tests/data/specs/build.sh
+
+test-compose:
+	cd tests && ./test_compose.sh
+
+test-multi-compose:
+	PYTHONPATH=$$(pwd) PATH=$$(pwd)/bin:$$PATH pungi-orchestrate --debug start tests/data/multi-compose.conf
+
+doc:
+	cd doc; make html

README

@@ -1,9 +0,0 @@
-Pungi
-An anaconda based installation spin tool
-
-This project is aimed at making a public / free tool to spin installation
-trees/isos of Fedora.  It will be written in python (for many obvious
-reasons).  Code style I hope will be of a simple "master" process that can
-call any number of subprocesses depending on a configuration set.
-
-See http://hosted.fedoraproject.org/projects/pungi for more information.

README.md

@@ -0,0 +1,39 @@
+# Pungi
+
+*Pungi* is a distribution compose tool.
+
+Composes are release snapshots that contain release deliverables such as:
+
+- installation trees
+    - RPMs
+    - repodata
+    - comps
+- (bootable) ISOs
+- kickstart trees
+    - anaconda images
+    - images for PXE boot
+
+
+## Tool overview
+
+*Pungi* consists of multiple separate executables backed by a common library.
+
+The main entry-point is the `pungi-koji` script. It loads the compose
+configuration and kicks off the process. Composing itself is done in phases.
+Each phase is responsible for generating some artifacts on disk and updating
+the `compose` object that is threaded through all the phases.
+
+*Pungi* itself does not actually do that much. Most of the actual work is
+delegated to separate executables. *Pungi* just makes sure that all the
+commands are invoked in the appropriate order and with correct arguments. It
+also moves the artifacts to correct locations.
+
+
+## Links
+
+- Documentation: https://docs.pagure.org/pungi/
+- Upstream GIT: https://pagure.io/pungi/
+- Issue tracker: https://pagure.io/pungi/issues
+- Questions can be asked in the *#fedora-releng* IRC channel on irc.libera.chat
+  or in the matrix room
+  [`#releng:fedoraproject.org`](https://matrix.to/#/#releng:fedoraproject.org)

TODO

@@ -0,0 +1,74 @@
+Random thoughts on what needs to be done before Pungi 4.0 is completed.
+
+
+Define building blocks and their metadata
+=========================================
+
+* rpms in yum repos
+* comps
+* kickstart trees
+* isos
+* kickstart trees
+* bootable images
+* readme files
+* license(s)
+
+
+Compose structure
+=================
+* topdir
+  * work, logs, etc.
+  * compose
+    * $variant
+      * $arch
+        * $content_type (rpms, isos, kickstart trees, etc.)
+          * actual content
+
+
+Split Pungi into smaller well-defined tools
+===========================================
+
+* process initial packages
+  * comps
+  * json mapping
+  * ???
+
+* grab initial package set
+  * yum repos
+  * koji instance (basically what mash does today)
+
+* resolve deps (gather)
+  * self-hosting
+  * fulltree
+  * multilib
+  * langpacks
+
+* create repos
+
+* create install images
+  * lorax
+
+* create isos
+  * isos
+    * bootable
+    * hybrid
+    * implant md5sum
+  * jigdo
+  * checksums
+
+* run tests
+  * just quick sanity tests
+
+* notification
+  * email
+  * messagebus
+
+
+Unsorted
+========
+* run any tasks in koji or local host
+* support for non-rpm content? (java artifacts, etc.)
+* docs!
+* unit tests!
+* use productmd for metadata: https://github.com/release-engineering/productmd/
+* use next-gen tools: createrepo_c, mergerepo_c, dnf, hawkey, libcomps

ToDo

@@ -1,5 +0,0 @@
-Drop release notes files in the tree
-Make pungi use the Fedora layout, composedir/topdir/os/<arch>/Fedora, composedir/topdir/<arch>/iso/, composedir/topdir/<arch>/debug/
-  Get debuginfo packages?
-Put working items in composedir/work
-Create a logging system, log to composedir/logs/

config/minimal-manifest

@@ -1,12 +0,0 @@
-kernel
-xorg-x11-fonts-ISO8859-1-75dpi
-busybox-anaconda
-dejavu-lgc-fonts
-xorg-x11-fonts-base
-memtest86+
-xorg-x11-drivers
-selinux-policy-targeted
-anaconda-runtime
-man
-joe
-grub

config/pungi.conf

@@ -1,21 +0,0 @@
-# Pungi config file
-#
-# # or ; can be used at the start of a line, ; only to comment inline.
-
-[default]
-product_name = Fedora ; The name used during install
-product_path = Fedora ; The directory where RPMS go
-iso_basename = F ; The first part of the iso file name
-bugurl = http://bugzilla.redhat.com ; Used for betanag
-comps = /etc/pungi/comps-fc7.xml ; Used to define package groupings and default installs
-manifest = /etc/pungi/minimal-manifest ; Used to determine what to bring in.  Supports Kickstart syntax
-yumconf = /etc/pungi/yum.conf.x86_64 ; Used to determine where to gather packages from
-destdir = /srv/pungi/Fedora ; Top level compose directory, must be clean
-cachedir = /srv/pungi/cache ; Cache used for repeat runs
-arch = x86_64 ; What arch to compose (must be same arch as system)
-version = devel ; Used both in install and part of the dest tree
-flavor = Custom ; Further define a given cut of the package set
-discs = 1 ; Number of discs needed to fit data.
-#cdsize = 4608.0 ; Not used if disc count is 1
-getsource = Yes ; Used to determine if we want source packages or not
-

config/yum.conf.i386

@@ -1,30 +0,0 @@
-[main]
-#keepcache=0
-#debuglevel=2
-pkgpolicy=newest
-distroverpkg=redhat-release
-tolerant=1
-exactarch=1
-obsoletes=1
-gpgcheck=1
-reposdir=./
-#plugins=1
-metadata_expire=1800
-#exclude=\*.i?86
-
-# PUT YOUR REPOS HERE OR IN separate files named file.repo
-# in /etc/yum.repos.d
-[development]
-name=Fedora Core - Development
-#baseurl=http://download.fedora.redhat.com/pub/fedora/linux/core/development/$basearch/os/
-mirrorlist=http://mirrors.fedoraproject.org/mirrorlist?repo=rawhide&arch=i386
-enabled=1
-gpgcheck=0
-
-[development-source]
-name=Fedora Core - Development Source
-#baseurl=http://download.fedora.redhat.com/pub/fedora/linux/core/development/source/SRPMS
-mirrorlist=http://mirrors.fedoraproject.org/mirrorlist?repo=rawhide-source&arch=i386
-enabled=1
-gpgcheck=0
-

config/yum.conf.x86_64

@@ -1,30 +0,0 @@
-[main]
-#keepcache=0
-#debuglevel=2
-pkgpolicy=newest
-distroverpkg=redhat-release
-tolerant=1
-exactarch=1
-obsoletes=1
-gpgcheck=1
-reposdir=./
-#plugins=1
-metadata_expire=1800
-#exclude=\*.i?86
-
-# PUT YOUR REPOS HERE OR IN separate files named file.repo
-# in /etc/yum.repos.d
-[development]
-name=Fedora Core - Development
-#baseurl=http://download.fedora.redhat.com/pub/fedora/linux/core/development/x86_64/os/
-mirrorlist=http://mirrors.fedoraproject.org/mirrorlist?repo=rawhide&arch=x86_64
-enabled=1
-gpgcheck=0
-
-[development-source]
-name=Fedora Core - Development Source
-#baseurl=http://download.fedora.redhat.com/pub/fedora/linux/core/development/x86_64/os/
-mirrorlist=http://mirrors.fedoraproject.org/mirrorlist?repo=rawhide-source&arch=x86_64
-enabled=1
-gpgcheck=0
-

contrib/tmpfiles.d/pungi-clean-cache.conf

@@ -0,0 +1,2 @@
+# Clean up pungi cache
+d /var/cache/pungi/createrepo_c/ - - - 30d

contrib/yum-dnf-compare/README

@@ -0,0 +1,19 @@
+This directory contains scripts to compare YUM and DNF based gathering code in
+Pungi.
+
+There are two scripts to help re-run the depsolving on existing code. As input
+they need .conf and .log file from an existing compose. They collect correct
+command line options from them and run the respective tool.
+
+Run:
+
+    $ run-dnf.sh Server.x86_64.conf
+    $ run-yum.sh Server.x86_64.conf
+
+The results are stored in a file with .log.dnf or .log.yum extensions. When
+--interactive is used as second argument of the scripts, the output is printed
+to terminal (useful for running in debugger).
+
+To compare the RPM package lists, run:
+
+    $ ./pungi-compare-depsolving Server.x86_64.log.yum Server.x86_64.log.dnf

contrib/yum-dnf-compare/pungi-compare-depsolving

@@ -0,0 +1,63 @@
+#!/usr/bin/python
+
+from __future__ import print_function
+
+
+import argparse
+import os
+import sys
+
+here = sys.path[0]
+if here != '/usr/bin':
+    # Git checkout
+    sys.path.insert(0, os.path.abspath(os.path.join(os.path.dirname(__file__), "..", "..")))
+
+from kobo.rpmlib import parse_nvra, make_nvra
+from pungi.wrappers.pungi import PungiWrapper
+
+
+def read_rpms(fn):
+    pw = PungiWrapper()
+    with open(fn, "r") as f:
+        data, _, _ = pw.parse_log(f)
+    result = set()
+    for i in data["rpm"]:
+        nvra = parse_nvra(i["path"])
+        result.add(make_nvra(nvra, add_rpm=True))
+
+    return result
+
+
+parser = argparse.ArgumentParser()
+parser.add_argument('old', metavar='OLD', default='pungi-yum.log')
+parser.add_argument('new', metavar='NEW', default='pungi-dnf.log')
+
+args = parser.parse_args()
+
+yum_rpms = read_rpms(args.old)
+dnf_rpms = read_rpms(args.new)
+
+removed = yum_rpms - dnf_rpms
+added = dnf_rpms - yum_rpms
+
+
+print("ADDED: %s" % len(added))
+for i in sorted(added):
+    print("    %s" % i)
+
+print()
+
+print("REMOVED: %s" % len(removed))
+for i in sorted(removed):
+    print("    %s" % i)
+
+print()
+
+print("ADDED:    %6s" % len(added))
+print("REMOVED:  %6s" % len(removed))
+print("YUM RPMS: %6s" % len(yum_rpms))
+print("DNF RPMS: %6s" % len(dnf_rpms))
+print("ALL RPMS: %6s" % len(yum_rpms | dnf_rpms))
+
+if added or removed:
+    sys.exit(1)

contrib/yum-dnf-compare/run-dnf.sh

@@ -0,0 +1,24 @@
+#!/bin/bash
+
+set -e
+set -u
+set -o pipefail
+
+HERE=$(dirname "$0")
+PATH=$HERE/../../bin:$PATH
+PYTHONPATH=$HERE/../../:${PYTHONPATH:-}
+export PATH PYTHONPATH
+
+CONF=$1
+LOG=${CONF%%.conf}.log
+ARCH=$(head -n1 "$LOG" | tr ' ' '\n' | grep -- '--arch=')
+
+CMD=(pungi-gather "--config=$CONF" "$ARCH" $(head -n1 "$LOG" | tr ' ' '\n' | grep '^--\(selfhosting\|fulltree\|greedy\|multilib\)'))
+
+echo "${CMD[@]}"
+if [ $# -le 1 ] || [ "$2" != "--interactive" ]; then
+    exec >"$LOG.dnf"
+fi
+exec 2>&1
+
+exec "${CMD[@]}"

contrib/yum-dnf-compare/run-yum.sh

@@ -0,0 +1,28 @@
+#!/bin/sh
+
+set -e
+set -o pipefail
+set -u
+
+export LANG=C
+
+HERE=$(dirname "$0")
+PATH=$HERE/../../bin:$PATH
+PYTHONPATH=$HERE/../../
+export PATH PYTHONPATH
+
+CONF="$1"
+LOG=${CONF%%.conf}.log
+
+tempdir=$(mktemp -d)
+trap 'rm -rf $tempdir' EXIT
+
+cmd=$(head -n1 "$LOG" | cut -d' ' -f2- | sed "s@--\(destdir\|cachedir\)=\(/[^/ ]*\)*@--\1=$tempdir/\1@g" | sed 's/^pungi3/pungi/' | sed "s@--config=/\([^/]*/\)*work/[^/]*/pungi/\([^ ]*\)@--config=$1@g")
+
+echo "$cmd"
+if [ $# -le 1 ] || [ "$2" != "--interactive" ]; then
+    exec >"$LOG.yum"
+fi
+exec 2>&1
+
+$cmd

doc/Makefile

@@ -0,0 +1,177 @@
+# Makefile for Sphinx documentation
+#
+
+# You can set these variables from the command line.
+SPHINXOPTS    =
+SPHINXBUILD   = sphinx-build
+PAPER         =
+BUILDDIR      = _build
+
+# User-friendly check for sphinx-build
+ifeq ($(shell which $(SPHINXBUILD) >/dev/null 2>&1; echo $$?), 1)
+$(error The '$(SPHINXBUILD)' command was not found. Make sure you have Sphinx installed, then set the SPHINXBUILD environment variable to point to the full path of the '$(SPHINXBUILD)' executable. Alternatively you can add the directory with the executable to your PATH. If you don't have Sphinx installed, grab it from http://sphinx-doc.org/)
+endif
+
+# Internal variables.
+PAPEROPT_a4     = -D latex_paper_size=a4
+PAPEROPT_letter = -D latex_paper_size=letter
+ALLSPHINXOPTS   = -d $(BUILDDIR)/doctrees $(PAPEROPT_$(PAPER)) $(SPHINXOPTS) .
+# the i18n builder cannot share the environment and doctrees with the others
+I18NSPHINXOPTS  = $(PAPEROPT_$(PAPER)) $(SPHINXOPTS) .
+
+.PHONY: help clean html dirhtml singlehtml pickle json htmlhelp qthelp devhelp epub latex latexpdf text man changes linkcheck doctest gettext
+
+help:
+	@echo "Please use \`make <target>' where <target> is one of"
+	@echo "  html       to make standalone HTML files"
+	@echo "  dirhtml    to make HTML files named index.html in directories"
+	@echo "  singlehtml to make a single large HTML file"
+	@echo "  pickle     to make pickle files"
+	@echo "  json       to make JSON files"
+	@echo "  htmlhelp   to make HTML files and a HTML help project"
+	@echo "  qthelp     to make HTML files and a qthelp project"
+	@echo "  devhelp    to make HTML files and a Devhelp project"
+	@echo "  epub       to make an epub"
+	@echo "  latex      to make LaTeX files, you can set PAPER=a4 or PAPER=letter"
+	@echo "  latexpdf   to make LaTeX files and run them through pdflatex"
+	@echo "  latexpdfja to make LaTeX files and run them through platex/dvipdfmx"
+	@echo "  text       to make text files"
+	@echo "  man        to make manual pages"
+	@echo "  texinfo    to make Texinfo files"
+	@echo "  info       to make Texinfo files and run them through makeinfo"
+	@echo "  gettext    to make PO message catalogs"
+	@echo "  changes    to make an overview of all changed/added/deprecated items"
+	@echo "  xml        to make Docutils-native XML files"
+	@echo "  pseudoxml  to make pseudoxml-XML files for display purposes"
+	@echo "  linkcheck  to check all external links for integrity"
+	@echo "  doctest    to run all doctests embedded in the documentation (if enabled)"
+
+clean:
+	rm -rf $(BUILDDIR)/*
+
+html:
+	$(SPHINXBUILD) -b html $(ALLSPHINXOPTS) $(BUILDDIR)/html
+	@echo
+	@echo "Build finished. The HTML pages are in $(BUILDDIR)/html."
+
+dirhtml:
+	$(SPHINXBUILD) -b dirhtml $(ALLSPHINXOPTS) $(BUILDDIR)/dirhtml
+	@echo
+	@echo "Build finished. The HTML pages are in $(BUILDDIR)/dirhtml."
+
+singlehtml:
+	$(SPHINXBUILD) -b singlehtml $(ALLSPHINXOPTS) $(BUILDDIR)/singlehtml
+	@echo
+	@echo "Build finished. The HTML page is in $(BUILDDIR)/singlehtml."
+
+pickle:
+	$(SPHINXBUILD) -b pickle $(ALLSPHINXOPTS) $(BUILDDIR)/pickle
+	@echo
+	@echo "Build finished; now you can process the pickle files."
+
+json:
+	$(SPHINXBUILD) -b json $(ALLSPHINXOPTS) $(BUILDDIR)/json
+	@echo
+	@echo "Build finished; now you can process the JSON files."
+
+htmlhelp:
+	$(SPHINXBUILD) -b htmlhelp $(ALLSPHINXOPTS) $(BUILDDIR)/htmlhelp
+	@echo
+	@echo "Build finished; now you can run HTML Help Workshop with the" \
+	      ".hhp project file in $(BUILDDIR)/htmlhelp."
+
+qthelp:
+	$(SPHINXBUILD) -b qthelp $(ALLSPHINXOPTS) $(BUILDDIR)/qthelp
+	@echo
+	@echo "Build finished; now you can run "qcollectiongenerator" with the" \
+	      ".qhcp project file in $(BUILDDIR)/qthelp, like this:"
+	@echo "# qcollectiongenerator $(BUILDDIR)/qthelp/Pungi.qhcp"
+	@echo "To view the help file:"
+	@echo "# assistant -collectionFile $(BUILDDIR)/qthelp/Pungi.qhc"
+
+devhelp:
+	$(SPHINXBUILD) -b devhelp $(ALLSPHINXOPTS) $(BUILDDIR)/devhelp
+	@echo
+	@echo "Build finished."
+	@echo "To view the help file:"
+	@echo "# mkdir -p $$HOME/.local/share/devhelp/Pungi"
+	@echo "# ln -s $(BUILDDIR)/devhelp $$HOME/.local/share/devhelp/Pungi"
+	@echo "# devhelp"
+
+epub:
+	$(SPHINXBUILD) -b epub $(ALLSPHINXOPTS) $(BUILDDIR)/epub
+	@echo
+	@echo "Build finished. The epub file is in $(BUILDDIR)/epub."
+
+latex:
+	$(SPHINXBUILD) -b latex $(ALLSPHINXOPTS) $(BUILDDIR)/latex
+	@echo
+	@echo "Build finished; the LaTeX files are in $(BUILDDIR)/latex."
+	@echo "Run \`make' in that directory to run these through (pdf)latex" \
+	      "(use \`make latexpdf' here to do that automatically)."
+
+latexpdf:
+	$(SPHINXBUILD) -b latex $(ALLSPHINXOPTS) $(BUILDDIR)/latex
+	@echo "Running LaTeX files through pdflatex..."
+	$(MAKE) -C $(BUILDDIR)/latex all-pdf
+	@echo "pdflatex finished; the PDF files are in $(BUILDDIR)/latex."
+
+latexpdfja:
+	$(SPHINXBUILD) -b latex $(ALLSPHINXOPTS) $(BUILDDIR)/latex
+	@echo "Running LaTeX files through platex and dvipdfmx..."
+	$(MAKE) -C $(BUILDDIR)/latex all-pdf-ja
+	@echo "pdflatex finished; the PDF files are in $(BUILDDIR)/latex."
+
+text:
+	$(SPHINXBUILD) -b text $(ALLSPHINXOPTS) $(BUILDDIR)/text
+	@echo
+	@echo "Build finished. The text files are in $(BUILDDIR)/text."
+
+man:
+	$(SPHINXBUILD) -b man $(ALLSPHINXOPTS) $(BUILDDIR)/man
+	@echo
+	@echo "Build finished. The manual pages are in $(BUILDDIR)/man."
+
+texinfo:
+	$(SPHINXBUILD) -b texinfo $(ALLSPHINXOPTS) $(BUILDDIR)/texinfo
+	@echo
+	@echo "Build finished. The Texinfo files are in $(BUILDDIR)/texinfo."
+	@echo "Run \`make' in that directory to run these through makeinfo" \
+	      "(use \`make info' here to do that automatically)."
+
+info:
+	$(SPHINXBUILD) -b texinfo $(ALLSPHINXOPTS) $(BUILDDIR)/texinfo
+	@echo "Running Texinfo files through makeinfo..."
+	make -C $(BUILDDIR)/texinfo info
+	@echo "makeinfo finished; the Info files are in $(BUILDDIR)/texinfo."
+
+gettext:
+	$(SPHINXBUILD) -b gettext $(I18NSPHINXOPTS) $(BUILDDIR)/locale
+	@echo
+	@echo "Build finished. The message catalogs are in $(BUILDDIR)/locale."
+
+changes:
+	$(SPHINXBUILD) -b changes $(ALLSPHINXOPTS) $(BUILDDIR)/changes
+	@echo
+	@echo "The overview file is in $(BUILDDIR)/changes."
+
+linkcheck:
+	$(SPHINXBUILD) -b linkcheck $(ALLSPHINXOPTS) $(BUILDDIR)/linkcheck
+	@echo
+	@echo "Link check complete; look for any errors in the above output " \
+	      "or in $(BUILDDIR)/linkcheck/output.txt."
+
+doctest:
+	$(SPHINXBUILD) -b doctest $(ALLSPHINXOPTS) $(BUILDDIR)/doctest
+	@echo "Testing of doctests in the sources finished, look at the " \
+	      "results in $(BUILDDIR)/doctest/output.txt."
+
+xml:
+	$(SPHINXBUILD) -b xml $(ALLSPHINXOPTS) $(BUILDDIR)/xml
+	@echo
+	@echo "Build finished. The XML files are in $(BUILDDIR)/xml."
+
+pseudoxml:
+	$(SPHINXBUILD) -b pseudoxml $(ALLSPHINXOPTS) $(BUILDDIR)/pseudoxml
+	@echo
+	@echo "Build finished. The pseudo-XML files are in $(BUILDDIR)/pseudoxml."

doc/_static/phases.svg

@@ -0,0 +1,557 @@
+<?xml version="1.0" encoding="UTF-8" standalone="no"?>
+<svg
+   width="610.46454"
+   height="327.16599"
+   viewBox="0 0 610.46457 327.16599"
+   id="svg2"
+   version="1.1"
+   inkscape:version="1.3.2 (091e20e, 2023-11-25)"
+   sodipodi:docname="phases.svg"
+   inkscape:export-filename="/home/lsedlar/repos/pungi/doc/_static/phases.png"
+   inkscape:export-xdpi="90"
+   inkscape:export-ydpi="90"
+   xmlns:inkscape="http://www.inkscape.org/namespaces/inkscape"
+   xmlns:sodipodi="http://sodipodi.sourceforge.net/DTD/sodipodi-0.dtd"
+   xmlns="http://www.w3.org/2000/svg"
+   xmlns:svg="http://www.w3.org/2000/svg"
+   xmlns:rdf="http://www.w3.org/1999/02/22-rdf-syntax-ns#"
+   xmlns:cc="http://creativecommons.org/ns#"
+   xmlns:dc="http://purl.org/dc/elements/1.1/">
+  <sodipodi:namedview
+     id="base"
+     pagecolor="#ffffff"
+     bordercolor="#666666"
+     borderopacity="1.0"
+     inkscape:pageopacity="1"
+     inkscape:pageshadow="2"
+     inkscape:zoom="1.5"
+     inkscape:cx="268"
+     inkscape:cy="260.66667"
+     inkscape:document-units="px"
+     inkscape:current-layer="layer1"
+     showgrid="false"
+     inkscape:window-width="1920"
+     inkscape:window-height="1027"
+     inkscape:window-x="0"
+     inkscape:window-y="25"
+     inkscape:window-maximized="1"
+     units="px"
+     inkscape:document-rotation="0"
+     showguides="true"
+     inkscape:guide-bbox="true"
+     fit-margin-top="7.4"
+     fit-margin-left="7.4"
+     fit-margin-right="7.4"
+     fit-margin-bottom="7.4"
+     lock-margins="true"
+     inkscape:showpageshadow="2"
+     inkscape:pagecheckerboard="0"
+     inkscape:deskcolor="#d1d1d1" />
+  <defs
+     id="defs4">
+    <marker
+       inkscape:isstock="true"
+       style="overflow:visible"
+       id="Arrow1Lend"
+       refX="0"
+       refY="0"
+       orient="auto"
+       inkscape:stockid="Arrow1Lend">
+      <path
+         inkscape:connector-curvature="0"
+         transform="matrix(-0.8,0,0,-0.8,-10,0)"
+         style="fill:#000000;fill-opacity:1;fill-rule:evenodd;stroke:#000000;stroke-width:1pt;stroke-opacity:1"
+         d="M 0,0 5,-5 -12.5,0 5,5 Z"
+         id="path4451" />
+    </marker>
+  </defs>
+  <metadata
+     id="metadata7">
+    <rdf:RDF>
+      <cc:Work
+         rdf:about="">
+        <dc:format>image/svg+xml</dc:format>
+        <dc:type
+           rdf:resource="http://purl.org/dc/dcmitype/StillImage" />
+      </cc:Work>
+    </rdf:RDF>
+  </metadata>
+  <g
+     transform="matrix(1.066667,0,0,1.066667,-99.07321,-903.45239)"
+     id="layer1"
+     inkscape:groupmode="layer"
+     inkscape:label="Vrstva 1">
+    <g
+       transform="translate(98.243246,-80.817124)"
+       id="g3411">
+      <rect
+         style="fill:#8ae234;fill-rule:evenodd;stroke:none;stroke-width:1px;stroke-linecap:butt;stroke-linejoin:miter;stroke-opacity:1"
+         id="rect3340"
+         width="26.295755"
+         height="49.214859"
+         x="953.49097"
+         y="49.250374"
+         transform="matrix(0,1,1,0,0,0)" />
+      <text
+         xml:space="preserve"
+         style="font-style:normal;font-weight:normal;line-height:0%;font-family:sans-serif;letter-spacing:0px;word-spacing:0px;fill:#000000;fill-opacity:1;stroke:none;stroke-width:1px;stroke-linecap:butt;stroke-linejoin:miter;stroke-opacity:1"
+         x="51.554729"
+         y="970.26605"
+         id="text3360"><tspan
+           sodipodi:role="line"
+           id="tspan3362"
+           x="51.554729"
+           y="970.26605"
+           style="font-size:13.1479px;line-height:1.25">Pkgset</tspan></text>
+    </g>
+    <g
+       transform="translate(56.378954,-80.817124)"
+       id="g3398">
+      <rect
+         y="553.98242"
+         x="953.49097"
+         height="46.01757"
+         width="26.295755"
+         id="rect3400"
+         style="fill:#3465a4;fill-rule:evenodd;stroke:none;stroke-width:1px;stroke-linecap:butt;stroke-linejoin:miter;stroke-opacity:1"
+         transform="matrix(0,1,1,0,0,0)" />
+      <text
+         xml:space="preserve"
+         style="font-style:normal;font-weight:normal;line-height:0%;font-family:sans-serif;letter-spacing:0px;word-spacing:0px;fill:#ffffff;fill-opacity:1;stroke:none;stroke-width:1px;stroke-linecap:butt;stroke-linejoin:miter;stroke-opacity:1"
+         x="557.61566"
+         y="971.33813"
+         id="text3396"><tspan
+           sodipodi:role="line"
+           id="tspan3398"
+           x="557.61566"
+           y="971.33813"
+           style="font-size:13.1479px;line-height:1.25">Test</tspan></text>
+    </g>
+    <g
+       id="g3720"
+       transform="translate(97.49995,-0.34404039)">
+      <rect
+         style="fill:#fce94f;fill-rule:evenodd;stroke:none;stroke-width:1px;stroke-linecap:butt;stroke-linejoin:miter;stroke-opacity:1"
+         id="rect3336"
+         width="26.295755"
+         height="39.669899"
+         x="873.01788"
+         y="2.3186533"
+         transform="matrix(0,1,1,0,0,0)" />
+      <text
+         xml:space="preserve"
+         style="font-style:normal;font-weight:normal;line-height:0%;font-family:sans-serif;letter-spacing:0px;word-spacing:0px;fill:#000000;fill-opacity:1;stroke:none;stroke-width:1px;stroke-linecap:butt;stroke-linejoin:miter;stroke-opacity:1"
+         x="6.2600794"
+         y="891.1604"
+         id="text3356"><tspan
+           sodipodi:role="line"
+           id="tspan3358"
+           x="6.2600794"
+           y="891.1604"
+           style="font-size:13.1479px;line-height:1.25">Init</tspan></text>
+    </g>
+    <path
+       inkscape:connector-curvature="0"
+       id="path3642"
+       d="M 100.90864,859.8891 H 654.22706"
+       style="fill:none;fill-rule:evenodd;stroke:#000000;stroke-width:1.17467px;stroke-linecap:butt;stroke-linejoin:miter;stroke-opacity:1;marker-end:url(#Arrow1Lend)" />
+    <g
+       transform="translate(26.249988)"
+       id="g262">
+      <g
+         id="g234">
+        <rect
+           transform="matrix(0,1,1,0,0,0)"
+           y="179.38934"
+           x="872.67383"
+           height="162.72726"
+           width="26.295755"
+           id="rect3342"
+           style="fill:#fcaf3e;fill-rule:evenodd;stroke:none;stroke-width:0.838448px;stroke-linecap:butt;stroke-linejoin:miter;stroke-opacity:1" />
+        <text
+           id="text3364"
+           y="890.72327"
+           x="181.69368"
+           style="font-style:normal;font-weight:normal;line-height:0%;font-family:sans-serif;letter-spacing:0px;word-spacing:0px;fill:#000000;fill-opacity:1;stroke:none;stroke-width:1px;stroke-linecap:butt;stroke-linejoin:miter;stroke-opacity:1"
+           xml:space="preserve"><tspan
+             style="font-size:13.1479px;line-height:1.25"
+             y="890.72327"
+             x="181.69368"
+             id="tspan3366"
+             sodipodi:role="line">Buildinstall</tspan></text>
+      </g>
+      <g
+         id="g3639"
+         transform="translate(75.925692,-0.34404039)">
+        <rect
+           transform="matrix(0,1,1,0,0,0)"
+           y="103.28194"
+           x="905.2099"
+           height="54.197887"
+           width="26.295755"
+           id="rect3344"
+           style="fill:#729fcf;fill-rule:evenodd;stroke:none;stroke-width:1px;stroke-linecap:butt;stroke-linejoin:miter;stroke-opacity:1" />
+        <text
+           id="text3368"
+           y="923.25934"
+           x="106.1384"
+           style="font-style:normal;font-weight:normal;line-height:0%;font-family:sans-serif;letter-spacing:0px;word-spacing:0px;fill:#000000;fill-opacity:1;stroke:none;stroke-width:1px;stroke-linecap:butt;stroke-linejoin:miter;stroke-opacity:1"
+           xml:space="preserve"><tspan
+             style="font-size:13.1479px;line-height:1.25"
+             y="923.25934"
+             x="106.1384"
+             id="tspan3370"
+             sodipodi:role="line">Gather</tspan></text>
+      </g>
+      <g
+         transform="translate(15.925722,63.405928)"
+         id="g3647">
+        <g
+           id="g3644">
+          <rect
+             style="fill:#ad7fa8;fill-rule:evenodd;stroke:none;stroke-width:1px;stroke-linecap:butt;stroke-linejoin:miter;stroke-opacity:1"
+             id="rect3346"
+             width="26.295755"
+             height="72.729973"
+             x="905.2099"
+             y="162.92607"
+             transform="matrix(0,1,1,0,0,0)" />
+        </g>
+        <text
+           id="text3372"
+           y="923.25934"
+           x="165.23042"
+           style="font-style:normal;font-weight:normal;line-height:0%;font-family:sans-serif;letter-spacing:0px;word-spacing:0px;fill:#000000;fill-opacity:1;stroke:none;stroke-width:1px;stroke-linecap:butt;stroke-linejoin:miter;stroke-opacity:1"
+           xml:space="preserve"><tspan
+             style="font-size:13.1479px;line-height:1.25"
+             y="923.25934"
+             x="165.23042"
+             id="tspan3374"
+             sodipodi:role="line">ExtraFiles</tspan></text>
+      </g>
+      <g
+         transform="translate(-2.824268,-0.34404039)"
+         id="g3658">
+        <rect
+           transform="matrix(0,1,1,0,0,0)"
+           y="241.10229"
+           x="905.2099"
+           height="78.636055"
+           width="26.295755"
+           id="rect3348"
+           style="fill:#e9b96e;fill-rule:evenodd;stroke:none;stroke-width:1px;stroke-linecap:butt;stroke-linejoin:miter;stroke-opacity:1" />
+        <text
+           id="text3376"
+           y="921.86945"
+           x="243.95874"
+           style="font-style:normal;font-weight:normal;line-height:0%;font-family:sans-serif;letter-spacing:0px;word-spacing:0px;fill:#000000;fill-opacity:1;stroke:none;stroke-width:1px;stroke-linecap:butt;stroke-linejoin:miter;stroke-opacity:1"
+           xml:space="preserve"><tspan
+             style="font-size:13.1479px;line-height:1.25"
+             y="921.86945"
+             x="243.95874"
+             id="tspan3378"
+             sodipodi:role="line">Createrepo</tspan></text>
+      </g>
+      <g
+         id="g3408"
+         transform="translate(-74.638308,113.77258)">
+        <rect
+           transform="matrix(0,1,1,0,0,0)"
+           y="254.60153"
+           x="823.54675"
+           height="53.653927"
+           width="26.295755"
+           id="rect3350-3"
+           style="fill:#729fcf;fill-rule:evenodd;stroke:none;stroke-width:1px;stroke-linecap:butt;stroke-linejoin:miter;stroke-opacity:1" />
+        <text
+           id="text3380-2"
+           y="840.3219"
+           x="256.90588"
+           style="font-style:normal;font-weight:normal;line-height:0%;font-family:sans-serif;text-align:start;letter-spacing:0px;word-spacing:0px;writing-mode:lr-tb;text-anchor:start;fill:#ffffff;fill-opacity:1;stroke:none;stroke-width:1px;stroke-linecap:butt;stroke-linejoin:miter;stroke-opacity:1"
+           xml:space="preserve"><tspan
+             style="font-size:13.1479px;line-height:1.25"
+             id="tspan3406"
+             sodipodi:role="line"
+             x="256.90588"
+             y="840.3219">OSTree</tspan></text>
+      </g>
+      <g
+         transform="translate(-252.46536,-85.861863)"
+         id="g288">
+        <g
+           transform="translate(0.56706579)"
+           id="g3653">
+          <rect
+             style="fill:#fcaf3e;fill-rule:evenodd;stroke:none;stroke-width:1px;stroke-linecap:butt;stroke-linejoin:miter;stroke-opacity:1"
+             id="rect3428"
+             width="26.295755"
+             height="101.85102"
+             x="1022.637"
+             y="490.33765"
+             transform="matrix(0,1,1,0,0,0)" />
+          <text
+             xml:space="preserve"
+             style="font-style:normal;font-variant:normal;font-weight:normal;font-stretch:normal;line-height:0%;font-family:sans-serif;-inkscape-font-specification:'sans-serif, Normal';text-align:start;letter-spacing:0px;word-spacing:0px;writing-mode:lr-tb;text-anchor:start;fill:#000000;fill-opacity:1;stroke:none;stroke-width:1px;stroke-linecap:butt;stroke-linejoin:miter;stroke-opacity:1"
+             x="492.642"
+             y="1039.4121"
+             id="text3430"><tspan
+               id="tspan283"
+               sodipodi:role="line"
+               x="492.642"
+               y="1039.4121"
+               style="font-size:12px;line-height:0">OSTreeInstaller</tspan></text>
+        </g>
+      </g>
+    </g>
+    <g
+       id="g2"
+       transform="translate(-1.4062678e-8,9.3749966)">
+      <rect
+         transform="matrix(0,1,1,0,0,0)"
+         style="fill:#e9b96e;fill-rule:evenodd;stroke:none;stroke-width:1.85901px;stroke-linecap:butt;stroke-linejoin:miter;stroke-opacity:1"
+         id="rect3338-1"
+         width="103.12497"
+         height="115.80065"
+         x="863.29883"
+         y="486.55563" />
+      <text
+         id="text3384-0"
+         y="921.73846"
+         x="489.56451"
+         style="font-style:normal;font-variant:normal;font-weight:normal;font-stretch:normal;line-height:0%;font-family:sans-serif;-inkscape-font-specification:'sans-serif, Normal';text-align:start;letter-spacing:0px;word-spacing:0px;writing-mode:lr-tb;text-anchor:start;fill:#000000;fill-opacity:1;stroke:none;stroke-width:1px;stroke-linecap:butt;stroke-linejoin:miter;stroke-opacity:1"
+         xml:space="preserve"><tspan
+           style="font-size:13.1475px;line-height:1.25"
+           id="tspan3391"
+           sodipodi:role="line"
+           x="489.56451"
+           y="921.73846">ImageChecksum</tspan></text>
+    </g>
+    <g
+       transform="translate(-42.209584,-80.817124)"
+       id="g3458">
+      <rect
+         transform="matrix(0,1,1,0,0,0)"
+         style="fill:#edd400;fill-rule:evenodd;stroke:none;stroke-width:1px;stroke-linecap:butt;stroke-linejoin:miter;stroke-opacity:1"
+         id="rect3338"
+         width="26.295755"
+         height="102.36562"
+         x="953.49097"
+         y="420.13605" />
+      <text
+         id="text3384"
+         y="971.54041"
+         x="422.99252"
+         style="font-style:normal;font-weight:normal;line-height:0%;font-family:sans-serif;letter-spacing:0px;word-spacing:0px;fill:#000000;fill-opacity:1;stroke:none;stroke-width:1px;stroke-linecap:butt;stroke-linejoin:miter;stroke-opacity:1"
+         xml:space="preserve"><tspan
+           y="971.54041"
+           x="422.99252"
+           id="tspan3386"
+           sodipodi:role="line"
+           style="font-size:13.1479px;line-height:1.25">Createiso</tspan></text>
+    </g>
+    <g
+       id="g3453"
+       transform="translate(-42.466031,-84.525321)">
+      <rect
+         transform="matrix(0,1,1,0,0,0)"
+         y="420.39337"
+         x="989.65247"
+         height="101.85102"
+         width="26.295755"
+         id="rect3352"
+         style="fill:#73d216;fill-rule:evenodd;stroke:none;stroke-width:1px;stroke-linecap:butt;stroke-linejoin:miter;stroke-opacity:1" />
+      <text
+         id="text3388"
+         y="1006.4276"
+         x="422.69772"
+         style="font-style:normal;font-weight:normal;line-height:0%;font-family:sans-serif;letter-spacing:0px;word-spacing:0px;fill:#000000;fill-opacity:1;stroke:none;stroke-width:1px;stroke-linecap:butt;stroke-linejoin:miter;stroke-opacity:1"
+         xml:space="preserve"><tspan
+           y="1006.4276"
+           x="422.69772"
+           id="tspan3390"
+           sodipodi:role="line"
+           style="font-size:13.1479px;line-height:1.25">LiveImages</tspan></text>
+    </g>
+    <g
+       id="g3448"
+       transform="translate(-42.466031,-88.485966)">
+      <rect
+         transform="matrix(0,1,1,0,0,0)"
+         y="420.39337"
+         x="1026.0664"
+         height="101.85102"
+         width="26.295755"
+         id="rect3354"
+         style="fill:#f57900;fill-rule:evenodd;stroke:none;stroke-width:1px;stroke-linecap:butt;stroke-linejoin:miter;stroke-opacity:1" />
+      <text
+         id="text3392"
+         y="1042.8416"
+         x="422.69772"
+         style="font-style:normal;font-weight:normal;line-height:0%;font-family:sans-serif;letter-spacing:0px;word-spacing:0px;fill:#000000;fill-opacity:1;stroke:none;stroke-width:1px;stroke-linecap:butt;stroke-linejoin:miter;stroke-opacity:1"
+         xml:space="preserve"><tspan
+           y="1042.8416"
+           x="422.69772"
+           id="tspan3394"
+           sodipodi:role="line"
+           style="font-size:13.1479px;line-height:1.25">ImageBuild</tspan></text>
+    </g>
+    <g
+       id="g3443"
+       transform="translate(-43.173123,-92.80219)">
+      <rect
+         style="fill:#edd400;fill-rule:evenodd;stroke:none;stroke-width:1px;stroke-linecap:butt;stroke-linejoin:miter;stroke-opacity:1"
+         id="rect3422"
+         width="26.295755"
+         height="101.85102"
+         x="1062.8359"
+         y="421.10046"
+         transform="matrix(0,1,1,0,0,0)" />
+      <text
+         xml:space="preserve"
+         style="font-style:normal;font-weight:normal;line-height:0%;font-family:sans-serif;text-align:start;letter-spacing:0px;word-spacing:0px;writing-mode:lr-tb;text-anchor:start;fill:#000000;fill-opacity:1;stroke:none;stroke-width:1px;stroke-linecap:butt;stroke-linejoin:miter;stroke-opacity:1"
+         x="423.40482"
+         y="1079.6111"
+         id="text3424"><tspan
+           id="tspan3434"
+           sodipodi:role="line"
+           x="423.40482"
+           y="1079.6111"
+           style="font-size:13.1479px;line-height:1.25">LiveMedia</tspan></text>
+    </g>
+    <rect
+       style="fill:#c17d11;fill-rule:evenodd;stroke:none;stroke-width:1.48416px;stroke-linecap:butt;stroke-linejoin:miter;stroke-opacity:1"
+       id="rect290"
+       width="26.295755"
+       height="224.35098"
+       x="1091.7223"
+       y="378.43698"
+       transform="matrix(0,1,1,0,0,0)" />
+    <text
+       xml:space="preserve"
+       style="font-style:normal;font-variant:normal;font-weight:normal;font-stretch:normal;line-height:0%;font-family:sans-serif;-inkscape-font-specification:'sans-serif, Normal';text-align:start;letter-spacing:0px;word-spacing:0px;writing-mode:lr-tb;text-anchor:start;fill:#000000;fill-opacity:1;stroke:none;stroke-width:1px;stroke-linecap:butt;stroke-linejoin:miter;stroke-opacity:1"
+       x="380.74133"
+       y="1106.6223"
+       id="text294"><tspan
+         y="1106.6223"
+         x="380.74133"
+         sodipodi:role="line"
+         id="tspan301"
+         style="font-size:12px;line-height:0">OSBS</tspan></text>
+    <g
+       transform="translate(-70.933542,-51.043149)"
+       id="g3819">
+      <rect
+         style="fill:#73d216;fill-rule:evenodd;stroke:none;stroke-width:1px;stroke-linecap:butt;stroke-linejoin:miter;stroke-opacity:1"
+         id="rect3801"
+         width="26.295755"
+         height="101.85102"
+         x="1052.2335"
+         y="448.86087"
+         transform="matrix(0,1,1,0,0,0)" />
+      <text
+         xml:space="preserve"
+         style="font-style:normal;font-weight:normal;line-height:0%;font-family:sans-serif;letter-spacing:0px;word-spacing:0px;fill:#000000;fill-opacity:1;stroke:none;stroke-width:1px;stroke-linecap:butt;stroke-linejoin:miter;stroke-opacity:1"
+         x="451.16522"
+         y="1069.0087"
+         id="text3805"><tspan
+           style="font-size:13.1479px;line-height:1.25"
+           sodipodi:role="line"
+           x="451.16522"
+           y="1069.0087"
+           id="tspan3812">ExtraIsos</tspan></text>
+    </g>
+    <rect
+       y="377.92242"
+       x="1122.3463"
+       height="224.24059"
+       width="26.295755"
+       id="rect87"
+       style="fill:#5ed4ec;fill-opacity:1;fill-rule:evenodd;stroke:none;stroke-width:1.48006px;stroke-linecap:butt;stroke-linejoin:miter;stroke-opacity:1"
+       transform="matrix(0,1,1,0,0,0)" />
+    <text
+       xml:space="preserve"
+       style="font-style:normal;font-weight:normal;line-height:0%;font-family:sans-serif;letter-spacing:0px;word-spacing:0px;fill:#000000;fill-opacity:1;stroke:none;stroke-width:1px;stroke-linecap:butt;stroke-linejoin:miter;stroke-opacity:1"
+       x="380.7789"
+       y="1140.3958"
+       id="text91"><tspan
+         style="font-size:13.1479px;line-height:1.25"
+         sodipodi:role="line"
+         id="tspan89"
+         x="380.7789"
+         y="1140.3958">Repoclosure</tspan></text>
+    <g
+       id="g206"
+       transform="translate(0,-1.8749994)">
+      <rect
+         style="fill:#fcd9a4;fill-opacity:1;fill-rule:evenodd;stroke:none;stroke-width:1.00033px;stroke-linecap:butt;stroke-linejoin:miter;stroke-opacity:1"
+         id="rect290-6"
+         width="26.295755"
+         height="101.91849"
+         x="1032.3469"
+         y="377.92731"
+         transform="matrix(0,1,1,0,0,0)" />
+      <text
+         xml:space="preserve"
+         style="font-style:normal;font-variant:normal;font-weight:normal;font-stretch:normal;line-height:0%;font-family:sans-serif;-inkscape-font-specification:'sans-serif, Normal';text-align:start;letter-spacing:0px;word-spacing:0px;writing-mode:lr-tb;text-anchor:start;fill:#000000;fill-opacity:1;stroke:none;stroke-width:1px;stroke-linecap:butt;stroke-linejoin:miter;stroke-opacity:1"
+         x="380.23166"
+         y="1049.1219"
+         id="text294-7"><tspan
+           y="1049.1219"
+           x="380.23166"
+           sodipodi:role="line"
+           id="tspan301-5"
+           style="font-size:12px;line-height:0">KiwiBuild</tspan></text>
+    </g>
+    <g
+       id="g3">
+      <g
+         id="g1">
+        <g
+           id="g4">
+          <rect
+             transform="matrix(0,1,1,0,0,0)"
+             style="fill:#729fcf;fill-rule:evenodd;stroke:none;stroke-width:1.83502px;stroke-linecap:butt;stroke-linejoin:miter;stroke-opacity:1"
+             id="rect3338-1-3"
+             width="103.12497"
+             height="115.80065"
+             x="983.44263"
+             y="486.55563" />
+          <text
+             id="text3384-0-6"
+             y="1038.8422"
+             x="489.56451"
+             style="font-style:normal;font-variant:normal;font-weight:normal;font-stretch:normal;line-height:0%;font-family:sans-serif;-inkscape-font-specification:'sans-serif, Normal';text-align:start;letter-spacing:0px;word-spacing:0px;writing-mode:lr-tb;text-anchor:start;fill:#000000;fill-opacity:1;stroke:none;stroke-width:1px;stroke-linecap:butt;stroke-linejoin:miter;stroke-opacity:1"
+             xml:space="preserve"><tspan
+               style="font-size:13.1475px;line-height:1.25"
+               id="tspan3391-7"
+               sodipodi:role="line"
+               x="489.56451"
+               y="1038.8422">ImageContainer</tspan></text>
+        </g>
+      </g>
+    </g>
+    <g
+       id="g206-1"
+       transform="translate(-0.04628921,28.701853)">
+      <rect
+         style="fill:#fcaf3e;fill-rule:evenodd;stroke:none;stroke-width:1.00033px;stroke-linecap:butt;stroke-linejoin:miter;stroke-opacity:1"
+         id="rect290-6-7"
+         width="26.295755"
+         height="101.91849"
+         x="1032.3469"
+         y="377.92731"
+         transform="matrix(0,1,1,0,0,0)" />
+      <text
+         xml:space="preserve"
+         style="font-style:normal;font-variant:normal;font-weight:normal;font-stretch:normal;line-height:0%;font-family:sans-serif;-inkscape-font-specification:'sans-serif, Normal';text-align:start;letter-spacing:0px;word-spacing:0px;writing-mode:lr-tb;text-anchor:start;fill:#000000;fill-opacity:1;stroke:none;stroke-width:1px;stroke-linecap:butt;stroke-linejoin:miter;stroke-opacity:1"
+         x="380.23166"
+         y="1049.1219"
+         id="text294-7-5"><tspan
+           y="1049.1219"
+           x="380.23166"
+           sodipodi:role="line"
+           id="tspan301-5-5"
+           style="font-size:12px;line-height:0">OSBuild</tspan></text>
+    </g>
+  </g>
+</svg>

doc/about.rst

@@ -0,0 +1,70 @@
+=============
+ About Pungi
+=============
+
+.. figure:: _static/pungi_snake-sm-dark.png
+    :align: right
+    :alt: Pungi Logo
+
+*Pungi* is a distribution compose tool.
+
+Composes are release snapshots that contain release deliverables such as:
+
+- installation trees
+
+  - RPMs
+  - repodata
+  - comps
+
+- (bootable) ISOs
+- kickstart trees
+
+  - anaconda images
+  - images for PXE boot
+
+
+Tool overview
+=============
+
+*Pungi* consists of multiple separate executables backed by a common library.
+
+The main entry-point is the ``pungi-koji`` script. It loads the compose
+configuration and kicks off the process. Composing itself is done in phases.
+Each phase is responsible for generating some artifacts on disk and updating
+the ``compose`` object that is threaded through all the phases.
+
+*Pungi* itself does not actually do that much. Most of the actual work is
+delegated to separate executables. *Pungi* just makes sure that all the
+commands are invoked in the appropriate order and with correct arguments. It
+also moves the artifacts to correct locations.
+
+The executable name ``pungi-koji`` comes from the fact that most of those
+separate executables submit tasks to Koji that does the actual work in an
+auditable way.
+
+However unlike doing everything manually in Koji, Pungi will make sure you are
+building all images from the same package set, and will produce even
+deliverables that Koji can not create like YUM repos and installer ISOs.
+
+
+Links
+=====
+- Upstream GIT: https://pagure.io/pungi/
+- Issue tracker: https://pagure.io/pungi/issues
+- Questions can be asked on *#fedora-releng* IRC channel on FreeNode
+
+
+Origin of name
+==============
+
+The name *Pungi* comes from the instrument used to charm snakes. *Anaconda*
+being the software Pungi was manipulating, and anaconda being a snake, led to
+the referential naming.
+
+The first name, which was suggested by Seth Vidal, was *FIST*, *Fedora
+Installation <Something> Tool*. That name was quickly discarded and replaced
+with Pungi.
+
+There was also a bit of an inside joke that when said aloud, it could sound
+like punji, which is `a sharpened stick at the bottom of a
+trap <https://en.wikipedia.org/wiki/Punji_stick>`_. Kind of like software…

doc/comps.rst

@@ -0,0 +1,27 @@
+.. _comps:
+
+Processing comps files
+======================
+
+The comps file that Pungi takes as input is not really pure comps as used by
+tools like DNF. There are extensions used to customize how the file is processed.
+
+The first step of Pungi processing is to retrieve the actual file. This can use
+anything that :ref:`scm_support` supports.
+
+Pungi extensions are ``arch`` attribute on ``packageref``, ``group`` and
+``environment`` tags. The value of this attribute is a comma separated list of
+architectures.
+
+Second step Pungi performs is creating a file for each architecture. This is
+done by removing all elements with incompatible ``arch`` attribute. No
+additional clean up is performed on this file. The resulting file is only used
+internally for the rest of the compose process.
+
+Third and final step is to create comps file for each Variant.Arch combination.
+This is the actual file that will be included in the compose. The start file is
+the original input file, from which all elements with incompatible architecture
+are removed. Then clean up is performed by removing all empty groups, removing
+non-existing groups from environments and categories and finally removing empty
+environments and categories. As a last step groups not listed in the variants
+file are removed.

doc/conf.py

@@ -0,0 +1,258 @@
+# -*- coding: utf-8 -*-
+#
+# Pungi documentation build configuration file, created by
+# sphinx-quickstart on Thu Jul  2 08:11:04 2015.
+#
+# This file is execfile()d with the current directory set to its
+# containing dir.
+#
+# Note that not all possible configuration values are present in this
+# autogenerated file.
+#
+# All configuration values have a default; values that are commented out
+# serve to show the default.
+
+import sys
+import os
+
+# If extensions (or modules to document with autodoc) are in another directory,
+# add these directories to sys.path here. If the directory is relative to the
+# documentation root, use os.path.abspath to make it absolute, like shown here.
+# sys.path.insert(0, os.path.abspath('.'))
+
+# -- General configuration ------------------------------------------------
+
+# If your documentation needs a minimal Sphinx version, state it here.
+# needs_sphinx = '1.0'
+
+# Add any Sphinx extension module names here, as strings. They can be
+# extensions coming with Sphinx (named 'sphinx.ext.*') or your custom
+# ones.
+extensions = []
+
+# Add any paths that contain templates here, relative to this directory.
+templates_path = ["_templates"]
+
+# The suffix of source filenames.
+source_suffix = ".rst"
+
+# The encoding of source files.
+# source_encoding = 'utf-8-sig'
+
+# The master toctree document.
+master_doc = "index"
+
+# General information about the project.
+project = "Pungi"
+copyright = "2016, Red Hat, Inc."
+
+# The version info for the project you're documenting, acts as replacement for
+# |version| and |release|, also used in various other places throughout the
+# built documents.
+#
+# The short X.Y version.
+version = "4.7"
+# The full version, including alpha/beta/rc tags.
+release = "4.7.0"
+
+# The language for content autogenerated by Sphinx. Refer to documentation
+# for a list of supported languages.
+# language = None
+
+# There are two options for replacing |today|: either, you set today to some
+# non-false value, then it is used:
+# today = ''
+# Else, today_fmt is used as the format for a strftime call.
+# today_fmt = '%B %d, %Y'
+
+# List of patterns, relative to source directory, that match files and
+# directories to ignore when looking for source files.
+exclude_patterns = ["_build"]
+
+# The reST default role (used for this markup: `text`) to use for all
+# documents.
+# default_role = None
+
+# If true, '()' will be appended to :func: etc. cross-reference text.
+# add_function_parentheses = True
+
+# If true, the current module name will be prepended to all description
+# unit titles (such as .. function::).
+# add_module_names = True
+
+# If true, sectionauthor and moduleauthor directives will be shown in the
+# output. They are ignored by default.
+# show_authors = False
+
+# The name of the Pygments (syntax highlighting) style to use.
+pygments_style = "sphinx"
+
+# A list of ignored prefixes for module index sorting.
+# modindex_common_prefix = []
+
+# If true, keep warnings as "system message" paragraphs in the built documents.
+# keep_warnings = False
+
+
+# -- Options for HTML output ----------------------------------------------
+
+# The theme to use for HTML and HTML Help pages.  See the documentation for
+# a list of builtin themes.
+html_theme = "default"
+
+# Theme options are theme-specific and customize the look and feel of a theme
+# further.  For a list of options available for each theme, see the
+# documentation.
+# html_theme_options = {}
+
+# Add any paths that contain custom themes here, relative to this directory.
+# html_theme_path = []
+
+# The name for this set of Sphinx documents.  If None, it defaults to
+# "<project> v<release> documentation".
+# html_title = None
+
+# A shorter title for the navigation bar.  Default is the same as html_title.
+# html_short_title = None
+
+# The name of an image file (relative to this directory) to place at the top
+# of the sidebar.
+# html_logo = None
+
+# The name of an image file (within the static path) to use as favicon of the
+# docs.  This file should be a Windows icon file (.ico) being 16x16 or 32x32
+# pixels large.
+# html_favicon = None
+
+# Add any paths that contain custom static files (such as style sheets) here,
+# relative to this directory. They are copied after the builtin static files,
+# so a file named "default.css" will overwrite the builtin "default.css".
+html_static_path = ["_static"]
+
+# Add any extra paths that contain custom files (such as robots.txt or
+# .htaccess) here, relative to this directory. These files are copied
+# directly to the root of the documentation.
+# html_extra_path = []
+
+# If not '', a 'Last updated on:' timestamp is inserted at every page bottom,
+# using the given strftime format.
+# html_last_updated_fmt = '%b %d, %Y'
+
+# If true, SmartyPants will be used to convert quotes and dashes to
+# typographically correct entities.
+# html_use_smartypants = True
+
+# Custom sidebar templates, maps document names to template names.
+# html_sidebars = {}
+
+# Additional templates that should be rendered to pages, maps page names to
+# template names.
+# html_additional_pages = {}
+
+# If false, no module index is generated.
+# html_domain_indices = True
+
+# If false, no index is generated.
+# html_use_index = True
+
+# If true, the index is split into individual pages for each letter.
+# html_split_index = False
+
+# If true, links to the reST sources are added to the pages.
+# html_show_sourcelink = True
+
+# If true, "Created using Sphinx" is shown in the HTML footer. Default is True.
+# html_show_sphinx = True
+
+# If true, "(C) Copyright ..." is shown in the HTML footer. Default is True.
+# html_show_copyright = True
+
+# If true, an OpenSearch description file will be output, and all pages will
+# contain a <link> tag referring to it.  The value of this option must be the
+# base URL from which the finished HTML is served.
+# html_use_opensearch = ''
+
+# This is the file name suffix for HTML files (e.g. ".xhtml").
+# html_file_suffix = None
+
+# Output file base name for HTML help builder.
+htmlhelp_basename = "Pungidoc"
+
+
+# -- Options for LaTeX output ---------------------------------------------
+
+latex_elements = {
+    # The paper size ('letterpaper' or 'a4paper').
+    #'papersize': 'letterpaper',
+    # The font size ('10pt', '11pt' or '12pt').
+    #'pointsize': '10pt',
+    # Additional stuff for the LaTeX preamble.
+    #'preamble': '',
+}
+
+# Grouping the document tree into LaTeX files. List of tuples
+# (source start file, target name, title,
+#  author, documentclass [howto, manual, or own class]).
+latex_documents = [
+    ("index", "Pungi.tex", "Pungi Documentation", "Daniel Mach", "manual"),
+]
+
+# The name of an image file (relative to this directory) to place at the top of
+# the title page.
+# latex_logo = None
+
+# For "manual" documents, if this is true, then toplevel headings are parts,
+# not chapters.
+# latex_use_parts = False
+
+# If true, show page references after internal links.
+# latex_show_pagerefs = False
+
+# If true, show URL addresses after external links.
+# latex_show_urls = False
+
+# Documents to append as an appendix to all manuals.
+# latex_appendices = []
+
+# If false, no module index is generated.
+# latex_domain_indices = True
+
+
+# -- Options for manual page output ---------------------------------------
+
+# One entry per manual page. List of tuples
+# (source start file, name, description, authors, manual section).
+man_pages = [("index", "pungi", "Pungi Documentation", ["Daniel Mach"], 1)]
+
+# If true, show URL addresses after external links.
+# man_show_urls = False
+
+
+# -- Options for Texinfo output -------------------------------------------
+
+# Grouping the document tree into Texinfo files. List of tuples
+# (source start file, target name, title, author,
+#  dir menu entry, description, category)
+texinfo_documents = [
+    (
+        "index",
+        "Pungi",
+        "Pungi Documentation",
+        "Daniel Mach",
+        "Pungi",
+        "One line description of project.",
+        "Miscellaneous",
+    ),
+]
+
+# Documents to append as an appendix to all manuals.
+# texinfo_appendices = []
+
+# If false, no module index is generated.
+# texinfo_domain_indices = True
+
+# How to display URL addresses: 'footnote', 'no', or 'inline'.
+# texinfo_show_urls = 'footnote'
+
+# If true, do not generate a @detailmenu in the "Top" node's menu.
+# texinfo_no_detailmenu = False

doc/contributing.rst

@@ -0,0 +1,166 @@
+=====================
+Contributing to Pungi
+=====================
+
+
+Set up development environment
+==============================
+
+In order to work on *Pungi*, you should install recent version of *Fedora*.
+
+Python2
+-------
+
+Fedora 29 is recommended because some packages are not available in newer Fedora release, e.g. python2-libcomps.
+
+Install required packages ::
+
+    $ sudo dnf install -y krb5-devel gcc make libcurl-devel python2-devel python2-createrepo_c kobo-rpmlib yum python2-libcomps python2-libselinx
+
+Python3
+-------
+
+Install required packages ::
+
+    $ sudo dnf install -y krb5-devel gcc make libcurl-devel python3-devel python3-createrepo_c python3-libcomps
+
+Developing
+==========
+
+Currently the development workflow for Pungi is on master branch:
+
+- Make your own fork at https://pagure.io/pungi
+- Clone your fork locally (replacing $USERNAME with your own)::
+
+    git clone git@pagure.io:forks/$USERNAME/pungi.git
+
+- cd into your local clone and add the remote upstream for rebasing::
+
+    cd pungi
+    git remote add upstream git@pagure.io:pungi.git
+
+  .. note::
+      This workflow assumes that you never ``git commit`` directly to the master
+      branch of your fork. This will make more sense when we cover rebasing
+      below.
+
+- create a topic branch based on master::
+
+    git branch my_topic_branch master
+    git checkout my_topic_branch
+
+
+- Make edits, changes, add new features, etc. and then make sure to pull
+  from upstream master and rebase before submitting a pull request::
+
+    # lets just say you edited setup.py for sake of argument
+    git checkout my_topic_branch
+
+    # make changes to setup.py
+    black setup.py
+    tox
+    git add setup.py
+    git commit -s -m "added awesome feature to setup.py"
+
+    # now we rebase
+    git checkout master
+    git pull --rebase upstream master
+    git push origin master
+    git push origin --tags
+    git checkout my_topic_branch
+    git rebase master
+
+    # resolve merge conflicts if any as a result of your development in
+    # your topic branch
+    git push origin my_topic_branch
+
+  .. note::
+      In order to for your commit to be merged:
+
+      - you must sign-off on it. Use ``-s`` option when running ``git commit``.
+
+      - The code must be well formatted via ``black`` and pass ``flake8`` checking. Run ``tox -e black,flake8`` to do the check.
+
+- Create pull request in the pagure.io web UI
+
+- For convenience, here is a bash shell function that can be placed in your
+  ~/.bashrc and called such as ``pullupstream pungi-4-devel`` that will
+  automate a large portion of the rebase steps from above::
+
+    pullupstream () {
+      if [[ -z "$1" ]]; then
+        printf "Error: must specify a branch name (e.g. - master, devel)\n"
+      else
+        pullup_startbranch=$(git describe --contains --all HEAD)
+        git checkout $1
+        git pull --rebase upstream master
+        git push origin $1
+        git push origin --tags
+        git checkout ${pullup_startbranch}
+      fi
+    }
+
+
+Testing
+=======
+
+You must write unit tests for any new code (except for trivial changes). Any
+code without sufficient test coverage may not be merged.
+
+To run all existing tests, suggested method is to use *tox*. ::
+
+    $ sudo dnf install python3-tox -y
+
+    $ tox -e py3
+    $ tox -e py27
+
+Alternatively you could create a vitualenv, install deps and run tests
+manually if you don't want to use tox. ::
+
+    $ sudo dnf install python3-virtualenvwrapper -y
+    $ mkvirtualenv --system-site-packages py3
+    $ workon py3
+    $ pip install -r requirements.txt -r test-requirements.txt
+    $ make test
+
+    # or with coverage
+    $ make test-coverage
+
+If you need to run specified tests, *pytest* is recommended. ::
+
+    # Activate virtualenv first
+
+    # Run tests
+    $ pytest tests/test_config.py
+    $ pytest tests/test_config.py -k test_pkgset_mismatch_repos
+
+In the ``tests/`` directory there is a shell script ``test_compose.sh`` that
+you can use to try and create a miniature compose on dummy data. The actual
+data will be created by running ``make test-data`` in project root. ::
+
+    $ sudo dnf -y install rpm-build createrepo_c isomd5sum genisoimage syslinux
+
+    # Activate virtualenv (the one created by tox could be used)
+    $ source .tox/py3/bin/activate
+
+    $ python setup.py develop
+    $ make test-data
+    $ make test-compose
+
+This testing compose does not actually use all phases that are available, and
+there is no checking that the result is correct. It only tells you whether it
+crashed or not.
+
+.. note::
+   Even when it finishes successfully, it may print errors about
+   ``repoclosure`` on *Server-Gluster.x86_64* in *test* phase. This is not a
+   bug.
+
+
+Documenting
+===========
+
+You must write documentation for any new features and functional changes.
+Any code without sufficient documentation may not be merged.
+
+To generate the documentation, run ``make doc`` in project root.

doc/examples.rst

@@ -0,0 +1,480 @@
+.. _examples:
+
+Big picture examples
+====================
+
+Actual Pungi configuration files can get very large. This pages brings two
+examples of (almost) full configuration for two different composes.
+
+Fedora Rawhide compose
+----------------------
+
+This is a shortened configuration for Fedora Radhide compose as of 2019-10-14.
+
+::
+
+    release_name = 'Fedora'
+    release_short = 'Fedora'
+    release_version = 'Rawhide'
+    release_is_layered = False
+
+    bootable = True
+    comps_file = {
+        'scm': 'git',
+        'repo': 'https://pagure.io/fedora-comps.git',
+        'branch': 'master',
+        'file': 'comps-rawhide.xml',
+        # Merge translations by running make. This command will generate the file.
+        'command': 'make comps-rawhide.xml'
+    }
+    module_defaults_dir = {
+        'scm': 'git',
+        'repo': 'https://pagure.io/releng/fedora-module-defaults.git',
+        'branch': 'main',
+        'dir': '.'
+    }
+    # Optional module obsoletes configuration which is merged
+    # into the module index and gets resolved
+    module_obsoletes_dir = {
+        'scm': 'git',
+        'repo': 'https://pagure.io/releng/fedora-module-defaults.git',
+        'branch': 'main',
+        'dir': 'obsoletes'
+    }
+
+    variants_file='variants-fedora.xml'
+    sigkeys = ['12C944D0']
+
+    # Put packages into subdirectories hashed by their initial letter.
+    hashed_directories = True
+
+    # There is a special profile for use with compose. It makes Pungi
+    # authenticate automatically as rel-eng user.
+    koji_profile = 'compose_koji'
+
+    # RUNROOT settings
+    runroot = True
+    runroot_channel = 'compose'
+    runroot_tag = 'f32-build'
+
+    # PKGSET
+    pkgset_source = 'koji'
+    pkgset_koji_tag = 'f32'
+    pkgset_koji_inherit = False
+
+    filter_system_release_packages = False
+
+    # GATHER
+    gather_method = {
+        '^.*': {                # For all variants
+            'comps': 'deps',    # resolve dependencies for packages from comps file
+            'module': 'nodeps', # but not for packages from modules
+        }
+    }
+    gather_backend = 'dnf'
+    gather_profiler = True
+    check_deps = False
+    greedy_method = 'build'
+
+    repoclosure_backend = 'dnf'
+
+    # CREATEREPO
+    createrepo_deltas = False
+    createrepo_database = True
+    createrepo_use_xz = True
+    createrepo_extra_args = ['--zck', '--zck-dict-dir=/usr/share/fedora-repo-zdicts/rawhide']
+
+    # CHECKSUMS
+    media_checksums = ['sha256']
+    media_checksum_one_file = True
+    media_checksum_base_filename = '%(release_short)s-%(variant)s-%(version)s-%(arch)s-%(date)s%(type_suffix)s.%(respin)s'
+
+    # CREATEISO
+    iso_hfs_ppc64le_compatible = False
+
+    # BUILDINSTALL
+    buildinstall_method = 'lorax'
+    buildinstall_skip = [
+        # No installer for Modular variant
+        ('^Modular$', {'*': True}),
+        # No 32 bit installer for Everything.
+        ('^Everything$', {'i386': True}),
+    ]
+
+    # Enables macboot on x86_64 for all variants and disables upgrade image building
+    # everywhere.
+    lorax_options = [
+      ('^.*$', {
+         'x86_64': {
+             'nomacboot': False
+         },
+         'ppc64le': {
+             # Use 3GB image size for ppc64le.
+             'rootfs_size': 3
+         },
+         '*': {
+             'noupgrade': True
+         }
+      })
+    ]
+
+    additional_packages = [
+        ('^(Server|Everything)$', {
+            '*': [
+                # Add all architectures of dracut package.
+                'dracut.*',
+                # All all packages matching this pattern
+                'autocorr-*',
+            ],
+        }),
+
+        ('^Everything$', {
+            # Everything should include all packages from the tag. This only
+            # applies to the native arch. Multilib will still be pulled in
+            # according to multilib rules.
+            '*': ['*'],
+        }),
+    ]
+
+    filter_packages = [
+        ("^.*$", {"*": ["glibc32", "libgcc32"]}),
+        ('(Server)$', {
+            '*': [
+                'kernel*debug*',
+                'kernel-kdump*',
+            ]
+        }),
+    ]
+
+    multilib = [
+        ('^Everything$', {
+            'x86_64': ['devel', 'runtime'],
+        })
+    ]
+
+    # These packages should never be multilib on any arch.
+    multilib_blacklist = {
+        '*': [
+            'kernel', 'kernel-PAE*', 'kernel*debug*', 'java-*', 'php*', 'mod_*', 'ghc-*'
+        ],
+    }
+
+    # These should be multilib even if they don't match the rules defined above.
+    multilib_whitelist = {
+        '*': ['wine', '*-static'],
+    }
+
+    createiso_skip = [
+        # Keep binary ISOs for Server, but not source ones.
+        ('^Server$', {'src': True}),
+
+        # Remove all other ISOs.
+        ('^Everything$', {'*': True, 'src': True}),
+        ('^Modular$', {'*': True, 'src': True}),
+    ]
+
+    # Image name respecting Fedora's image naming policy
+    image_name_format = '%(release_short)s-%(variant)s-%(disc_type)s-%(arch)s-%(version)s-%(date)s%(type_suffix)s.%(respin)s.iso'
+    # Use the same format for volume id
+    image_volid_formats = [
+        '%(release_short)s-%(variant)s-%(disc_type)s-%(arch)s-%(version)s'
+    ]
+    # Used by Pungi to replace 'Cloud' with 'C' (etc.) in ISO volume IDs.
+    # There is a hard 32-character limit on ISO volume IDs, so we use
+    # these to try and produce short enough but legible IDs. Note this is
+    # duplicated in Koji for live images, as livemedia-creator does not
+    # allow Pungi to tell it what volume ID to use. Note:
+    # https://fedoraproject.org/wiki/User:Adamwill/Draft_fedora_image_naming_policy
+    volume_id_substitutions = {
+                     'Beta': 'B',
+                  'Rawhide': 'rawh',
+               'Silverblue': 'SB',
+                 'Cinnamon': 'Cinn',
+                    'Cloud': 'C',
+             'Design_suite': 'Dsgn',
+           'Electronic_Lab': 'Elec',
+               'Everything': 'E',
+           'Scientific_KDE': 'SciK',
+                 'Security': 'Sec',
+                   'Server': 'S',
+              'Workstation': 'WS',
+    }
+
+    disc_types = {
+        'boot': 'netinst',
+        'live': 'Live',
+    }
+
+    translate_paths = [
+       ('/mnt/koji/compose/', 'https://kojipkgs.fedoraproject.org/compose/'),
+    ]
+
+    # These will be inherited by live_media, live_images and image_build
+    global_ksurl = 'git+https://pagure.io/fedora-kickstarts.git?#HEAD'
+    global_release = '!RELEASE_FROM_LABEL_DATE_TYPE_RESPIN'
+    global_version = 'Rawhide'
+    # live_images ignores this in favor of live_target
+    global_target = 'f32'
+
+    image_build = {
+        '^Container$': [
+            {
+                'image-build': {
+                        'format': [('docker', 'tar.xz')],
+                        'name': 'Fedora-Container-Base',
+                        'kickstart': 'fedora-container-base.ks',
+                        'distro': 'Fedora-22',
+                        'disk_size': 5,
+                        'arches': ['armhfp', 'aarch64', 'ppc64le', 's390x', 'x86_64'],
+                        'repo': 'Everything',
+                        'install_tree_from': 'Everything',
+                        'subvariant': 'Container_Base',
+                        'failable': ['*'],
+                        },
+                'factory-parameters': {
+                    'dockerversion': "1.10.1",
+                    'docker_cmd':  '[ "/bin/bash" ]',
+                    'docker_env': '[ "DISTTAG=f32container", "FGC=f32", "container=oci" ]',
+                    'docker_label': '{ "name": "fedora", "license": "MIT", "vendor": "Fedora Project", "version": "32"}',
+                },
+            },
+        ],
+    }
+
+    live_media = {
+        '^Workstation$': [
+                {
+                    'name': 'Fedora-Workstation-Live',
+                    'kickstart': 'fedora-live-workstation.ks',
+                    # Variants.xml also contains aarch64 and armhfp, but there
+                    # should be no live media for those arches.
+                    'arches': ['x86_64', 'ppc64le'],
+                    'failable': ['ppc64le'],
+                    # Take packages and install tree from Everything repo.
+                    'repo': 'Everything',
+                    'install_tree_from': 'Everything',
+                }
+            ],
+        '^Spins': [
+            # There are multiple media for Spins variant. They use subvariant
+            # field so that they can be identified in the metadata.
+            {
+                'name': 'Fedora-KDE-Live',
+                'kickstart': 'fedora-live-kde.ks',
+                'arches': ['x86_64'],
+                'repo': 'Everything',
+                'install_tree_from': 'Everything',
+                'subvariant': 'KDE'
+
+            },
+            {
+                'name': 'Fedora-Xfce-Live',
+                'kickstart': 'fedora-live-xfce.ks',
+                'arches': ['x86_64'],
+                'failable': ['*'],
+                'repo': 'Everything',
+                'install_tree_from': 'Everything',
+                'subvariant': 'Xfce'
+            },
+        ],
+    }
+
+    failable_deliverables = [
+        # Installer and ISOs for server failing do not abort the compose.
+        ('^Server$', { 
+            '*': ['buildinstall', 'iso'],
+        }),
+        ('^.*$', {
+            # Buildinstall is not blocking
+            'src': ['buildinstall'],
+            # Nothing on i386, ppc64le blocks the compose
+            'i386': ['buildinstall', 'iso'],
+            'ppc64le': ['buildinstall', 'iso'],
+            's390x': ['buildinstall', 'iso'],
+        })
+    ]
+
+    ostree = {
+        "^Silverblue$": {
+            "version": "!OSTREE_VERSION_FROM_LABEL_DATE_TYPE_RESPIN",
+            # To get config, clone master branch from this repo and take
+            # treefile from there.
+            "treefile": "fedora-silverblue.yaml",
+            "config_url": "https://pagure.io/workstation-ostree-config.git",
+            "config_branch": "master",
+            # Consume packages from Everything
+            "repo": "Everything",
+            # Don't create a reference in the ostree repo (signing automation does that).
+            "tag_ref": False,
+            # Don't use change detection in ostree.
+            "force_new_commit": True,
+            # Use unified core mode for rpm-ostree composes
+            "unified_core": True,
+            # This is the location for the repo where new commit will be
+            # created. Note that this is outside of the compose dir.
+            "ostree_repo": "/mnt/koji/compose/ostree/repo/",
+            "ostree_ref": "fedora/rawhide/${basearch}/silverblue",
+            "arches": ["x86_64", "ppc64le", "aarch64"],
+            "failable": ['*'],
+        }
+    }
+
+    ostree_container = {
+        "^Sagano$": {
+            "treefile": "fedora-tier-0-38.yaml",
+            "config_url": "https://gitlab.com/CentOS/cloud/sagano.git",
+            "config_branch": "main",
+            # Consume packages from Everything
+            "repo": "Everything",
+            # Automatically generate a reasonable version
+            "version": "!OSTREE_VERSION_FROM_LABEL_DATE_TYPE_RESPIN",
+            # Only run this for x86_64 even if Sagano has more arches
+            "arches": ["x86_64"],
+        }
+    }
+
+    ostree_installer = [
+        ("^Silverblue$", {
+            "x86_64": {
+                "repo": "Everything",
+                "release": None,
+                "rootfs_size": "8",
+                # Take templates from this repository.
+                'template_repo': 'https://pagure.io/fedora-lorax-templates.git',
+                'template_branch': 'master',
+                # Use following templates.
+                "add_template": ["ostree-based-installer/lorax-configure-repo.tmpl",
+                                 "ostree-based-installer/lorax-embed-repo.tmpl",
+                                 "ostree-based-installer/lorax-embed-flatpaks.tmpl"],
+                # And add these variables for the templates.
+                "add_template_var": [
+                    "ostree_install_repo=https://kojipkgs.fedoraproject.org/compose/ostree/repo/",
+                    "ostree_update_repo=https://ostree.fedoraproject.org",
+                    "ostree_osname=fedora",
+                    "ostree_oskey=fedora-32-primary",
+                    "ostree_contenturl=mirrorlist=https://ostree.fedoraproject.org/mirrorlist",
+                    "ostree_install_ref=fedora/rawhide/x86_64/silverblue",
+                    "ostree_update_ref=fedora/rawhide/x86_64/silverblue",
+                    "flatpak_remote_name=fedora",
+                    "flatpak_remote_url=oci+https://registry.fedoraproject.org",
+                    "flatpak_remote_refs=runtime/org.fedoraproject.Platform/x86_64/f30 app/org.gnome.Baobab/x86_64/stable",
+                ],
+                'failable': ['*'],
+            },
+        })
+    ]
+
+
+RCM Tools compose
+-----------------
+
+This is a small compose used to deliver packages to Red Hat internal users. The
+configuration is split into two files.
+
+::
+
+    # rcmtools-common.conf 
+
+    release_name = "RCM Tools"
+    release_short = "RCMTOOLS"
+    release_version = "2.0"
+    release_type = "updates"
+    release_is_layered = True
+    createrepo_c = True
+    createrepo_checksum = "sha256"
+
+    # PKGSET
+    pkgset_source = "koji"
+    koji_profile = "brew"
+    pkgset_koji_inherit = True
+
+
+    # GENERAL SETTINGS
+    bootable = False
+    comps_file = "rcmtools-comps.xml"
+    variants_file = "rcmtools-variants.xml"
+    sigkeys = ["3A3A33A3"]
+
+
+    # RUNROOT settings
+    runroot = False
+
+
+    # GATHER
+    gather_method = "deps"
+    check_deps = True
+
+    additional_packages = [
+        ('.*', {
+            '*': ['puddle', 'rcm-nexus'],
+            }
+        ),
+    ]
+
+    # Set repoclosure_strictness to fatal to avoid installation dependency
+    # issues in production composes
+    repoclosure_strictness = [
+        ("^.*$", {
+            "*": "fatal"
+        })
+    ]
+
+
+Configuration specific for different base products is split into separate files.
+
+::
+
+    # rcmtools-common.conf 
+    from rcmtools-common import *
+
+    # BASE PRODUCT
+    base_product_name = "Red Hat Enterprise Linux"
+    base_product_short = "RHEL"
+    base_product_version = "7"
+
+    # PKGSET
+    pkgset_koji_tag = "rcmtools-rhel-7-compose"
+
+    # remove i386 arch on rhel7
+    tree_arches = ["aarch64", "ppc64le", "s390x", "x86_64"]
+
+    check_deps = False
+
+    # Packages in these repos are available to satisfy dependencies inside the
+    # compose, but will not be pulled in.
+    gather_lookaside_repos = [
+        ("^Client|Client-optional$", {
+            "x86_64": [
+                "http://example.redhat.com/rhel/7/Client/x86_64/os/",
+                "http://example.redhat.com/rhel/7/Client/x86_64/optional/os/",
+            ],
+        }),
+         ("^Workstation|Workstation-optional$", {
+            "x86_64": [
+                "http://example.redhat.com/rhel/7/Workstation/x86_64/os/",
+                "http://example.redhat.com/rhel/7/Workstation/x86_64/optional/os/",
+            ],
+        }),
+        ("^Server|Server-optional$", {
+            "aarch64": [
+                "http://example.redhat.com/rhel/7/Server/aarch64/os/",
+                "http://example.redhat.com/rhel/7/Server/aarch64/optional/os/",
+            ],
+            "ppc64": [
+                "http://example.redhat.com/rhel/7/Server/ppc64/os/",
+                "http://example.redhat.com/rhel/7/Server/ppc64/optional/os/",
+            ],
+            "ppc64le": [
+                "http://example.redhat.com/rhel/7/Server/ppc64le/os/",
+                "http://example.redhat.com/rhel/7/Server/ppc64le/optional/os/",
+            ],
+            "s390x": [
+                "http://example.redhat.com/rhel/7/Server/s390x/os/",
+                "http://example.redhat.com/rhel/7/Server/s390x/optional/os/",
+            ],
+            "x86_64": [
+                "http://example.redhat.com/rhel/7/Server/x86_64/os/",
+                "http://example.redhat.com/rhel/7/Server/x86_64/optional/os/",
+            ],
+        })
+    ]

doc/format.rst

@@ -0,0 +1,90 @@
+==================
+Config file format
+==================
+
+The configuration file parser is provided by `kobo
+<https://github.com/release-engineering/kobo>`_
+
+The file follows a Python-like format. It consists of a sequence of variables
+that have a value assigned to them. ::
+
+    variable = value
+
+The variable names must follow the same convention as Python code: start with a
+letter and consist of letters, digits and underscores only.
+
+The values can be either an integer, float, boolean (``True`` or ``False``), a
+string or ``None``. Strings must be enclosed in either single or double quotes.
+
+Complex types are supported as well.
+
+A list is enclosed in square brackets and items are separated with commas.
+There can be a comma after the last item as well. ::
+
+   a_list = [1,
+             2,
+             3,
+            ]
+
+A tuple works like a list, but is enclosed in parenthesis. ::
+
+    a_tuple = (1, "one")
+
+A dictionary is wrapped in brackets, and consists of ``key: value`` pairs
+separated by commas. The keys can only be formed from basic types (int, float,
+string). ::
+
+    a_dict = {
+        'foo': 'bar',
+        1: None
+    }
+
+The value assigned to a variable can also be taken from another variable. ::
+
+    one = 1
+    another = one
+
+Anything on a line after a ``#`` symbol is ignored and functions as a comment.
+
+
+Importing other files
+=====================
+
+It is possible to include another configuration file. The files are looked up
+relative to the currently processed file.
+
+The general structure of import is: ::
+
+    from FILENAME import WHAT
+
+The ``FILENAME`` should be just the base name of the file without extension
+(which must be ``.conf``). ``WHAT`` can either be a comma separated list of
+variables or ``*``. ::
+
+    # Opens constants.conf and brings PI and E into current scope.
+    from constants import PI, E
+
+    # Opens common.conf and brings everything defined in that file into current
+    # file as well.
+    from common import *
+
+.. note::
+    Pungi will copy the configuration file given on command line into the
+    ``logs/`` directory. Only this single file will be copied, not any included
+    ones. (Copying included files requires a fix in kobo library.)
+
+    The JSON-formatted dump of configuration is correct though.
+
+Formatting strings
+==================
+
+String interpolation is available as well. It uses a ``%``-encoded format. See
+Python documentation for more details. ::
+
+    joined = "%s %s" % (var_a, var_b)
+
+    a_dict = {
+        "fst": 1,
+        "snd": 2,
+    }
+    another = "%(fst)s %(snd)s" % a_dict

doc/gathering.rst

@@ -0,0 +1,102 @@
+==================
+Gathering packages
+==================
+
+A compose created by Pungi consists of one or more variants. A variant contains
+a subset of the content targeted at a particular use case.
+
+There are different types of variants. The type affects how packages are
+gathered into the variant.
+
+The inputs for gathering are defined by various gather sources. Packages from
+all sources are collected to create a big list of package names, comps groups
+names and a list of packages that should be filtered out.
+
+.. note::
+   The inputs for both explicit package list and comps file are interpreted as
+   RPM names, not any arbitrary provides nor source package name.
+
+Next, ``gather_method`` defines how the list is processed. For ``nodeps``, the
+results from source are used pretty much as is [#]_. For ``deps`` method, a
+process will be launched to figure out what dependencies are needed and those
+will be pulled in.
+
+.. [#] The lists are filtered based on what packages are available in the
+   package set, but nothing else will be pulled in.
+
+
+Variant types
+=============
+
+*Variant*
+    is a base type that has no special behaviour.
+
+*Addon*
+    is built on top of a regular variant. Any packages that should go to both
+    the addon and its parent will be removed from addon. Packages that are only
+    in addon but pulled in because of ``gather_fulltree`` option will be moved
+    to parent.
+
+*Integrated Layered Product*
+    works similarly to *addon*. Additionally, all packages from addons on the
+    same parent variant are removed integrated layered products.
+
+    The main difference between an *addon* and *integrated layered product* is
+    that *integrated layered product* has its own identity in the metadata
+    (defined with product name and version).
+
+    .. note::
+        There's also *Layered Product* as a term, but this is not related to
+        variants. It's used to describe a product that is not a standalone
+        operating system and is instead meant to be used on some other base
+        system.
+
+*Optional*
+    contains packages that complete the base variants' package set. It always
+    has ``fulltree`` and ``selfhosting`` enabled, so it contains build
+    dependencies and packages which were not specifically requested for base
+    variant.
+
+
+Some configuration options are overridden for particular variant types.
+
+.. table:: Depsolving configuration
+
+   +-----------+--------------+--------------+
+   | Variant   | Fulltree     | Selfhosting  |
+   +===========+==============+==============+
+   | base      | configurable | configurable |
+   +-----------+--------------+--------------+
+   | addon/ILP | enabled      | disabled     |
+   +-----------+--------------+--------------+
+   | optional  | enabled      | enabled      |
+   +-----------+--------------+--------------+
+
+
+Profiling
+=========
+
+Profiling data on the ``pungi-gather`` tool can be enabled by setting the
+``gather_profiler`` configuration option to ``True``.
+
+
+Modular compose
+===============
+
+A compose with ``gather_source`` set to ``module`` is called *modular*. The
+package list is determined by a list of modules.
+
+The list of modules that will be put into a variant is defined in the
+``variants.xml`` file. The file can contain either *Name:Stream* or
+*Name:Stream:Version* references. See `Module Naming Policy
+<https://pagure.io/modularity/blob/master/f/source/development/building-modules/naming-policy.rst>`_
+for details. When *Version* is missing from the specification, Pungi will ask
+PDC for the latest one.
+
+The module metadata in PDC contains a list of RPMs in the module as well as
+Koji tag from which the packages can be retrieved.
+
+Restrictions
+------------
+
+ * A modular compose must always use Koji as a package set source.

doc/index.rst

@@ -0,0 +1,25 @@
+.. Pungi documentation master file, created by
+   sphinx-quickstart on Thu Jul  2 08:11:04 2015.
+   You can adapt this file completely to your liking, but it should at least
+   contain the root `toctree` directive.
+
+Welcome to Pungi's documentation!
+=================================
+
+Contents:
+
+.. toctree::
+    :maxdepth: 2
+
+    about
+    phases
+    format
+    configuration
+    examples
+    scm_support
+    messaging
+    gathering
+    koji
+    comps
+    contributing
+    testing

doc/koji.rst

@@ -0,0 +1,105 @@
+======================
+Getting data from koji
+======================
+
+When Pungi is configured to get packages from a Koji tag, it somehow needs to
+access the actual RPM files.
+
+Historically, this required the storage used by Koji to be directly available
+on the host where Pungi was running. This was usually achieved by using NFS for
+the Koji volume, and mounting it on the compose host.
+
+The compose could be created directly on the same volume. In such case the
+packages would be hardlinked, significantly reducing space consumption.
+
+The compose could also be created on a different storage, in which case the
+packages would either need to be copied over or symlinked. Using symlinks
+requires that anything that accesses the compose (e.g. a download server) would
+also need to mount the Koji volume in the same location.
+
+There is also a risk with symlinks that the package in Koji can change (due to
+being resigned for example), which would invalidate composes linking to it.
+
+
+Using Koji without direct mount
+===============================
+
+It is possible now to run a compose from a Koji tag without direct access to
+Koji storage.
+
+Pungi can download the packages over HTTP protocol, store them in a local
+cache, and consume them from there.
+
+The local cache has similar structure to what is on the Koji volume.
+
+When Pungi needs some package, it has a path on Koji volume. It will replace
+the ``topdir`` with the cache location. If such file exists, it will be used.
+If it doesn't exist, it will be downloaded from Koji (by replacing the
+``topdir`` with ``topurl``).
+
+::
+
+    Koji path                            /mnt/koji/packages/foo/1/1.fc38/data/signed/abcdef/noarch/foo-1-1.fc38.noarch.rpm
+    Koji URL    https://kojipkgs.fedoraproject.org/packages/foo/1/1.fc38/data/signed/abcdef/noarch/foo-1-1.fc38.noarch.rpm
+    Local path                  /mnt/compose/cache/packages/foo/1/1.fc38/data/signed/abcdef/noarch/foo-1-1.fc38.noarch.rpm
+
+The packages can be hardlinked from this cache directory.
+
+
+Cleanup
+-------
+
+While the approach above allows each RPM to be downloaded only once, it will
+eventually result in the Koji volume being mirrored locally. Most of the
+packages will however no longer be needed.
+
+There is a script ``pungi-cache-cleanup`` that can help with that. It can find
+and remove files from the cache that are no longer needed.
+
+A file is no longer needed if it has a single link (meaning it is only in the
+cache, not in any compose), and it has mtime older than a given threshold.
+
+It doesn't make sense to delete files that are hardlinked in an existing
+compose as it would not save any space anyway.
+
+The mtime check is meant to preserve files that are downloaded but not actually
+used in a compose, like a subpackage that is not included in any variant. Every
+time its existence in the local cache is checked, the mtime is updated.
+
+
+Race conditions?
+----------------
+
+It should be safe to have multiple compose hosts share the same storage volume
+for generated composes and local cache.
+
+If a cache file is accessed and it exists, there's no risk of race condition.
+
+If two composes need the same file at the same time and it is not present yet,
+one of them will take a lock on it and start downloading. The other will wait
+until the download is finished.
+
+The lock is only valid for a set amount of time (5 minutes) to avoid issues
+where the downloading process is killed in a way that blocks it from releasing
+the lock.
+
+If the file is large and network slow, the limit may not be enough finish
+downloading. In that case the second process will steal the lock while the
+first process is still downloading. This will result in the same file being
+downloaded twice.
+
+When the first process finishes the download, it will put the file into the
+local cache location. When the second process finishes, it will atomically
+replace it, but since it's the same file it will be the same file.
+
+If the first compose already managed to hardlink the file before it gets
+replaced, there will be two copies of the file present locally.
+
+
+Integrity checking
+------------------
+
+There is minimal integrity checking. RPM packages belonging to real builds will
+be check to match the checksum provided by Koji hub.
+
+There is no checking for scratch builds or any images.

doc/messaging.rst

@@ -0,0 +1,45 @@
+.. _messaging:
+
+Progress notification
+=====================
+
+*Pungi* has the ability to emit notification messages about progress and
+general status of the compose. These can be used to e.g. send messages to
+*fedmsg*. This is implemented by actually calling a separate script.
+
+The script will be called with one argument describing action that just
+happened. A JSON-encoded object will be passed to standard input to provide
+more information about the event. At the very least, the object will contain a
+``compose_id`` key.
+
+The notification script inherits working directory from the parent process and it
+can be called from the same directory ``pungi-koji`` is called from. The working directory
+is listed at the start of main log.
+
+Currently these messages are sent:
+
+ * ``status-change`` -- when composing starts, finishes or fails; a ``status``
+   key is provided to indicate details
+ * ``phase-start`` -- on start of a phase
+ * ``phase-stop`` -- when phase is finished
+ * ``createiso-targets`` -- with a list of images to be created
+ * ``createiso-imagedone`` -- when any single image is finished
+ * ``createiso-imagefail`` -- when any single image fails to create
+ * ``fail-to-start`` -- when there are incorrect CLI options or errors in
+   configuration file; this message does not contain ``compose_id`` nor is it
+   started in the compose directory (which does not exist yet)
+ * ``ostree`` -- when a new commit is created, this message will announce its
+   hash and the name of ref it is meant for.
+
+For phase related messages ``phase_name`` key is provided as well.
+
+A ``pungi-fedmsg-notification`` script is provided and understands this
+interface.
+
+Setting it up
+-------------
+
+The script should be provided as a command line argument
+``--notification-script``. ::
+
+    --notification-script=pungi-fedmsg-notification

doc/phases.rst

@@ -0,0 +1,175 @@
+.. _phases:
+
+Phases
+======
+
+Each invocation of ``pungi-koji`` consists of a set of phases.
+
+.. image:: _static/phases.svg
+   :alt: phase diagram
+
+Most of the phases run sequentially (left-to-right in the diagram), but there
+are use cases where multiple phases run in parallel. This happens for phases
+whose main point is to wait for a Koji task to finish.
+
+Init
+----
+
+The first phase to ever run. Can not be skipped. It prepares the comps files
+for variants (by filtering out groups and packages that should not be there).
+See :doc:`comps` for details about how this is done.
+
+Pkgset
+------
+
+This phase loads a set of packages that should be composed. It has two separate
+results: it prepares repos with packages in ``work/`` directory (one per arch)
+for further processing, and it returns a data structure with mapping of
+packages to architectures.
+
+Buildinstall
+------------
+
+Spawns a bunch of threads, each of which runs the ``lorax`` command. The
+commands create ``boot.iso`` and other boot configuration files. The image is
+finally linked into the ``compose/`` directory as netinstall media.
+
+The created images are also needed for creating live media or other images in
+later phases.
+
+With ``lorax`` this phase runs one task per variant.arch combination.
+
+Gather
+------
+
+This phase uses data collected by ``pkgset`` phase and figures out what
+packages should be in each variant. The basic mapping can come from comps file,
+a JSON mapping or ``additional_packages`` config option. This inputs can then
+be enriched by adding all dependencies. See :doc:`gathering` for details.
+
+Once the mapping is finalized, the packages are linked to appropriate places
+and the ``rpms.json`` manifest is created.
+
+ExtraFiles
+----------
+
+This phase collects extra files from the configuration and copies them to the
+compose directory. The files are described by a JSON file in the compose
+subtree where the files are copied. This metadata is meant to be distributed
+with the data (on ISO images).
+
+Createrepo
+----------
+
+This phase creates RPM repositories for each variant.arch tree. It is actually
+reading the ``rpms.json`` manifest to figure out which packages should be
+included.
+
+OSTree
+------
+
+Updates an ostree repository with a new commit with packages from the compose.
+The repository lives outside of the compose and is updated immediately. If the
+compose fails in a later stage, the commit will not be reverted.
+
+Implementation wise, this phase runs ``rpm-ostree`` command in Koji runroot (to
+allow running on different arches).
+
+Createiso
+---------
+
+Generates ISO files and accumulates enough metadata to be able to create
+``image.json`` manifest. The file is however not created in this phase, instead
+it is dumped in the ``pungi-koji`` script itself.
+
+The files include a repository with all RPMs from the variant. There will be
+multiple images if the packages do not fit on a single image.
+
+The image will be bootable if ``buildinstall`` phase is enabled and the
+packages fit on a single image.
+
+There can also be images with source repositories. These are never bootable.
+
+ExtraIsos
+---------
+
+This phase is very similar to ``createiso``, except it combines content from
+multiple variants onto a single image. Packages, repodata and extra files from
+each configured variant are put into a subdirectory. Additional extra files can
+be put into top level of the image. The image will be bootable if the main
+variant is bootable.
+
+LiveImages, LiveMedia
+---------------------
+
+Creates media in Koji with ``koji spin-livecd``, ``koji spin-appliance`` or
+``koji spin-livemedia`` command. When the media are finished, the images are
+copied into the ``compose/`` directory and metadata for images is updated.
+
+ImageBuild
+----------
+
+This phase wraps up ``koji image-build``. It also updates the metadata
+ultimately responsible for ``images.json`` manifest.
+
+KiwiBuild
+---------
+
+Similarly to image build, this phases creates a koji `kiwiBuild` task. In the
+background it uses Kiwi to create images.
+
+OSBuild
+-------
+
+Similarly to image build, this phases creates a koji `osbuild` task. In the
+background it uses OSBuild Composer to create images.
+
+OSBS
+----
+
+This phase builds container base images in `OSBS
+<http://osbs.readthedocs.io/en/latest/index.html>`_.
+
+The finished images are available in registry provided by OSBS, but not
+downloaded directly into the compose. The is metadata about the created image
+in ``compose/metadata/osbs.json``.
+
+ImageContainer
+--------------
+
+This phase builds a container image in OSBS, and stores the metadata in the
+same file as OSBS phase. The container produced here wraps a different image,
+created it ImageBuild or OSBuild phase. It can be useful to deliver a VM image
+to containerized environments.
+
+OSTreeInstaller
+---------------
+
+Creates bootable media that carry an ostree repository as a payload. These
+images are created by running ``lorax`` with special templates. Again it runs
+in Koji runroot.
+
+Repoclosure
+-----------
+
+Run ``repoclosure`` on each repository. By default errors are only reported
+in the log, the compose will still be considered a success. The actual error
+has to be looked up in the compose logs directory. Configuration allows customizing this.
+
+ImageChecksum
+-------------
+
+Responsible for generating checksums for the images. The checksums are stored
+in image manifest as well as files on disk. The list of images to be processed
+is obtained from the image manifest. This way all images will get the same
+checksums irrespective of the phase that created them.
+
+Test
+----
+
+This phase is supposed to run some sanity checks on the finished compose.
+
+The only test is to check all images listed the metadata and verify that they
+look sane. For ISO files headers are checked to verify the format is correct,
+and for bootable media a check is run to verify they have properties that allow
+booting.

doc/scm_support.rst

@@ -0,0 +1,100 @@
+.. _scm_support:
+
+Exporting files from SCM
+========================
+
+Multiple places in Pungi can use files from external storage. The configuration
+is similar independently of the backend that is used, although some features
+may be different.
+
+The so-called ``scm_dict`` is always put into configuration as a dictionary,
+which can contain following keys.
+
+* ``scm`` -- indicates which SCM system is used. This is always required.
+  Allowed values are:
+
+  * ``file`` -- copies files from local filesystem
+  * ``git`` -- copies files from a Git repository
+  * ``cvs`` -- copies files from a CVS repository
+  * ``rpm`` -- copies files from a package in the compose
+  * ``koji`` -- downloads archives from a given build in Koji build system
+
+* ``repo``
+
+  * for Git and CVS backends this should be URL to the repository
+  * for RPM backend this should be a shell style glob matching package names
+    (or a list of such globs)
+  * for file backend this should be empty
+  * for Koji backend this should be an NVR or package name
+
+* ``branch``
+
+  * branch name for Git and CVS backends, with ``master`` and ``HEAD`` as defaults
+  * Koji tag for koji backend if only package name is given
+  * otherwise should not be specified
+
+* ``file`` -- a list of files that should be exported.
+
+* ``dir`` -- a directory that should be exported. All its contents will be
+  exported. This option is mutually exclusive with ``file``.
+
+* ``command`` -- defines a shell command to run after Git clone to generate the
+  needed file (for example to run ``make``). Only supported in Git backend.
+
+* ``options`` -- a dictionary of additional configuration options. These are
+  specific to different backends.
+
+  Currently supported values for Git:
+
+  * ``credential_helper`` -- path to a credential helper used to supply
+    username/password for remotes that require authentication.
+
+
+Koji examples
+-------------
+
+There are two different ways how to configure the Koji backend. ::
+
+    {
+        # Download all *.tar files from build my-image-1.0-1.
+        "scm": "koji",
+        "repo": "my-image-1.0-1",
+        "file": "*.tar",
+    }
+
+    {
+        # Find latest build of my-image in tag my-tag and take files from
+        # there.
+        "scm": "koji",
+        "repo": "my-image",
+        "branch": "my-tag",
+        "file": "*.tar",
+    }
+
+Using both tag name and exact NVR will result in error: the NVR would be
+interpreted as a package name, and would not match anything.
+
+
+``file`` vs. ``dir``
+--------------------
+
+Exactly one of these two options has to be specified. Documentation for each
+configuration option should specify whether it expects a file or a directory.
+
+For ``extra_files`` phase either key is valid and should be chosen depending on
+what the actual use case.
+
+
+Caveats
+-------
+
+The ``rpm`` backend can only be used in phases that would extract the files
+after ``pkgset`` phase finished. You can't get comps file from a package.
+
+Depending on Git repository URL configuration Pungi can only export the
+requested content using ``git archive``. When a command should run this is not
+possible and a clone is always needed.
+
+When using ``koji`` backend, it is required to provide configuration for Koji
+profile to be used (``koji_profile``). It is not possible to contact multiple
+different Koji instances.

doc/testing.rst

@@ -0,0 +1,42 @@
+===============
+ Testing Pungi
+===============
+
+
+Test Data
+=========
+Tests require test data and not all of it is available in git.
+You must create test repositories before running the tests::
+
+    make test-data
+
+Requirements: createrepo_c, rpmbuild
+
+
+Unit Tests
+==========
+Unit tests cover functionality of Pungi python modules.
+You can run all of them at once::
+
+    make test
+
+which is shortcut to::
+
+    python2 setup.py test
+    python3 setup.py test
+
+You can alternatively run individual tests::
+
+    cd tests
+    ./<test>.py [<class>[.<test>]]
+
+
+Functional Tests
+================
+Because compose is quite complex process and not everything is covered with
+unit tests yet, the easiest way how to test if your changes did not break
+anything badly is to start a compose on a relatively small and well defined
+package set::
+
+    cd tests
+    ./test_compose.sh

doc/update-docs.sh

@@ -0,0 +1,40 @@
+#!/bin/bash
+
+# Copyright (C) 2015 Red Hat, Inc.
+# SPDX-License-Identifier:      GPL-2.0
+
+trap cleanup EXIT
+
+function cleanup() {
+    printf "Run cleanup\\n"
+    rm -rf  "$dir_pungi" "$dir_pungi_doc"
+}
+
+if [ -z "$1" ]; then
+    printf "Usage:\\n"
+    printf "\\t%s release_version\\n" "$0"
+    exit 1
+fi
+
+set -e
+dir_pungi=$(mktemp -d /tmp/pungi.XXX) || { echo "Failed to create temp directory"; exit 1; }
+git clone https://pagure.io/pungi.git "$dir_pungi"
+pushd "$dir_pungi"/doc
+make html
+popd
+
+dir_pungi_doc=$(mktemp -d /tmp/pungi-doc.XXX) || { echo "Failed to create temp directory"; exit 1; }
+git clone ssh://git@pagure.io/docs/pungi.git "$dir_pungi_doc"
+pushd "$dir_pungi_doc"
+git rm -fr ./*
+cp -r "$dir_pungi"/doc/_build/html/* ./
+pushd "$dir_pungi"/doc
+git checkout 4.0.x
+make html
+popd
+mkdir 4.0
+cp -r "$dir_pungi"/doc/_build/html/* ./4.0/
+git add .
+git commit -s -m "update rendered pungi docs for release $1"
+git push origin master
+popd

git-changelog

@@ -0,0 +1,105 @@
+#!/usr/bin/env python
+#
+# git-changelog - Output a rpm changelog
+#
+# Copyright (C) 2009-2010  Red Hat, Inc.
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU Lesser General Public License as published
+# by the Free Software Foundation; either version 2.1 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Lesser General Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+#
+# Author: David Cantrell <dcantrell@redhat.com>
+# Author: Brian C. Lane <bcl@redhat.com>
+
+from __future__ import print_function
+
+import subprocess
+import textwrap
+from argparse import ArgumentParser
+
+
+class ChangeLog:
+    def __init__(self, name, version):
+        self.name = name
+        self.version = version
+
+    def _getCommitDetail(self, commit, field):
+        proc = subprocess.Popen(
+            ["git", "log", "-1", "--pretty=format:%s" % field, commit],
+            stdout=subprocess.PIPE,
+            stderr=subprocess.PIPE,
+            universal_newlines=True,
+        ).communicate()
+
+        ret = proc[0].strip('\n').split('\n')
+
+        if field == '%aE' and len(ret) == 1 and ret[0].find('@') != -1:
+            ret = ret[0].split('@')[0]
+        elif len(ret) == 1:
+            ret = ret[0]
+        else:
+            ret = filter(lambda x: x != '', ret)
+
+        return ret
+
+    def getLog(self):
+        if not self.name:
+            range = "%s.." % (self.version)
+        else:
+            range = "%s-%s.." % (self.name, self.version)
+        proc = subprocess.Popen(
+            ["git", "log", "--pretty=oneline", "--no-merges", range],
+            stdout=subprocess.PIPE,
+            stderr=subprocess.PIPE,
+            universal_newlines=True,
+        ).communicate()
+        lines = filter(lambda x: x.find('l10n: ') != 41,
+                       proc[0].strip('\n').split('\n'))
+
+        log = []
+        for line in lines:
+            fields = line.split(' ')
+            commit = fields[0]
+
+            summary = self._getCommitDetail(commit, "%s")
+            author = self._getCommitDetail(commit, "%aE")
+
+            log.append(("%s (%s)" % (summary.strip(), author)))
+
+        return log
+
+    def formatLog(self):
+        s = ""
+        for msg in self.getLog():
+            sublines = textwrap.wrap(msg, 77)
+            s = s + "- %s\n" % sublines[0]
+
+            if len(sublines) > 1:
+                for subline in sublines[1:]:
+                    s = s + "  %s\n" % subline
+
+        return s
+
+
+def main():
+    parser = ArgumentParser()
+    parser.add_argument("-n", "--name",
+                        help="Name of package used in tags")
+    parser.add_argument("-v", "--version",
+                        help="Last version, changelog is commits after this tag")
+    args = parser.parse_args()
+
+    cl = ChangeLog(args.name, args.version)
+    print(cl.formatLog())
+
+if __name__ == "__main__":
+    main()

pungi

@@ -1,187 +0,0 @@
-#!/usr/bin/python -tt
-# This program is free software; you can redistribute it and/or modify
-# it under the terms of the GNU General Public License as published by
-# the Free Software Foundation; version 2 of the License.
-#
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU Library General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License
-# along with this program; if not, write to the Free Software
-# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
-
-import os
-import pypungi.gather
-import pypungi.pungi
-import yum
-
-from ConfigParser import SafeConfigParser
-
-def main():
-    # Set some default variables, can be overrided in config file
-
-    # Turn this into a dict someday, to iterate over when setting defaults
-    flavor = ""
-    osdir = "os"
-    sourcedir = "source"
-    debugdir = "debug"
-    isodir = "iso"
-    cdsize = "685.0"
-    relnotefilere = "eula.txt fedora.css GPL README-BURNING-ISOS-en_US.txt RELEASE-NOTES-en_US.html ^RPM-GPG"
-    relnotedirre = "images stylesheet-images"
-    relnotepkgs = "fedora-release fedora-release-notes"
-    
-    (opts, args) = get_arguments()
-
-    config = SafeConfigParser()
-    config.read(opts.config)
-
-    if "default" not in config.sections():
-        print ("Check that the file %s exists and that it has a 'default' section" % opts.config)
-        sys.exit(1)
-
-    if not config.has_option('default', 'flavor'):
-        config.set('default', 'flavor', flavor)
-
-    if not config.has_option('default', 'osdir'):
-        config.set('default', 'osdir', osdir)
-
-    if not config.has_option('default', 'sourcedir'):
-        config.set('default', 'sourcedir', sourcedir)
-
-    if not config.has_option('default', 'debugdir'):
-        config.set('default', 'debugdir', debugdir)
-
-    if not config.has_option('default', 'isodir'):
-        config.set('default', 'isodir', isodir)
-
-    if not config.has_option('default', 'cdsize'):
-        config.set('default', 'cdsize', cdsize)
-
-    if not config.has_option('default', 'relnotefilere'):
-        config.set('default', 'relnotefilere', relnotefilere)
-
-    if not config.has_option('default', 'relnotedirre'):
-        config.set('default', 'relnotedirre', relnotedirre)
-
-    if not config.has_option('default', 'relnotepkgs'):
-        config.set('default', 'relnotepkgs', relnotepkgs)
-
-    # set some other defaults
-    if not config.has_option('default', 'product_path'):
-        config.set('default', 'product_path', config.get('default', 'product_name'))
-
-    if not config.has_option('default', 'iso_basename'):
-        config.set('default', 'iso_basename', config.get('default', 'product_name'))
-
-    pkglist = get_packagelist(config.get('default', 'manifest'))
-
-    if not opts.destdir == "*CONFFILE*":
-        config.set('default', 'destdir', opts.destdir)
-
-    destdir = config.get('default', 'destdir')
-
-    if not os.path.exists(destdir):
-        try:
-            os.makedirs(destdir)
-        except OSError, e:
-            print >> sys.stderr, "Error: Cannot create destination dir %s" % destdir
-            sys.exit(1)
-
-    cachedir = config.get('default', 'cachedir')
-
-    if not os.path.exists(cachedir):
-        try:
-            os.makedirs(cachedir)
-        except OSError, e:
-            print >> sys.stderr, "Error: Cannot create cache dir %s" % cachedir
-            sys.exit(1)
-
-    # Actually do work.
-    if not config.get('default', 'arch') == 'source':
-        if opts.do_all or opts.do_gather:
-            mygather = pypungi.gather.Gather(config, pkglist)
-            mygather.getPackageObjects()
-            mygather.downloadPackages()
-            if config.getboolean('default', 'getsource'):
-                mygather.getSRPMList()
-                mygather.downloadSRPMs()
-
-        mypungi = pypungi.pungi.Pungi(config)
-
-        if opts.do_all or opts.do_buildinstall:
-           mypungi.doBuildinstall()
-           mypungi.doGetRelnotes()
-
-        if opts.do_all or opts.do_packageorder:
-           mypungi.doPackageorder()
-
-        if opts.do_all or opts.do_splittree:
-             mypungi.doSplittree()
-
-        if opts.do_all or opts.do_createiso:
-           mypungi.doCreateSplitrepo()
-           mypungi.doCreateIsos()
-
-    # Do things slightly different for src.
-    if config.get('default', 'arch') == 'source':
-        # we already have all the content gathered
-        mypungi = pypungi.pungi.Pungi(config)
-        mypungi.topdir = os.path.join(config.get('default', 'destdir'),
-                                      config.get('default', 'version'),
-                                      config.get('default', 'flavor'),
-                                      'source', 'SRPM')
-        if opts.do_all or opts.do_splittree:
-            mypungi.doSplitSRPMs()
-
-        if opts.do_all or opts.do_createiso:
-            mypungi.doCreateIsos()
-
-if __name__ == '__main__':
-    from optparse import OptionParser
-    import sys
-
-    def get_arguments():
-        parser = OptionParser(version="%prog 0.3.2")
-        parser.add_option("--destdir", default="*CONFFILE*", dest="destdir",
-          help='destination directory (defaults to current directory)')
-        parser.add_option("-c", "--conf", default='/etc/pungi/pungi.conf', dest="config",
-          help='Config file to use')
-        parser.add_option("--all-stages", action="store_true", default=True, dest="do_all",
-          help="Enable ALL stages")
-        parser.add_option("-G", action="store_true", default=False, dest="do_gather",
-          help="Flag to enable processing the Gather stage")
-        parser.add_option("-B", action="store_true", default=False, dest="do_buildinstall",
-          help="Flag to enable processing the BuildInstall stage")
-        parser.add_option("-P", action="store_true", default=False, dest="do_packageorder",
-          help="Flag to enable processing the Package Order stage")
-        parser.add_option("-S", action="store_true", default=False, dest="do_splittree",
-          help="Flag to enable processing the SplitTree stage")
-        parser.add_option("-I", action="store_true", default=False, dest="do_createiso",
-          help="Flag to enable processing the CreateISO stage")
-
-
-        (opts, args) = parser.parse_args()
-        if opts.do_gather or opts.do_buildinstall or opts.do_packageorder or opts.do_splittree or opts.do_createiso:
-            opts.do_all = False
-        if len(sys.argv) < 2:
-            parser.print_help()
-            sys.exit(0)
-        return (opts, args)
-
-    def get_packagelist(manifest):
-    # Get the list of packages from the manifest file
-        try:
-            manifestfile = open(manifest, 'r')
-
-        except IOError:
-            print >> sys.stderr, "pungi: No such file:\'%s\'" % manifest
-            sys.exit(1)
-
-        pkglist = manifestfile.readlines()
-        manifestfile.close()
-        return pkglist
-
-    main()

pungi/__init__.py

@@ -0,0 +1,33 @@
+# -*- coding: utf-8 -*-
+
+import os
+import re
+
+
+def get_full_version():
+    """
+    Find full version of Pungi: if running from git, this will return cleaned
+    output of `git describe`, otherwise it will look for installed version.
+    """
+    location = os.path.join(os.path.dirname(os.path.realpath(__file__)), "..")
+    if os.path.isdir(os.path.join(location, ".git")):
+        import subprocess
+
+        proc = subprocess.Popen(
+            ["git", "--git-dir=%s/.git" % location, "describe", "--tags"],
+            stdout=subprocess.PIPE,
+            universal_newlines=True,
+        )
+        output, _ = proc.communicate()
+        return re.sub(r"-1.fc\d\d?", "", output.strip().replace("pungi-", ""))
+    else:
+        import subprocess
+
+        proc = subprocess.Popen(
+            ["rpm", "-q", "pungi"], stdout=subprocess.PIPE, universal_newlines=True
+        )
+        (output, err) = proc.communicate()
+        if not err:
+            return output.rstrip()
+        else:
+            return "unknown"

pungi/arch.py

@@ -0,0 +1,115 @@
+# -*- coding: utf-8 -*-
+
+
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; version 2 of the License.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Library General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, see <https://gnu.org/licenses/>.
+
+from .arch_utils import arches as ALL_ARCHES
+from .arch_utils import getBaseArch, getMultiArchInfo, getArchList
+
+TREE_ARCH_YUM_ARCH_MAP = {
+    "i386": "i686",
+    "sparc": "sparc64v",
+    "arm": "armv7l",
+    "armhfp": "armv7hnl",
+}
+
+
+def tree_arch_to_yum_arch(tree_arch):
+    # this is basically an opposite to pungi.arch_utils.getBaseArch()
+    yum_arch = TREE_ARCH_YUM_ARCH_MAP.get(tree_arch, tree_arch)
+    return yum_arch
+
+
+def get_multilib_arch(yum_arch):
+    arch_info = getMultiArchInfo(yum_arch)
+    if arch_info is None:
+        return None
+    return arch_info[0]
+
+
+def get_valid_multilib_arches(tree_arch):
+    yum_arch = tree_arch_to_yum_arch(tree_arch)
+    multilib_arch = get_multilib_arch(yum_arch)
+    if not multilib_arch:
+        return []
+    return [i for i in getArchList(multilib_arch) if i not in ("noarch", "src")]
+
+
+def get_valid_arches(tree_arch, multilib=True, add_noarch=True, add_src=False):
+    result = []
+
+    yum_arch = tree_arch_to_yum_arch(tree_arch)
+    for arch in getArchList(yum_arch):
+        if arch not in result:
+            result.append(arch)
+
+    if not multilib:
+        for i in get_valid_multilib_arches(tree_arch):
+            while i in result:
+                result.remove(i)
+
+    if add_noarch and "noarch" not in result:
+        result.append("noarch")
+
+    if add_src and "src" not in result:
+        result.append("src")
+
+    return result
+
+
+def get_compatible_arches(arch, multilib=False):
+    tree_arch = getBaseArch(arch)
+    compatible_arches = get_valid_arches(tree_arch, multilib=multilib)
+    return compatible_arches
+
+
+def is_valid_arch(arch):
+    if arch in ("noarch", "src", "nosrc"):
+        return True
+    if arch in ALL_ARCHES:
+        return True
+    return False
+
+
+def split_name_arch(name_arch):
+    if "." in name_arch:
+        name, arch = name_arch.rsplit(".", 1)
+        if not is_valid_arch(arch):
+            name, arch = name_arch, None
+    else:
+        name, arch = name_arch, None
+    return name, arch
+
+
+def is_excluded(package, arches, logger=None):
+    """Check if package is excluded from given architectures."""
+    if any(
+        getBaseArch(exc_arch) == 'x86_64' for exc_arch in package.exclusivearch
+    ) and 'x86_64_v2' not in package.exclusivearch:
+        package.exclusivearch.append('x86_64_v2')
+
+    if package.excludearch and set(package.excludearch) & set(arches):
+        if logger:
+            logger.debug(
+                "Excluding (EXCLUDEARCH: %s): %s"
+                % (sorted(set(package.excludearch)), package.file_name)
+            )
+        return True
+    if package.exclusivearch and not (set(package.exclusivearch) & set(arches)):
+        if logger:
+            logger.debug(
+                "Excluding (EXCLUSIVEARCH: %s): %s"
+                % (sorted(set(package.exclusivearch)), package.file_name)
+            )
+        return True
+    return False

pungi/arch_utils.py

@@ -0,0 +1,368 @@
+# A copy of some necessary parts from yum.rpmUtils.arch, with slightly changes:
+# 1. _ppc64_native_is_best changed to True
+# 2. code style fixes for flake8 reported errors
+
+import os
+import rpm
+import ctypes
+import struct
+
+# _ppc64_native_is_best is False in yum's source code, but patched with a
+# separate patch when built from source rpm, so we set it to True here.
+_ppc64_native_is_best = True
+
+# dict mapping arch -> ( multicompat, best personality, biarch personality )
+multilibArches = {
+    "x86_64": ("athlon", "x86_64", "athlon"),
+    "sparc64v": ("sparcv9v", "sparcv9v", "sparc64v"),
+    "sparc64": ("sparcv9", "sparcv9", "sparc64"),
+    "ppc64": ("ppc", "ppc", "ppc64"),
+    "s390x": ("s390", "s390x", "s390"),
+}
+if _ppc64_native_is_best:
+    multilibArches["ppc64"] = ("ppc", "ppc64", "ppc64")
+
+arches = {
+    # ia32
+    "athlon": "i686",
+    "i686": "i586",
+    "geode": "i586",
+    "i586": "i486",
+    "i486": "i386",
+    "i386": "noarch",
+    # amd64
+    "x86_64": "athlon",
+    "amd64": "x86_64",
+    "ia32e": "x86_64",
+    # x86-64-v2
+    "x86_64_v2": "noarch",
+    # ppc64le
+    "ppc64le": "noarch",
+    # ppc
+    "ppc64p7": "ppc64",
+    "ppc64pseries": "ppc64",
+    "ppc64iseries": "ppc64",
+    "ppc64": "ppc",
+    "ppc": "noarch",
+    # s390{,x}
+    "s390x": "s390",
+    "s390": "noarch",
+    # sparc
+    "sparc64v": "sparcv9v",
+    "sparc64": "sparcv9",
+    "sparcv9v": "sparcv9",
+    "sparcv9": "sparcv8",
+    "sparcv8": "sparc",
+    "sparc": "noarch",
+    # alpha
+    "alphaev7": "alphaev68",
+    "alphaev68": "alphaev67",
+    "alphaev67": "alphaev6",
+    "alphaev6": "alphapca56",
+    "alphapca56": "alphaev56",
+    "alphaev56": "alphaev5",
+    "alphaev5": "alphaev45",
+    "alphaev45": "alphaev4",
+    "alphaev4": "alpha",
+    "alpha": "noarch",
+    # arm
+    "armv7l": "armv6l",
+    "armv6l": "armv5tejl",
+    "armv5tejl": "armv5tel",
+    "armv5tel": "noarch",
+    # arm hardware floating point
+    "armv7hnl": "armv7hl",
+    "armv7hl": "armv6hl",
+    "armv6hl": "noarch",
+    # arm64
+    "arm64": "noarch",
+    # aarch64
+    "aarch64": "noarch",
+    # super-h
+    "sh4a": "sh4",
+    "sh4": "noarch",
+    "sh3": "noarch",
+    # itanium
+    "ia64": "noarch",
+}
+
+#  Will contain information parsed from /proc/self/auxv via _parse_auxv().
+# Should move into rpm really.
+_aux_vector = {
+    "platform": "",
+    "hwcap": 0,
+}
+
+
+def isMultiLibArch(arch=None):  # pragma: no cover
+    """returns true if arch is a multilib arch, false if not"""
+    if arch is None:
+        arch = canonArch
+
+    if arch not in arches:  # or we could check if it is noarch
+        return 0
+
+    if arch in multilibArches:
+        return 1
+
+    if arches[arch] in multilibArches:
+        return 1
+
+    return 0
+
+
+def getArchList(thisarch=None):  # pragma: no cover
+    # this returns a list of archs that are compatible with arch given
+    if not thisarch:
+        thisarch = canonArch
+
+    archlist = [thisarch]
+    while thisarch in arches:
+        thisarch = arches[thisarch]
+        archlist.append(thisarch)
+
+    # hack hack hack
+    # sparc64v is also sparc64 compat
+    if archlist[0] == "sparc64v":
+        archlist.insert(1, "sparc64")
+
+    # if we're a weirdo arch - add noarch on there.
+    if len(archlist) == 1 and archlist[0] == thisarch:
+        archlist.append("noarch")
+    return archlist
+
+
+def _try_read_cpuinfo():  # pragma: no cover
+    """Try to read /proc/cpuinfo ... if we can't ignore errors (ie. proc not
+    mounted)."""
+    try:
+        with open("/proc/cpuinfo", "r") as f:
+            return f.readlines()
+    except Exception:
+        return []
+
+
+def _parse_auxv():  # pragma: no cover
+    """Read /proc/self/auxv and parse it into global dict for easier access
+    later on, very similar to what rpm does."""
+    # In case we can't open and read /proc/self/auxv, just return
+    try:
+        with open("/proc/self/auxv", "rb") as f:
+            data = f.read()
+    except Exception:
+        return
+
+    # Define values from /usr/include/elf.h
+    AT_PLATFORM = 15
+    AT_HWCAP = 16
+    fmtlen = struct.calcsize("LL")
+    offset = 0
+    platform = ctypes.c_char_p()
+
+    # Parse the data and fill in _aux_vector dict
+    while offset <= len(data) - fmtlen:
+        at_type, at_val = struct.unpack_from("LL", data, offset)
+        if at_type == AT_PLATFORM:
+            platform.value = at_val
+            _aux_vector["platform"] = platform.value
+        if at_type == AT_HWCAP:
+            _aux_vector["hwcap"] = at_val
+        offset = offset + fmtlen
+
+
+def getCanonX86Arch(arch):  # pragma: no cover
+    if arch == "i586":
+        for line in _try_read_cpuinfo():
+            if line.startswith("model name"):
+                if line.find("Geode(TM)") != -1:
+                    return "geode"
+                break
+        return arch
+    # only athlon vs i686 isn't handled with uname currently
+    if arch != "i686":
+        return arch
+
+    # if we're i686 and AuthenticAMD, then we should be an athlon
+    for line in _try_read_cpuinfo():
+        if line.startswith("vendor") and line.find("AuthenticAMD") != -1:
+            return "athlon"
+        elif line.startswith("vendor") and line.find("HygonGenuine") != -1:
+            return "athlon"
+        # i686 doesn't guarantee cmov, but we depend on it
+        elif line.startswith("flags"):
+            if line.find("cmov") == -1:
+                return "i586"
+            break
+
+    return arch
+
+
+def getCanonARMArch(arch):  # pragma: no cover
+    # the %{_target_arch} macro in rpm will let us know the abi we are using
+    target = rpm.expandMacro("%{_target_cpu}")
+    if target.startswith("armv6h"):
+        return target
+    if target.startswith("armv7h"):
+        return target
+    return arch
+
+
+def getCanonPPCArch(arch):  # pragma: no cover
+    # FIXME: should I do better handling for mac, etc?
+    if arch != "ppc64":
+        return arch
+
+    machine = None
+    for line in _try_read_cpuinfo():
+        if line.find("machine") != -1:
+            machine = line.split(":")[1]
+            break
+
+    platform = _aux_vector["platform"]
+    if machine is None and not platform:
+        return arch
+
+    try:
+        if platform.startswith("power") and int(platform[5:].rstrip("+")) >= 7:
+            return "ppc64p7"
+    except Exception:
+        pass
+
+    if machine is None:
+        return arch
+
+    if machine.find("CHRP IBM") != -1:
+        return "ppc64pseries"
+    if machine.find("iSeries") != -1:
+        return "ppc64iseries"
+    return arch
+
+
+def getCanonSPARCArch(arch):  # pragma: no cover
+    # Deal with sun4v, sun4u, sun4m cases
+    SPARCtype = None
+    for line in _try_read_cpuinfo():
+        if line.startswith("type"):
+            SPARCtype = line.split(":")[1]
+            break
+    if SPARCtype is None:
+        return arch
+
+    if SPARCtype.find("sun4v") != -1:
+        if arch.startswith("sparc64"):
+            return "sparc64v"
+        else:
+            return "sparcv9v"
+    if SPARCtype.find("sun4u") != -1:
+        if arch.startswith("sparc64"):
+            return "sparc64"
+        else:
+            return "sparcv9"
+    if SPARCtype.find("sun4m") != -1:
+        return "sparcv8"
+    return arch
+
+
+def getCanonX86_64Arch(arch):  # pragma: no cover
+    if arch != "x86_64":
+        return arch
+
+    vendor = None
+    for line in _try_read_cpuinfo():
+        if line.startswith("vendor_id"):
+            vendor = line.split(":")[1]
+            break
+    if vendor is None:
+        return arch
+
+    if vendor.find("Authentic AMD") != -1 or vendor.find("AuthenticAMD") != -1:
+        return "amd64"
+    if vendor.find("HygonGenuine") != -1:
+        return "amd64"
+    if vendor.find("GenuineIntel") != -1:
+        return "ia32e"
+    return arch
+
+
+def getCanonArch(skipRpmPlatform=0):  # pragma: no cover
+    if not skipRpmPlatform and os.access("/etc/rpm/platform", os.R_OK):
+        try:
+            f = open("/etc/rpm/platform", "r")
+            line = f.readline()
+            f.close()
+            (arch, vendor, opersys) = line.split("-", 2)
+            return arch
+        except Exception:
+            pass
+
+    arch = os.uname()[4]
+
+    _parse_auxv()
+
+    if len(arch) == 4 and arch[0] == "i" and arch[2:4] == "86":
+        return getCanonX86Arch(arch)
+
+    if arch.startswith("arm"):
+        return getCanonARMArch(arch)
+    if arch.startswith("ppc"):
+        return getCanonPPCArch(arch)
+    if arch.startswith("sparc"):
+        return getCanonSPARCArch(arch)
+    if arch == "x86_64":
+        return getCanonX86_64Arch(arch)
+
+    return arch
+
+
+canonArch = getCanonArch()
+
+
+# this gets you the "compat" arch of a biarch pair
+def getMultiArchInfo(arch=canonArch):  # pragma: no cover
+    if arch in multilibArches:
+        return multilibArches[arch]
+    if arch in arches and arches[arch] != "noarch":
+        return getMultiArchInfo(arch=arches[arch])
+    return None
+
+
+def getBaseArch(myarch=None):  # pragma: no cover
+    """returns 'base' arch for myarch, if specified, or canonArch if not.
+    base arch is the arch before noarch in the arches dict if myarch is not
+    a key in the multilibArches."""
+
+    if not myarch:
+        myarch = canonArch
+
+    if myarch not in arches:  # this is dumb, but <shrug>
+        return myarch
+
+    if myarch.startswith("sparc64"):
+        return "sparc"
+    elif myarch == "ppc64le":
+        return "ppc64le"
+    elif myarch.startswith("ppc64") and not _ppc64_native_is_best:
+        return "ppc"
+    elif myarch.startswith("arm64"):
+        return "arm64"
+    elif myarch.startswith("armv6h"):
+        return "armhfp"
+    elif myarch.startswith("armv7h"):
+        return "armhfp"
+    elif myarch.startswith("arm"):
+        return "arm"
+
+    if isMultiLibArch(arch=myarch):
+        if myarch in multilibArches:
+            return myarch
+        else:
+            return arches[myarch]
+
+    if myarch in arches:
+        basearch = myarch
+        value = arches[basearch]
+        while value != "noarch":
+            basearch = value
+            value = arches[basearch]
+
+        return basearch

pungi/common.py

@@ -0,0 +1,34 @@
+# -*- coding: utf-8 -*-
+
+
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; version 2 of the License.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Library General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, see <https://gnu.org/licenses/>.
+
+
+class OptionsBase(object):
+    def __init__(self, **kwargs):
+        """
+        inherit and initialize attributes
+        call self.merge_options(**kwargs) at the end
+        """
+        pass
+
+    def merge_options(self, **kwargs):
+        """
+        override defaults with user defined values
+        """
+        for key, value in kwargs.items():
+            if not hasattr(self, key):
+                raise ValueError(
+                    "Invalid option in %s: %s" % (self.__class__.__name__, key)
+                )
+            setattr(self, key, value)

pungi/compose.py

@@ -0,0 +1,779 @@
+# -*- coding: utf-8 -*-
+
+
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; version 2 of the License.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Library General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, see <https://gnu.org/licenses/>.
+
+
+__all__ = ("Compose",)
+
+
+import contextlib
+import errno
+import logging
+import os
+import time
+import tempfile
+import shutil
+import json
+import socket
+
+import kobo.log
+import kobo.tback
+import requests
+from requests.exceptions import RequestException
+from productmd.composeinfo import ComposeInfo
+from productmd.images import Images
+from dogpile.cache import make_region
+
+
+from pungi.graph import SimpleAcyclicOrientedGraph
+from pungi.wrappers.variants import VariantsXmlParser
+from pungi.paths import Paths
+from pungi.wrappers.kojiwrapper import KojiDownloadProxy
+from pungi.wrappers.scm import get_file_from_scm
+from pungi.util import (
+    makedirs,
+    get_arch_variant_data,
+    get_format_substs,
+    get_variant_data,
+    retry,
+    translate_path_raw,
+)
+from pungi.metadata import compose_to_composeinfo
+
+try:
+    # This is available since productmd >= 1.18
+    # TODO: remove this once the version is distributed widely enough
+    from productmd.composeinfo import SUPPORTED_MILESTONES
+except ImportError:
+    SUPPORTED_MILESTONES = ["RC", "Update", "SecurityFix"]
+
+
+def is_status_fatal(status_code):
+    """Check if status code returned from CTS reports an error that is unlikely
+    to be fixed by retrying. Generally client errors (4XX) are fatal, with the
+    exception of 401 Unauthorized which could be caused by transient network
+    issue between compose host and KDC.
+    """
+    if status_code == 401:
+        return False
+    return status_code >= 400 and status_code < 500
+
+
+@retry(wait_on=RequestException)
+def retry_request(method, url, data=None, json_data=None, auth=None):
+    """
+    :param str method: Reqest method.
+    :param str url: Target URL.
+    :param dict data: form-urlencoded data to send in the body of the request.
+    :param dict json_data: json data to send in the body of the request.
+    """
+    request_method = getattr(requests, method)
+    rv = request_method(url, data=data, json=json_data, auth=auth)
+    if is_status_fatal(rv.status_code):
+        try:
+            error = rv.json()
+        except ValueError:
+            error = rv.text
+        raise RuntimeError("%s responded with %d: %s" % (url, rv.status_code, error))
+    rv.raise_for_status()
+    return rv
+
+
+class BearerAuth(requests.auth.AuthBase):
+    def __init__(self, token):
+        self.token = token
+
+    def __call__(self, r):
+        r.headers["authorization"] = "Bearer " + self.token
+        return r
+
+
+@contextlib.contextmanager
+def cts_auth(pungi_conf):
+    """
+    :param dict pungi_conf: dict obj of pungi.json config.
+    """
+    auth = None
+    token = None
+    cts_keytab = pungi_conf.get("cts_keytab")
+    cts_oidc_token_url = os.environ.get("CTS_OIDC_TOKEN_URL", "") or pungi_conf.get(
+        "cts_oidc_token_url"
+    )
+
+    try:
+        if cts_keytab:
+            # requests-kerberos cannot accept custom keytab, we need to use
+            # environment variable for this. But we need to change environment
+            # only temporarily just for this single requests.post.
+            # So at first backup the current environment and revert to it
+            # after the requests call.
+            from requests_kerberos import HTTPKerberosAuth
+
+            auth = HTTPKerberosAuth()
+            environ_copy = dict(os.environ)
+            if "$HOSTNAME" in cts_keytab:
+                cts_keytab = cts_keytab.replace("$HOSTNAME", socket.gethostname())
+            os.environ["KRB5_CLIENT_KTNAME"] = cts_keytab
+            os.environ["KRB5CCNAME"] = "DIR:%s" % tempfile.mkdtemp()
+        elif cts_oidc_token_url:
+            cts_oidc_client_id = os.environ.get(
+                "CTS_OIDC_CLIENT_ID", ""
+            ) or pungi_conf.get("cts_oidc_client_id", "")
+            token = retry_request(
+                "post",
+                cts_oidc_token_url,
+                data={
+                    "grant_type": "client_credentials",
+                    "client_id": cts_oidc_client_id,
+                    "client_secret": os.environ.get("CTS_OIDC_CLIENT_SECRET", ""),
+                },
+            ).json()["access_token"]
+            auth = BearerAuth(token)
+            del token
+
+        yield auth
+    except Exception as e:
+        # Avoid leaking client secret in trackback
+        e.show_locals = False
+        raise e
+    finally:
+        if cts_keytab:
+            shutil.rmtree(os.environ["KRB5CCNAME"].split(":", 1)[1])
+            os.environ.clear()
+            os.environ.update(environ_copy)
+
+
+def get_compose_info(
+    conf,
+    compose_type="production",
+    compose_date=None,
+    compose_respin=None,
+    compose_label=None,
+    parent_compose_ids=None,
+    respin_of=None,
+):
+    """
+    Creates inncomplete ComposeInfo to generate Compose ID
+    """
+    ci = ComposeInfo()
+    ci.release.name = conf["release_name"]
+    ci.release.short = conf["release_short"]
+    ci.release.version = conf["release_version"]
+    ci.release.is_layered = True if conf.get("base_product_name", "") else False
+    ci.release.type = conf.get("release_type", "ga").lower()
+    ci.release.internal = bool(conf.get("release_internal", False))
+    if ci.release.is_layered:
+        ci.base_product.name = conf["base_product_name"]
+        ci.base_product.short = conf["base_product_short"]
+        ci.base_product.version = conf["base_product_version"]
+        ci.base_product.type = conf.get("base_product_type", "ga").lower()
+
+    ci.compose.label = compose_label
+    ci.compose.type = compose_type
+    ci.compose.date = compose_date or time.strftime("%Y%m%d", time.localtime())
+    ci.compose.respin = compose_respin or 0
+    ci.compose.id = ci.create_compose_id()
+
+    cts_url = conf.get("cts_url")
+    if cts_url:
+        # Create compose in CTS and get the reserved compose ID.
+        url = os.path.join(cts_url, "api/1/composes/")
+        data = {
+            "compose_info": json.loads(ci.dumps()),
+            "parent_compose_ids": parent_compose_ids,
+            "respin_of": respin_of,
+        }
+        with cts_auth(conf) as authentication:
+            rv = retry_request("post", url, json_data=data, auth=authentication)
+
+        # Update local ComposeInfo with received ComposeInfo.
+        cts_ci = ComposeInfo()
+        cts_ci.loads(rv.text)
+        ci.compose.respin = cts_ci.compose.respin
+        ci.compose.id = cts_ci.compose.id
+
+    return ci
+
+
+def write_compose_info(compose_dir, ci):
+    """
+    Write ComposeInfo `ci` to `compose_dir` subdirectories.
+    """
+    makedirs(compose_dir)
+    with open(os.path.join(compose_dir, "COMPOSE_ID"), "w") as f:
+        f.write(ci.compose.id)
+    work_dir = os.path.join(compose_dir, "work", "global")
+    makedirs(work_dir)
+    ci.dump(os.path.join(work_dir, "composeinfo-base.json"))
+
+
+def update_compose_url(compose_id, compose_dir, conf):
+    cts_url = conf.get("cts_url", None)
+    if cts_url:
+        url = os.path.join(cts_url, "api/1/composes", compose_id)
+        tp = conf.get("translate_paths", None)
+        compose_url = translate_path_raw(tp, compose_dir)
+        if compose_url == compose_dir:
+            # We do not have a URL, do not attempt the update.
+            return
+        data = {
+            "action": "set_url",
+            "compose_url": compose_url,
+        }
+        with cts_auth(conf) as authentication:
+            return retry_request("patch", url, json_data=data, auth=authentication)
+
+
+def get_compose_dir(
+    topdir,
+    conf,
+    compose_type="production",
+    compose_date=None,
+    compose_respin=None,
+    compose_label=None,
+    already_exists_callbacks=None,
+    parent_compose_ids=None,
+    respin_of=None,
+):
+    already_exists_callbacks = already_exists_callbacks or []
+
+    ci = get_compose_info(
+        conf,
+        compose_type,
+        compose_date,
+        compose_respin,
+        compose_label,
+        parent_compose_ids,
+        respin_of,
+    )
+
+    cts_url = conf.get("cts_url", None)
+    if cts_url:
+        # Create compose directory.
+        compose_dir = os.path.join(topdir, ci.compose.id)
+        os.makedirs(compose_dir)
+    else:
+        while 1:
+            ci.compose.id = ci.create_compose_id()
+
+            compose_dir = os.path.join(topdir, ci.compose.id)
+
+            exists = False
+            # TODO: callbacks to determine if a composeid was already used
+            # for callback in already_exists_callbacks:
+            #     if callback(data):
+            #         exists = True
+            #         break
+
+            # already_exists_callbacks fallback: does target compose_dir exist?
+            try:
+                os.makedirs(compose_dir)
+            except OSError as ex:
+                if ex.errno == errno.EEXIST:
+                    exists = True
+                else:
+                    raise
+
+            if exists:
+                ci = get_compose_info(
+                    conf,
+                    compose_type,
+                    compose_date,
+                    ci.compose.respin + 1,
+                    compose_label,
+                )
+                continue
+            break
+
+    write_compose_info(compose_dir, ci)
+    return compose_dir
+
+
+class Compose(kobo.log.LoggingBase):
+    def __init__(
+        self,
+        conf,
+        topdir,
+        skip_phases=None,
+        just_phases=None,
+        old_composes=None,
+        koji_event=None,
+        supported=False,
+        logger=None,
+        notifier=None,
+    ):
+        kobo.log.LoggingBase.__init__(self, logger)
+        # TODO: check if minimal conf values are set
+        self.conf = conf
+        # This is a dict mapping UID to Variant objects. It only contains top
+        # level variants.
+        self.variants = {}
+        # This is a similar mapping, but contains even nested variants.
+        self.all_variants = {}
+        self.topdir = os.path.abspath(topdir)
+        self.skip_phases = skip_phases or []
+        self.just_phases = just_phases or []
+        self.old_composes = old_composes or []
+        self.koji_event = koji_event or conf.get("koji_event")
+        self.notifier = notifier
+
+        self._old_config = None
+
+        # path definitions
+        self.paths = Paths(self)
+
+        # Set up logging to file
+        if logger:
+            kobo.log.add_file_logger(
+                logger, self.paths.log.log_file("global", "pungi.log")
+            )
+            kobo.log.add_file_logger(
+                logger, self.paths.log.log_file("global", "excluding-arch.log")
+            )
+
+            class PungiLogFilter(logging.Filter):
+                def filter(self, record):
+                    return (
+                        False
+                        if record.funcName and record.funcName == "is_excluded"
+                        else True
+                    )
+
+            class ExcludingArchLogFilter(logging.Filter):
+                def filter(self, record):
+                    message = record.getMessage()
+                    if "Populating package set for arch:" in message or (
+                        record.funcName and record.funcName == "is_excluded"
+                    ):
+                        return True
+                    else:
+                        return False
+
+            for handler in logger.handlers:
+                if isinstance(handler, logging.FileHandler):
+                    log_file_name = os.path.basename(handler.stream.name)
+                    if log_file_name == "pungi.global.log":
+                        handler.addFilter(PungiLogFilter())
+                    elif log_file_name == "excluding-arch.global.log":
+                        handler.addFilter(ExcludingArchLogFilter())
+
+        # to provide compose_id, compose_date and compose_respin
+        self.ci_base = ComposeInfo()
+        self.ci_base.load(
+            os.path.join(self.paths.work.topdir(arch="global"), "composeinfo-base.json")
+        )
+
+        self.supported = supported
+        if (
+            self.compose_label
+            and self.compose_label.split("-")[0] in SUPPORTED_MILESTONES
+        ):
+            self.log_info(
+                "Automatically setting 'supported' flag due to label: %s."
+                % self.compose_label
+            )
+            self.supported = True
+
+        self.im = Images()
+        self.im.compose.id = self.compose_id
+        self.im.compose.type = self.compose_type
+        self.im.compose.date = self.compose_date
+        self.im.compose.respin = self.compose_respin
+        self.im.metadata_path = self.paths.compose.metadata()
+
+        self.containers_metadata = {}
+
+        # Stores list of deliverables that failed, but did not abort the
+        # compose.
+        # {deliverable: [(Variant.uid, arch, subvariant)]}
+        self.failed_deliverables = {}
+        self.attempted_deliverables = {}
+        self.required_deliverables = {}
+
+        if self.conf.get("dogpile_cache_backend", None):
+            self.cache_region = make_region().configure(
+                self.conf.get("dogpile_cache_backend"),
+                expiration_time=self.conf.get("dogpile_cache_expiration_time", 3600),
+                arguments=self.conf.get("dogpile_cache_arguments", {}),
+            )
+        else:
+            self.cache_region = make_region().configure("dogpile.cache.null")
+
+        self.koji_downloader = KojiDownloadProxy.from_config(self.conf, self._logger)
+
+    get_compose_info = staticmethod(get_compose_info)
+    write_compose_info = staticmethod(write_compose_info)
+    get_compose_dir = staticmethod(get_compose_dir)
+    update_compose_url = staticmethod(update_compose_url)
+
+    def __getitem__(self, name):
+        return self.variants[name]
+
+    @property
+    def compose_id(self):
+        return self.ci_base.compose.id
+
+    @property
+    def compose_date(self):
+        return self.ci_base.compose.date
+
+    @property
+    def compose_respin(self):
+        return self.ci_base.compose.respin
+
+    @property
+    def compose_type(self):
+        return self.ci_base.compose.type
+
+    @property
+    def compose_type_suffix(self):
+        return self.ci_base.compose.type_suffix
+
+    @property
+    def compose_label(self):
+        return self.ci_base.compose.label
+
+    @property
+    def compose_label_major_version(self):
+        return self.ci_base.compose.label_major_version
+
+    @property
+    def has_comps(self):
+        return bool(self.conf.get("comps_file", False))
+
+    @property
+    def has_module_defaults(self):
+        return bool(self.conf.get("module_defaults_dir", False))
+
+    @property
+    def has_module_obsoletes(self):
+        return bool(self.conf.get("module_obsoletes_dir", False))
+
+    @property
+    def config_dir(self):
+        return os.path.dirname(self.conf._open_file or "")
+
+    @property
+    def should_create_yum_database(self):
+        """Explicit configuration trumps all. Otherwise check gather backend
+        and only create it for Yum.
+        """
+        config = self.conf.get("createrepo_database")
+        if config is not None:
+            return config
+        return self.conf["gather_backend"] == "yum"
+
+    def read_variants(self):
+        # TODO: move to phases/init ?
+        variants_file = self.paths.work.variants_file(arch="global")
+
+        scm_dict = self.conf["variants_file"]
+        if isinstance(scm_dict, dict):
+            file_name = os.path.basename(scm_dict["file"])
+            if scm_dict["scm"] == "file":
+                scm_dict["file"] = os.path.join(
+                    self.config_dir, os.path.basename(scm_dict["file"])
+                )
+        else:
+            file_name = os.path.basename(scm_dict)
+            scm_dict = os.path.join(self.config_dir, scm_dict)
+
+        self.log_debug("Writing variants file: %s", variants_file)
+        tmp_dir = self.mkdtemp(prefix="variants_file_")
+        get_file_from_scm(scm_dict, tmp_dir, compose=self)
+        shutil.copy2(os.path.join(tmp_dir, file_name), variants_file)
+        shutil.rmtree(tmp_dir)
+
+        tree_arches = self.conf.get("tree_arches", None)
+        tree_variants = self.conf.get("tree_variants", None)
+        with open(variants_file, "r") as file_obj:
+            parser = VariantsXmlParser(
+                file_obj, tree_arches, tree_variants, logger=self._logger
+            )
+            self.variants = parser.parse()
+
+        self.all_variants = {}
+        for variant in self.get_variants():
+            self.all_variants[variant.uid] = variant
+
+        # populate ci_base with variants - needed for layered-products (compose_id)
+        # FIXME - compose_to_composeinfo is no longer needed and has been
+        #         removed, but I'm not entirely sure what this is needed for
+        #         or if it is at all
+        self.ci_base = compose_to_composeinfo(self)
+
+    def get_variants(self, types=None, arch=None):
+        result = []
+        for i in self.variants.values():
+            if (not types or i.type in types) and (not arch or arch in i.arches):
+                result.append(i)
+            result.extend(i.get_variants(types=types, arch=arch))
+        return sorted(set(result))
+
+    def get_arches(self):
+        result = set()
+        for variant in self.get_variants():
+            for arch in variant.arches:
+                result.add(arch)
+        return sorted(result)
+
+    @property
+    def status_file(self):
+        """Path to file where the compose status will be stored."""
+        if not hasattr(self, "_status_file"):
+            self._status_file = os.path.join(self.topdir, "STATUS")
+        return self._status_file
+
+    def _log_failed_deliverables(self):
+        for kind, data in self.failed_deliverables.items():
+            for variant, arch, subvariant in data:
+                self.log_info(
+                    "Failed %s on variant <%s>, arch <%s>, subvariant <%s>."
+                    % (kind, variant, arch, subvariant)
+                )
+        log = os.path.join(self.paths.log.topdir("global"), "deliverables.json")
+        with open(log, "w") as f:
+            json.dump(
+                {
+                    "required": self.required_deliverables,
+                    "failed": self.failed_deliverables,
+                    "attempted": self.attempted_deliverables,
+                },
+                f,
+                indent=4,
+            )
+
+    def write_status(self, stat_msg):
+        if stat_msg not in ("STARTED", "FINISHED", "DOOMED", "TERMINATED"):
+            self.log_warning("Writing nonstandard compose status: %s" % stat_msg)
+        old_status = self.get_status()
+        if stat_msg == old_status:
+            return
+        if old_status == "FINISHED":
+            msg = "Could not modify a FINISHED compose: %s" % self.topdir
+            self.log_error(msg)
+            raise RuntimeError(msg)
+
+        if stat_msg == "FINISHED" and self.failed_deliverables:
+            stat_msg = "FINISHED_INCOMPLETE"
+
+        self._log_failed_deliverables()
+
+        with open(self.status_file, "w") as f:
+            f.write(stat_msg + "\n")
+
+        if self.notifier:
+            self.notifier.send("status-change", status=stat_msg)
+
+    def get_status(self):
+        if not os.path.isfile(self.status_file):
+            return
+        return open(self.status_file, "r").read().strip()
+
+    def get_image_name(
+        self, arch, variant, disc_type="dvd", disc_num=1, suffix=".iso", format=None
+    ):
+        """Create a filename for image with given parameters.
+
+        :raises RuntimeError: when unknown ``disc_type`` is given
+        """
+        default_format = "{compose_id}-{variant}-{arch}-{disc_type}{disc_num}{suffix}"
+        format = format or self.conf.get("image_name_format", default_format)
+
+        if isinstance(format, dict):
+            conf = get_variant_data(self.conf, "image_name_format", variant)
+            format = conf[0] if conf else default_format
+
+        if arch == "src":
+            arch = "source"
+
+        if disc_num:
+            disc_num = int(disc_num)
+        else:
+            disc_num = ""
+
+        kwargs = {
+            "arch": arch,
+            "disc_type": disc_type,
+            "disc_num": disc_num,
+            "suffix": suffix,
+        }
+        if variant.type == "layered-product":
+            variant_uid = variant.parent.uid
+            kwargs["compose_id"] = self.ci_base[variant.uid].compose_id
+        else:
+            variant_uid = variant.uid
+        args = get_format_substs(self, variant=variant_uid, **kwargs)
+        try:
+            return (format % args).format(**args)
+        except KeyError as err:
+            raise RuntimeError(
+                "Failed to create image name: unknown format element: %s" % err
+            )
+
+    def can_fail(self, variant, arch, deliverable):
+        """Figure out if deliverable can fail on variant.arch.
+
+        Variant can be None.
+        """
+        failable = get_arch_variant_data(
+            self.conf, "failable_deliverables", arch, variant
+        )
+        return deliverable in failable
+
+    def attempt_deliverable(self, variant, arch, kind, subvariant=None):
+        """Log information about attempted deliverable."""
+        variant_uid = variant.uid if variant else ""
+        self.attempted_deliverables.setdefault(kind, []).append(
+            (variant_uid, arch, subvariant)
+        )
+
+    def require_deliverable(self, variant, arch, kind, subvariant=None):
+        """Log information about attempted deliverable."""
+        variant_uid = variant.uid if variant else ""
+        self.required_deliverables.setdefault(kind, []).append(
+            (variant_uid, arch, subvariant)
+        )
+
+    def fail_deliverable(self, variant, arch, kind, subvariant=None):
+        """Log information about failed deliverable."""
+        variant_uid = variant.uid if variant else ""
+        self.failed_deliverables.setdefault(kind, []).append(
+            (variant_uid, arch, subvariant)
+        )
+
+    @property
+    def image_release(self):
+        """Generate a value to pass to Koji as image release.
+
+        If this compose has a label, the version from it will be used,
+        otherwise we will create a string with date, compose type and respin.
+        """
+        if self.compose_label:
+            milestone, release = self.compose_label.split("-")
+            return release
+
+        return "%s%s.%s" % (
+            self.compose_date,
+            self.ci_base.compose.type_suffix,
+            self.compose_respin,
+        )
+
+    @property
+    def image_version(self):
+        """Generate a value to pass to Koji as image version.
+
+        The value is based on release version. If compose has a label, the
+        milestone from it is appended to the version (unless it is RC).
+        """
+        version = self.ci_base.release.version
+        if self.compose_label and not self.compose_label.startswith("RC-"):
+            milestone, release = self.compose_label.split("-")
+            return "%s_%s" % (version, milestone)
+
+        return version
+
+    def mkdtemp(self, arch=None, variant=None, suffix="", prefix="tmp"):
+        """
+        Create and return a unique temporary directory under dir of
+        <compose_topdir>/work/{global,<arch>}/tmp[-<variant>]/
+        """
+        path = os.path.join(self.paths.work.tmp_dir(arch=arch, variant=variant))
+        tmpdir = tempfile.mkdtemp(suffix=suffix, prefix=prefix, dir=path)
+        os.chmod(tmpdir, 0o755)
+        return tmpdir
+
+    def dump_containers_metadata(self):
+        """Create a file with container metadata if there are any containers."""
+        if not self.containers_metadata:
+            return
+        with open(self.paths.compose.metadata("osbs.json"), "w") as f:
+            json.dump(
+                self.containers_metadata,
+                f,
+                indent=4,
+                sort_keys=True,
+                separators=(",", ": "),
+            )
+
+    def traceback(self, detail=None, show_locals=True):
+        """Store an extended traceback. This method should only be called when
+        handling an exception.
+
+        :param str detail: Extra information appended to the filename
+        """
+        basename = "traceback"
+        if detail:
+            basename += "-" + detail
+        tb_path = self.paths.log.log_file("global", basename)
+        self.log_error("Extended traceback in: %s", tb_path)
+        tback = kobo.tback.Traceback(show_locals=show_locals).get_traceback()
+        # Kobo 0.36.0 returns traceback as str, older versions return bytes
+        with open(tb_path, "wb" if isinstance(tback, bytes) else "w") as f:
+            f.write(tback)
+
+    def load_old_compose_config(self):
+        """
+        Helper method to load Pungi config dump from old compose.
+        """
+        if not self._old_config:
+            config_dump_full = self.paths.log.log_file("global", "config-dump")
+            config_dump_full = self.paths.old_compose_path(config_dump_full)
+            if not config_dump_full:
+                return None
+
+            self.log_info("Loading old config file: %s", config_dump_full)
+            with open(config_dump_full, "r") as f:
+                self._old_config = json.load(f)
+
+        return self._old_config
+
+
+def get_ordered_variant_uids(compose):
+    if not hasattr(compose, "_ordered_variant_uids"):
+        ordered_variant_uids = _prepare_variant_as_lookaside(compose)
+        # Some variants were not mentioned in configuration value
+        # 'variant_as_lookaside' and its run order is not crucial (that
+        # means there are no dependencies inside this group). They will be
+        # processed first. A-Z sorting is for reproducibility.
+        unordered_variant_uids = sorted(
+            set(compose.all_variants.keys()) - set(ordered_variant_uids)
+        )
+        setattr(
+            compose,
+            "_ordered_variant_uids",
+            unordered_variant_uids + ordered_variant_uids,
+        )
+    return getattr(compose, "_ordered_variant_uids")
+
+
+def _prepare_variant_as_lookaside(compose):
+    """
+    Configuration value 'variant_as_lookaside' contains variant pairs <variant,
+    its lookaside>. In that pair lookaside variant have to be processed first.
+    Structure can be represented as a oriented graph. Its spanning line shows
+    order how to process this set of variants.
+    """
+    variant_as_lookaside = compose.conf.get("variant_as_lookaside", [])
+    graph = SimpleAcyclicOrientedGraph()
+    for variant, lookaside_variant in variant_as_lookaside:
+        try:
+            graph.add_edge(variant, lookaside_variant)
+        except ValueError as e:
+            raise ValueError(
+                "There is a bad configuration in 'variant_as_lookaside': %s" % e
+            )
+
+    variant_processing_order = reversed(graph.prune_graph())
+    return list(variant_processing_order)

pungi/compose_metadata/discinfo.py

@@ -0,0 +1,93 @@
+# -*- coding: utf-8 -*-
+
+
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; version 2 of the License.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Library General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, see <https://gnu.org/licenses/>.
+
+
+"""
+The .discinfo file contains metadata about media.
+Following fields are part of the .discinfo file,
+one record per line:
+- timestamp
+- release
+- architecture
+- disc number (optional)
+"""
+
+
+__all__ = (
+    "read_discinfo",
+    "write_discinfo",
+    "write_media_repo",
+)
+
+
+import os
+import time
+
+
+def write_discinfo(file_path, description, arch, disc_numbers=None, timestamp=None):
+    """
+    Write a .discinfo file:
+    """
+    disc_numbers = disc_numbers or ["ALL"]
+    if not isinstance(disc_numbers, list):
+        raise TypeError(
+            "Invalid type: disc_numbers type is %s; expected: <list>"
+            % type(disc_numbers)
+        )
+    if not timestamp:
+        timestamp = os.environ.get("SOURCE_DATE_EPOCH", "%f" % time.time())
+    with open(file_path, "w") as f:
+        f.write("%s\n" % timestamp)
+        f.write("%s\n" % description)
+        f.write("%s\n" % arch)
+        if disc_numbers:
+            f.write("%s\n" % ",".join([str(i) for i in disc_numbers]))
+    return timestamp
+
+
+def read_discinfo(file_path):
+    result = {}
+    with open(file_path, "r") as f:
+        result["timestamp"] = f.readline().strip()
+        result["description"] = f.readline().strip()
+        result["arch"] = f.readline().strip()
+        disc_numbers = f.readline().strip()
+    if not disc_numbers:
+        result["disc_numbers"] = None
+    elif disc_numbers == "ALL":
+        result["disc_numbers"] = ["ALL"]
+    else:
+        result["disc_numbers"] = [int(i) for i in disc_numbers.split(",")]
+    return result
+
+
+def write_media_repo(file_path, description, timestamp):
+    """
+    Write media.repo file for the disc to be used on installed system.
+    PackageKit uses this.
+    """
+    data = [
+        "[InstallMedia]",
+        "name=%s" % description,
+        "mediaid=%s" % timestamp,
+        "metadata_expire=-1",
+        "gpgcheck=0",
+        "cost=500",
+        "",
+    ]
+
+    with open(file_path, "w") as repo_file:
+        repo_file.write("\n".join(data))
+    return timestamp

pungi/config.py

@@ -0,0 +1,79 @@
+# -*- coding: utf-8 -*-
+
+
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; version 2 of the License.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Library General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, see <https://gnu.org/licenses/>.
+
+
+import os
+import sys
+import time
+
+from ConfigParser import SafeConfigParser
+
+from .arch_utils import getBaseArch
+
+# In development, `here` will point to the bin/ directory with scripts.
+here = sys.path[0]
+MULTILIBCONF = (
+    os.path.join(os.path.dirname(__file__), "..", "share", "multilib")
+    if here != "/usr/bin"
+    else "/usr/share/pungi/multilib"
+)
+
+
+class Config(SafeConfigParser):
+    def __init__(self, pungirc=None):
+        SafeConfigParser.__init__(self)
+
+        self.add_section("pungi")
+        self.add_section("lorax")
+
+        self.set("pungi", "osdir", "os")
+        self.set("pungi", "sourcedir", "source")
+        self.set("pungi", "debugdir", "debug")
+        self.set("pungi", "isodir", "iso")
+        self.set("pungi", "multilibconf", MULTILIBCONF)
+        self.set(
+            "pungi", "relnotefilere", "LICENSE README-BURNING-ISOS-en_US.txt ^RPM-GPG"
+        )
+        self.set("pungi", "relnotedirre", "")
+        self.set(
+            "pungi", "relnotepkgs", "fedora-repos fedora-release fedora-release-notes"
+        )
+        self.set("pungi", "product_path", "Packages")
+        self.set("pungi", "cachedir", "/var/cache/pungi")
+        self.set("pungi", "compress_type", "xz")
+        self.set("pungi", "arch", getBaseArch())
+        self.set("pungi", "family", "Fedora")
+        self.set("pungi", "iso_basename", "Fedora")
+        self.set("pungi", "version", time.strftime("%Y%m%d", time.localtime()))
+        self.set("pungi", "variant", "")
+        self.set("pungi", "destdir", os.getcwd())
+        self.set("pungi", "workdirbase", "/work")
+        self.set("pungi", "bugurl", "https://bugzilla.redhat.com")
+        self.set("pungi", "cdsize", "695.0")
+        self.set("pungi", "debuginfo", "True")
+        self.set("pungi", "alldeps", "True")
+        self.set("pungi", "isfinal", "False")
+        self.set("pungi", "nohash", "False")
+        self.set("pungi", "full_archlist", "False")
+        self.set("pungi", "multilib", "")
+        self.set("pungi", "lookaside_repos", "")
+        self.set("pungi", "resolve_deps", "True")
+        self.set("pungi", "no_dvd", "False")
+        self.set("pungi", "nomacboot", "False")
+        self.set("pungi", "rootfs_size", "False")
+
+        # if missing, self.read() is a noop, else change 'defaults'
+        if pungirc:
+            self.read(os.path.expanduser(pungirc))

pungi/createiso.py

@@ -0,0 +1,201 @@
+# -*- coding: utf-8 -*-
+
+from __future__ import print_function
+
+import os
+import six
+from collections import namedtuple
+from kobo.shortcuts import run
+from six.moves import shlex_quote
+
+from .wrappers import iso
+from .wrappers.jigdo import JigdoWrapper
+
+from .phases.buildinstall import BOOT_CONFIGS, BOOT_IMAGES
+
+
+CreateIsoOpts = namedtuple(
+    "CreateIsoOpts",
+    [
+        "buildinstall_method",
+        "boot_iso",
+        "arch",
+        "output_dir",
+        "jigdo_dir",
+        "iso_name",
+        "volid",
+        "graft_points",
+        "supported",
+        "os_tree",
+        "hfs_compat",
+        "use_xorrisofs",
+        "iso_level",
+        "script_dir",
+    ],
+)
+CreateIsoOpts.__new__.__defaults__ = (None,) * len(CreateIsoOpts._fields)
+
+
+def quote(str):
+    """Quote an argument for shell, but make sure $TEMPLATE variable will be
+    expanded.
+    """
+    if str.startswith("$TEMPLATE"):
+        return "$TEMPLATE%s" % shlex_quote(str.replace("$TEMPLATE", "", 1))
+    return shlex_quote(str)
+
+
+def emit(f, cmd):
+    """Print line of shell code into the stream."""
+    if isinstance(cmd, six.string_types):
+        print(cmd, file=f)
+    else:
+        print(" ".join([quote(x) for x in cmd]), file=f)
+
+
+FIND_TEMPLATE_SNIPPET = """if ! TEMPLATE="$($(head -n1 $(which lorax) | cut -c3-) -c 'import pylorax; print(pylorax.find_templates())')"; then TEMPLATE=/usr/share/lorax; fi"""  # noqa: E501
+
+
+def make_image(f, opts):
+    mkisofs_kwargs = {}
+
+    if opts.buildinstall_method:
+        if opts.buildinstall_method == "lorax":
+            emit(f, FIND_TEMPLATE_SNIPPET)
+            mkisofs_kwargs["boot_args"] = iso.get_boot_options(
+                opts.arch,
+                os.path.join("$TEMPLATE", "config_files/ppc"),
+                hfs_compat=opts.hfs_compat,
+            )
+
+    # ppc(64) doesn't seem to support utf-8
+    if opts.arch in ("ppc", "ppc64", "ppc64le"):
+        mkisofs_kwargs["input_charset"] = None
+
+    cmd = iso.get_mkisofs_cmd(
+        opts.iso_name,
+        None,
+        volid=opts.volid,
+        exclude=["./lost+found"],
+        graft_points=opts.graft_points,
+        use_xorrisofs=opts.use_xorrisofs,
+        iso_level=opts.iso_level,
+        **mkisofs_kwargs
+    )
+    emit(f, cmd)
+
+
+def implant_md5(f, opts):
+    cmd = iso.get_implantisomd5_cmd(opts.iso_name, opts.supported)
+    emit(f, cmd)
+
+
+def run_isohybrid(f, opts):
+    """If the image is bootable, it should include an MBR or GPT so that it can
+    be booted when written to USB disk. This is done by running isohybrid on
+    the image.
+    """
+    if opts.buildinstall_method and opts.arch in ["x86_64", "i386"]:
+        cmd = iso.get_isohybrid_cmd(opts.iso_name, opts.arch)
+        emit(f, cmd)
+
+
+def make_manifest(f, opts):
+    emit(f, iso.get_manifest_cmd(opts.iso_name, opts.use_xorrisofs))
+
+
+def make_jigdo(f, opts):
+    jigdo = JigdoWrapper()
+    files = [{"path": opts.os_tree, "label": None, "uri": None}]
+    cmd = jigdo.get_jigdo_cmd(
+        os.path.join(opts.output_dir, opts.iso_name),
+        files,
+        output_dir=opts.jigdo_dir,
+        no_servers=True,
+        report="noprogress",
+    )
+    emit(f, cmd)
+
+
+def _get_perms(fs_path):
+    """Compute proper permissions for a file.
+
+    This mimicks what -rational-rock option of genisoimage does. All read bits
+    are set, so that files and directories are globally readable. If any
+    execute bit is set for a file, set them all. No writes are allowed and
+    special bits are erased too.
+    """
+    statinfo = os.stat(fs_path)
+    perms = 0o444
+    if statinfo.st_mode & 0o111:
+        perms |= 0o111
+    return perms
+
+
+def write_xorriso_commands(opts):
+    # Create manifest for the boot.iso listing all contents
+    boot_iso_manifest = "%s.manifest" % os.path.join(
+        opts.script_dir, os.path.basename(opts.boot_iso)
+    )
+    run(
+        iso.get_manifest_cmd(
+            opts.boot_iso, opts.use_xorrisofs, output_file=boot_iso_manifest
+        )
+    )
+    # Find which files may have been updated by pungi. This only includes a few
+    # files from tweaking buildinstall and .discinfo metadata. There's no good
+    # way to detect whether the boot config files actually changed, so we may
+    # be updating files in the ISO with the same data.
+    UPDATEABLE_FILES = set(BOOT_IMAGES + BOOT_CONFIGS + [".discinfo"])
+    updated_files = set()
+    excluded_files = set()
+    with open(boot_iso_manifest) as f:
+        for line in f:
+            path = line.lstrip("/").rstrip("\n")
+            if path in UPDATEABLE_FILES:
+                updated_files.add(path)
+            else:
+                excluded_files.add(path)
+
+    script = os.path.join(opts.script_dir, "xorriso-%s.txt" % id(opts))
+    with open(script, "w") as f:
+        for cmd in iso.xorriso_commands(
+            opts.arch, opts.boot_iso, os.path.join(opts.output_dir, opts.iso_name)
+        ):
+            emit(f, " ".join(cmd))
+        emit(f, "-volid %s" % opts.volid)
+
+        with open(opts.graft_points) as gp:
+            for line in gp:
+                iso_path, fs_path = line.strip().split("=", 1)
+                if iso_path in excluded_files:
+                    continue
+                cmd = "-update" if iso_path in updated_files else "-map"
+                emit(f, "%s %s %s" % (cmd, fs_path, iso_path))
+                emit(f, "-chmod 0%o %s" % (_get_perms(fs_path), iso_path))
+
+        emit(f, "-chown_r 0 /")
+        emit(f, "-chgrp_r 0 /")
+        emit(f, "-end")
+    return script
+
+
+def write_script(opts, f):
+    if bool(opts.jigdo_dir) != bool(opts.os_tree):
+        raise RuntimeError("jigdo_dir must be used together with os_tree")
+
+    emit(f, "#!/bin/bash")
+    emit(f, "set -ex")
+    emit(f, "cd %s" % opts.output_dir)
+
+    if opts.use_xorrisofs and opts.buildinstall_method:
+        script = write_xorriso_commands(opts)
+        emit(f, "xorriso -dialog on <%s" % script)
+    else:
+        make_image(f, opts)
+        run_isohybrid(f, opts)
+
+    implant_md5(f, opts)
+    make_manifest(f, opts)
+    if opts.jigdo_dir:
+        make_jigdo(f, opts)

pungi/dnf_wrapper.py

@@ -0,0 +1,155 @@
+# -*- coding: utf-8 -*-
+
+
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; version 2 of the License.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Library General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, see <https://gnu.org/licenses/>.
+
+
+# TODO: remove all DNF hacks, possibly this whole file
+
+
+import dnf
+
+import dnf.conf
+import dnf.repo
+import dnf.sack
+
+import pungi.arch
+
+try:
+    import dnf.rpm as dnf_arch
+except ImportError:
+    import dnf.arch as dnf_arch
+
+
+class Conf(dnf.conf.Conf):
+    # This is only modified to get our custom Substitutions class in.
+    def __init__(self, arch, *args, **kwargs):
+        super(Conf, self).__init__(*args, **kwargs)
+        self.substitutions = Substitutions(arch)
+
+
+class Substitutions(dict):
+    # DNF version of Substitutions detects host arch. We don't want that.
+    def __init__(self, arch):
+        super(Substitutions, self).__init__()
+        self["arch"] = arch
+        self["basearch"] = dnf_arch.basearch(arch)
+
+
+class DnfWrapper(dnf.Base):
+    def __init__(self, *args, **kwargs):
+        super(DnfWrapper, self).__init__(*args, **kwargs)
+        self.arch_wrapper = ArchWrapper(self.conf.substitutions["arch"])
+        self.comps_wrapper = CompsWrapper(self)
+
+    def add_repo(
+        self, repoid, baseurl=None, enablegroups=True, lookaside=False, **kwargs
+    ):
+        self.repos.add_new_repo(
+            repoid,
+            self.conf,
+            baseurl=[baseurl],
+            enabledgroups=enablegroups,
+            priority=10 if lookaside else 20,
+            module_hotfixes=True,
+            **kwargs
+        )
+
+
+class CompsWrapper(object):
+    def __init__(self, dnf_obj):
+        self.dnf = dnf_obj
+
+    def __getitem__(self, name):
+        return self.groups[name]
+
+    @property
+    def comps(self):
+        return self.dnf.comps
+
+    @property
+    def groups(self):
+        result = {}
+        for i in self.comps.groups:
+            result[i.id] = i
+        return result
+
+    def get_packages_from_group(
+        self,
+        group_id,
+        include_default=True,
+        include_optional=True,
+        include_conditional=True,
+    ):
+        packages = []
+        conditional = []
+
+        group = self.groups[group_id]
+
+        # add mandatory packages
+        packages.extend([i.name for i in group.mandatory_packages])
+
+        # add default packages
+        if include_default:
+            packages.extend([i.name for i in group.default_packages])
+
+        # add optional packages
+        if include_optional:
+            packages.extend([i.name for i in group.optional_packages])
+
+        for package in group.conditional_packages:
+            conditional.append({"name": package.requires, "install": package.name})
+
+        return packages, conditional
+
+    def get_comps_packages(self, groups, exclude_groups):
+        packages = set()
+        conditional = []
+
+        if isinstance(groups, list):
+            groups = dict([(i, 1) for i in groups])
+
+        for group_id, group_include in sorted(groups.items()):
+            if group_id in exclude_groups:
+                continue
+
+            include_default = group_include in (1, 2)
+            include_optional = group_include in (2,)
+            include_conditional = True
+            pkgs, cond = self.get_packages_from_group(
+                group_id, include_default, include_optional, include_conditional
+            )
+            packages.update(pkgs)
+            for i in cond:
+                if i not in conditional:
+                    conditional.append(i)
+        return list(packages), conditional
+
+    def get_langpacks(self):
+        result = []
+        for name, install in self.comps._i.langpacks.items():
+            result.append({"name": name, "install": install})
+        return result
+
+
+class ArchWrapper(object):
+    def __init__(self, arch):
+        self.base_arch = dnf_arch.basearch(arch)
+        self.all_arches = pungi.arch.get_valid_arches(
+            self.base_arch, multilib=True, add_noarch=True
+        )
+        self.native_arches = pungi.arch.get_valid_arches(
+            self.base_arch, multilib=False, add_noarch=True
+        )
+        self.multilib_arches = pungi.arch.get_valid_multilib_arches(self.base_arch)
+        self.source_arches = ["src", "nosrc"]

pungi/errors.py

@@ -0,0 +1,20 @@
+# -*- coding: utf-8 -*-
+
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; version 2 of the License.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Library General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, see <https://gnu.org/licenses/>.
+
+
+class UnsignedPackagesError(RuntimeError):
+    """Raised when package set fails to find a properly signed copy of an
+    RPM."""
+
+    pass

pungi/graph.py

@@ -0,0 +1,105 @@
+# -*- coding: utf-8 -*-
+
+
+class SimpleAcyclicOrientedGraph(object):
+    """
+    Stores a graph data structure and allows operation with it.
+    Example data: {'P1': ['P2'], 'P3': ['P4', 'P5'], 'P2': 'P3'}
+    Graph is constructed by adding oriented edges one by one. It can not contain cycles.
+    Main result is spanning line, it determines ordering of the nodes.
+    """
+
+    def __init__(self):
+        self._graph = {}
+        self._all_nodes = set()
+
+    def add_edge(self, start, end):
+        """
+        Add one edge from node 'start' to node 'end'.
+        This operation must not create a cycle in the graph.
+        """
+        if start == end:
+            raise ValueError(
+                "Can not add this kind of edge into graph: %s-%s" % (start, end)
+            )
+        self._graph.setdefault(start, [])
+        if end not in self._graph[start]:
+            self._graph[start].append(end)
+        self._all_nodes.add(start)
+        self._all_nodes.add(end)
+        # try to find opposite direction path (from end to start)
+        # to detect newly created cycle
+        path = SimpleAcyclicOrientedGraph.find_path(self._graph, end, start)
+        if path:
+            raise ValueError("There is a cycle in the graph: %s" % path)
+
+    def get_active_nodes(self):
+        """
+        nodes connected to any edge
+        """
+        active_nodes = set()
+        for start, ends in self._graph.items():
+            active_nodes.add(start)
+            active_nodes.update(ends)
+        return active_nodes
+
+    def is_final_endpoint(self, node):
+        """
+        edge(s) ends in this node; no other edge starts in this node
+        """
+        if node not in self._all_nodes:
+            return ValueError("This node is not found in the graph: %s" % node)
+        if node not in self.get_active_nodes():
+            return False
+        return False if node in self._graph else True
+
+    def remove_final_endpoint(self, node):
+        """"""
+        remove_start_points = []
+        for start, ends in self._graph.items():
+            if node in ends:
+                ends.remove(node)
+                if not ends:
+                    remove_start_points.append(start)
+        for start in remove_start_points:
+            del self._graph[start]
+
+    @staticmethod
+    def find_path(graph, start, end, path=[]):
+        """
+        find path among nodes 'start' and 'end' recursively
+        """
+        path = path + [start]
+        if start == end:
+            return path
+        if start not in graph:
+            return None
+        for node in graph[start]:
+            if node not in path:
+                newpath = SimpleAcyclicOrientedGraph.find_path(graph, node, end, path)
+                if newpath:
+                    return newpath
+        return None
+
+    def prune_graph(self):
+        """
+        Construct spanning_line by pruning the graph.
+        Looking for endpoints and remove them one by one until graph is empty.
+        """
+        spanning_line = []
+        while self._graph:
+            for node in sorted(self._all_nodes):
+                if self.is_final_endpoint(node):
+                    self.remove_final_endpoint(node)
+                    spanning_line.insert(0, node)
+                    # orphan node = no edge is connected with this node
+                    orphans = self._all_nodes - self.get_active_nodes()
+                    if orphans:
+                        # restart iteration not to set size self._all_nodes
+                        # during iteration
+                        break
+            for orphan in orphans:
+                if orphan not in spanning_line:
+                    spanning_line.insert(0, orphan)
+                self._all_nodes.remove(orphan)
+        return spanning_line

pungi/ks.py

@@ -0,0 +1,223 @@
+# -*- coding: utf-8 -*-
+
+
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; version 2 of the License.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Library General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, see <https://gnu.org/licenses/>.
+
+
+"""
+Pungi adds several new sections to kickstarts.
+
+
+FULLTREE EXCLUDES
+-----------------
+Fulltree excludes allow us to define SRPM names
+we don't want to be part of fulltree processing.
+
+Syntax:
+%fulltree-excludes
+<srpm_name>
+<srpm_name>
+...
+%end
+
+
+MULTILIB BLACKLIST
+------------------
+List of RPMs which are prevented from becoming multilib.
+
+Syntax:
+%multilib-blacklist
+<rpm_name>
+<rpm_name>
+...
+%end
+
+
+MULTILIB WHITELIST
+------------------
+List of RPMs which will become multilib (but only if native package is pulled in).
+
+Syntax:
+%multilib-whitelist
+<rpm_name>
+<rpm_name>
+...
+%end
+
+
+PREPOPULATE
+-----------
+To make sure no package is left behind between 2 composes,
+we can explicitly add <name>.<arch> records to the %prepopulate section.
+These will be added to the input list and marked with 'prepopulate' flag.
+
+Syntax:
+%prepopulate
+<rpm_name>.<rpm_arch>
+<rpm_name>.<rpm_arch>
+...
+%end
+"""
+
+
+import pykickstart.parser
+import pykickstart.sections
+from pykickstart.constants import GROUP_REQUIRED, GROUP_DEFAULT
+
+
+class FulltreeExcludesSection(pykickstart.sections.Section):
+    sectionOpen = "%fulltree-excludes"
+
+    def handleLine(self, line):
+        if not self.handler:
+            return
+
+        (h, s, t) = line.partition("#")
+        line = h.rstrip()
+
+        self.handler.fulltree_excludes.add(line)
+
+
+class MultilibBlacklistSection(pykickstart.sections.Section):
+    sectionOpen = "%multilib-blacklist"
+
+    def handleLine(self, line):
+        if not self.handler:
+            return
+
+        (h, s, t) = line.partition("#")
+        line = h.rstrip()
+
+        self.handler.multilib_blacklist.add(line)
+
+
+class MultilibWhitelistSection(pykickstart.sections.Section):
+    sectionOpen = "%multilib-whitelist"
+
+    def handleLine(self, line):
+        if not self.handler:
+            return
+
+        (h, s, t) = line.partition("#")
+        line = h.rstrip()
+
+        self.handler.multilib_whitelist.add(line)
+
+
+class PrepopulateSection(pykickstart.sections.Section):
+    sectionOpen = "%prepopulate"
+
+    def handleLine(self, line):
+        if not self.handler:
+            return
+
+        (h, s, t) = line.partition("#")
+        line = h.rstrip()
+
+        self.handler.prepopulate.add(line)
+
+
+class KickstartParser(pykickstart.parser.KickstartParser):
+    def setupSections(self):
+        pykickstart.parser.KickstartParser.setupSections(self)
+        self.registerSection(FulltreeExcludesSection(self.handler))
+        self.registerSection(MultilibBlacklistSection(self.handler))
+        self.registerSection(MultilibWhitelistSection(self.handler))
+        self.registerSection(PrepopulateSection(self.handler))
+
+    def get_packages(self, dnf_obj):
+        packages = set()
+        conditional_packages = []
+
+        packages.update(self.handler.packages.packageList)
+
+        for ks_group in self.handler.packages.groupList:
+            group_id = ks_group.name
+
+            if ks_group.include == GROUP_REQUIRED:
+                include_default = False
+                include_optional = False
+            elif ks_group.include == GROUP_DEFAULT:
+                include_default = True
+                include_optional = False
+            else:
+                include_default = True
+                include_optional = True
+
+            (
+                group_packages,
+                group_conditional_packages,
+            ) = dnf_obj.comps_wrapper.get_packages_from_group(
+                group_id,
+                include_default=include_default,
+                include_optional=include_optional,
+                include_conditional=True,
+            )
+            packages.update(group_packages)
+            for i in group_conditional_packages:
+                if i not in conditional_packages:
+                    conditional_packages.append(i)
+
+        return packages, conditional_packages
+
+    def get_excluded_packages(self, dnf_obj):
+        excluded = set()
+        excluded.update(self.handler.packages.excludedList)
+
+        for ks_group in self.handler.packages.excludedGroupList:
+            group_id = ks_group.name
+            include_default = False
+            include_optional = False
+
+            if ks_group.include == 1:
+                include_default = True
+
+            if ks_group.include == 2:
+                include_default = True
+                include_optional = True
+
+            (
+                group_packages,
+                group_conditional_packages,
+            ) = dnf_obj.comps_wrapper.get_packages_from_group(
+                group_id,
+                include_default=include_default,
+                include_optional=include_optional,
+                include_conditional=False,
+            )
+            excluded.update(group_packages)
+
+        return excluded
+
+
+HandlerClass = pykickstart.version.returnClassForVersion()
+
+
+class PungiHandler(HandlerClass):
+    def __init__(self, *args, **kwargs):
+        HandlerClass.__init__(self, *args, **kwargs)
+        self.fulltree_excludes = set()
+        self.multilib_blacklist = set()
+        self.multilib_whitelist = set()
+        self.prepopulate = set()
+
+
+def get_ksparser(ks_path=None):
+    """
+    Return a kickstart parser instance.
+    Read kickstart if ks_path provided.
+    """
+    ksparser = KickstartParser(PungiHandler())
+    if ks_path:
+        ksparser.readKickstart(ks_path)
+    return ksparser

pungi/linker.py

@@ -0,0 +1,247 @@
+# -*- coding: utf-8 -*-
+
+
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; version 2 of the License.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Library General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, see <https://gnu.org/licenses/>.
+
+
+import contextlib
+import errno
+import os
+import shutil
+
+import kobo.log
+from kobo.shortcuts import relative_path
+from kobo.threads import WorkerThread, ThreadPool
+
+from pungi.util import makedirs
+
+
+class LinkerPool(ThreadPool):
+    def __init__(self, link_type="hardlink-or-copy", logger=None):
+        ThreadPool.__init__(self, logger)
+        self.link_type = link_type
+        self.linker = Linker()
+
+    @classmethod
+    def with_workers(cls, num_workers, *args, **kwargs):
+        pool = cls(*args, **kwargs)
+        for _ in range(num_workers):
+            pool.add(LinkerThread(pool))
+        return pool
+
+
+@contextlib.contextmanager
+def linker_pool(link_type="hardlink-or-copy", num_workers=10):
+    """Create a linker and make sure it is stopped no matter what."""
+    linker = LinkerPool.with_workers(num_workers=num_workers, link_type=link_type)
+    linker.start()
+    try:
+        yield linker
+    finally:
+        linker.stop()
+
+
+class LinkerThread(WorkerThread):
+    def process(self, item, num):
+        src, dst = item
+
+        if (num % 100 == 0) or (num == self.pool.queue_total):
+            self.pool.log_debug(
+                "Linked %s out of %s packages" % (num, self.pool.queue_total)
+            )
+
+        directory = os.path.dirname(dst)
+        makedirs(directory)
+        self.pool.linker.link(src, dst, link_type=self.pool.link_type)
+
+
+class Linker(kobo.log.LoggingBase):
+    def __init__(self, always_copy=None, test=False, logger=None):
+        kobo.log.LoggingBase.__init__(self, logger=logger)
+        self.always_copy = always_copy or []
+        self.test = test
+        self._inode_map = {}
+
+    def _is_same_type(self, path1, path2):
+        if not os.path.islink(path1) == os.path.islink(path2):
+            return False
+        if not os.path.isdir(path1) == os.path.isdir(path2):
+            return False
+        if not os.path.isfile(path1) == os.path.isfile(path2):
+            return False
+        return True
+
+    def _is_same(self, path1, path2):
+        if path1 == path2:
+            return True
+        if os.path.islink(path2) and not os.path.exists(path2):
+            # Broken symlink
+            return True
+        if os.path.getsize(path1) != os.path.getsize(path2):
+            return False
+        if int(os.path.getmtime(path1)) != int(os.path.getmtime(path2)):
+            return False
+        return True
+
+    def symlink(self, src, dst, relative=True):
+        if src == dst:
+            return
+
+        # Always hardlink or copy scratch builds
+        if "/work/tasks/" in src:
+            self._link_file(src, dst, "hardlink-or-copy")
+
+        old_src = src
+        if relative:
+            src = relative_path(src, dst)
+
+        msg = "Symlinking %s -> %s" % (dst, src)
+        if self.test:
+            self.log_info("TEST: %s" % msg)
+            return
+        self.log_info(msg)
+
+        try:
+            os.symlink(src, dst)
+        except OSError as ex:
+            if ex.errno != errno.EEXIST:
+                raise
+            if os.path.islink(dst) and self._is_same(old_src, dst):
+                if os.readlink(dst) != src:
+                    raise
+                self.log_debug(
+                    "The same file already exists, skipping symlink %s -> %s"
+                    % (dst, src)
+                )
+            else:
+                raise
+
+    def hardlink(self, src, dst):
+        if src == dst:
+            return
+
+        msg = "Hardlinking %s to %s" % (src, dst)
+        if self.test:
+            self.log_info("TEST: %s" % msg)
+            return
+        self.log_info(msg)
+
+        try:
+            os.link(src, dst)
+        except OSError as ex:
+            if ex.errno != errno.EEXIST:
+                raise
+            if self._is_same(src, dst):
+                if not self._is_same_type(src, dst):
+                    self.log_error(
+                        "File %s already exists but has different type than %s"
+                        % (dst, src)
+                    )
+                    raise
+                self.log_debug(
+                    "The same file already exists, skipping hardlink %s to %s"
+                    % (src, dst)
+                )
+            else:
+                raise
+
+    def copy(self, src, dst):
+        if src == dst:
+            return True
+
+        if os.path.islink(src):
+            msg = "Copying symlink %s to %s" % (src, dst)
+        else:
+            msg = "Copying file %s to %s" % (src, dst)
+
+        if self.test:
+            self.log_info("TEST: %s" % msg)
+            return
+        self.log_info(msg)
+
+        if os.path.exists(dst):
+            if self._is_same(src, dst):
+                if not self._is_same_type(src, dst):
+                    self.log_error(
+                        "File %s already exists but has different type than %s"
+                        % (dst, src)
+                    )
+                    raise OSError(errno.EEXIST, "File exists")
+                self.log_debug(
+                    "The same file already exists, skipping copy %s to %s" % (src, dst)
+                )
+                return
+            else:
+                raise OSError(errno.EEXIST, "File exists")
+
+        if os.path.islink(src):
+            if not os.path.islink(dst):
+                os.symlink(os.readlink(src), dst)
+                return
+            return
+
+        src_stat = os.stat(src)
+        src_key = (src_stat.st_dev, src_stat.st_ino)
+        if src_key in self._inode_map:
+            # (st_dev, st_ino) found in the mapping
+            self.log_debug(
+                "Harlink detected, hardlinking in destination %s to %s"
+                % (self._inode_map[src_key], dst)
+            )
+            os.link(self._inode_map[src_key], dst)
+            return
+
+        # BEWARE: shutil.copy2 automatically *rewrites* existing files
+        shutil.copy2(src, dst)
+        self._inode_map[src_key] = dst
+
+    def _link_file(self, src, dst, link_type):
+        if link_type == "hardlink":
+            self.hardlink(src, dst)
+        elif link_type == "copy":
+            self.copy(src, dst)
+        elif link_type in ("symlink", "abspath-symlink"):
+            if os.path.islink(src):
+                self.copy(src, dst)
+            else:
+                relative = link_type != "abspath-symlink"
+                self.symlink(src, dst, relative)
+        elif link_type == "hardlink-or-copy":
+            try:
+                self.hardlink(src, dst)
+            except OSError as ex:
+                if ex.errno == errno.EXDEV:
+                    self.copy(src, dst)
+                else:
+                    raise
+        else:
+            raise ValueError("Unknown link_type: %s" % link_type)
+
+    def link(self, src, dst, link_type="hardlink-or-copy"):
+        """Link directories recursively."""
+        if os.path.isfile(src) or os.path.islink(src):
+            self._link_file(src, dst, link_type)
+            return
+
+        if os.path.isfile(dst):
+            raise OSError(errno.EEXIST, "File exists")
+
+        if not self.test:
+            if not os.path.exists(dst):
+                makedirs(dst)
+            shutil.copystat(src, dst)
+
+        for i in os.listdir(src):
+            src_path = os.path.join(src, i)
+            dst_path = os.path.join(dst, i)
+            self.link(src_path, dst_path, link_type)

pungi/media_split.py

@@ -0,0 +1,127 @@
+# -*- coding: utf-8 -*-
+
+
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; version 2 of the License.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Library General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, see <https://gnu.org/licenses/>.
+
+
+import os
+
+
+SIZE_UNITS = {
+    "b": 1,
+    "k": 1024,
+    "M": 1024**2,
+    "G": 1024**3,
+}
+
+
+def convert_media_size(size):
+    if isinstance(size, str):
+        if size[-1] in SIZE_UNITS:
+            num = int(size[:-1])
+            units = size[-1]
+        else:
+            num = int(size)
+            units = "b"
+        result = num * SIZE_UNITS[units]
+    else:
+        result = int(size)
+
+    if result <= 0:
+        raise ValueError("Media size must be a positive number: %s" % size)
+
+    return result
+
+
+def convert_file_size(size, block_size=2048):
+    """round file size to block"""
+    blocks = int(size / block_size)
+    if size % block_size:
+        blocks += 1
+    return blocks * block_size
+
+
+class MediaSplitter(object):
+    """
+    MediaSplitter splits files so that they fit on a media of given size.
+
+    Each file added to the spliter has a size in bytes that will be rounded to
+    the nearest multiple of block size. If the file is sticky, it will be
+    included on each disk. The files will be on disks in the same order they
+    are added; there is no re-ordering. The number of disk is thus not the
+    possible minimum.
+    """
+
+    def __init__(self, media_size, compose=None, logger=None):
+        self.media_size = media_size
+        self.files = []  # to preserve order
+        self.file_sizes = {}
+        self.sticky_files = set()
+        self.compose = compose
+        self.logger = logger
+        if not self.logger and self.compose:
+            self.logger = self.compose._logger
+
+    def add_file(self, name, size, sticky=False):
+        name = os.path.normpath(name)
+        size = int(size)
+        old_size = self.file_sizes.get(name, None)
+
+        if old_size is not None and old_size != size:
+            raise ValueError(
+                "File size mismatch; file: %s; sizes: %s vs %s" % (name, old_size, size)
+            )
+        if self.media_size and size > self.media_size:
+            raise ValueError("File is larger than media size: %s" % name)
+
+        self.files.append(name)
+        self.file_sizes[name] = size
+        if sticky:
+            self.sticky_files.add(name)
+
+    @property
+    def total_size(self):
+        return sum(self.file_sizes.values())
+
+    @property
+    def total_size_in_blocks(self):
+        return sum([convert_file_size(i) for i in list(self.file_sizes.values())])
+
+    def split(self, first_disk=0, all_disks=0):
+        all_files = []
+        sticky_files = []
+        sticky_files_size = 0
+
+        for name in self.files:
+            if name in self.sticky_files:
+                sticky_files.append(name)
+                sticky_files_size += convert_file_size(self.file_sizes[name])
+            else:
+                all_files.append(name)
+
+        disks = []
+        disk = {}
+        # as it would be on single medium (sticky_files just once)
+        total_size_single = sticky_files_size
+        while all_files:
+            name = all_files.pop(0)
+            size = convert_file_size(self.file_sizes[name])
+
+            if not disks or (self.media_size and disk["size"] + size > self.media_size):
+                disk = {"size": sticky_files_size, "files": sticky_files[:]}
+                disks.append(disk)
+
+            disk["files"].append(name)
+            disk["size"] += size
+            total_size_single += size
+        return disks

pungi/metadata.py

@@ -0,0 +1,537 @@
+# -*- coding: utf-8 -*-
+
+
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; version 2 of the License.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Library General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, see <https://gnu.org/licenses/>.
+
+
+import copy
+import os
+import time
+
+import productmd.composeinfo
+import productmd.treeinfo
+from productmd.common import get_major_version
+from kobo.shortcuts import relative_path, compute_file_checksums
+
+from pungi.compose_metadata.discinfo import write_discinfo as create_discinfo
+from pungi.compose_metadata.discinfo import write_media_repo as create_media_repo
+
+
+def get_description(compose, variant, arch):
+    if "release_discinfo_description" in compose.conf:
+        result = compose.conf["release_discinfo_description"]
+    elif variant.type == "layered-product":
+        # we need to make sure the layered product behaves as it was composed separately
+        result = "%s %s for %s %s" % (
+            variant.release_name,
+            variant.release_version,
+            compose.conf["release_name"],
+            get_major_version(compose.conf["release_version"]),
+        )
+    else:
+        result = "%s %s" % (
+            compose.conf["release_name"],
+            compose.conf["release_version"],
+        )
+        if compose.conf.get("base_product_name", ""):
+            result += " for %s %s" % (
+                compose.conf["base_product_name"],
+                compose.conf["base_product_version"],
+            )
+
+    result = result % {"variant_name": variant.name, "arch": arch}
+    return result
+
+
+def write_discinfo(compose, arch, variant):
+    if variant.type == "addon":
+        return
+    os_tree = compose.paths.compose.os_tree(arch, variant)
+    path = os.path.join(os_tree, ".discinfo")
+    # description = get_volid(compose, arch, variant)
+    description = get_description(compose, variant, arch)
+    return create_discinfo(path, description, arch)
+
+
+def write_media_repo(compose, arch, variant, timestamp=None):
+    if variant.type == "addon":
+        return
+    os_tree = compose.paths.compose.os_tree(arch, variant)
+    path = os.path.join(os_tree, "media.repo")
+    # description = get_volid(compose, arch, variant)
+    description = get_description(compose, variant, arch)
+    return create_media_repo(path, description, timestamp)
+
+
+def compose_to_composeinfo(compose):
+    ci = productmd.composeinfo.ComposeInfo()
+
+    # compose
+    ci.compose.id = compose.compose_id
+    ci.compose.type = compose.compose_type
+    ci.compose.date = compose.compose_date
+    ci.compose.respin = compose.compose_respin
+    ci.compose.label = compose.compose_label
+    ci.compose.final = compose.supported
+
+    # product
+    ci.release.name = compose.conf["release_name"]
+    ci.release.version = compose.conf["release_version"]
+    ci.release.short = compose.conf["release_short"]
+    ci.release.is_layered = True if compose.conf.get("base_product_name", "") else False
+    ci.release.type = compose.conf["release_type"].lower()
+    ci.release.internal = bool(compose.conf["release_internal"])
+
+    # base product
+    if ci.release.is_layered:
+        ci.base_product.name = compose.conf["base_product_name"]
+        ci.base_product.version = compose.conf["base_product_version"]
+        ci.base_product.short = compose.conf["base_product_short"]
+        ci.base_product.type = compose.conf["base_product_type"].lower()
+
+    def dump_variant(variant, parent=None):
+        var = productmd.composeinfo.Variant(ci)
+
+        tree_arches = compose.conf.get("tree_arches")
+        if tree_arches and not (set(variant.arches) & set(tree_arches)):
+            return None
+
+        # variant details
+        # remove dashes from variant ID, rely on productmd verification
+        var.id = variant.id.replace("-", "")
+        var.uid = variant.uid
+        var.name = variant.name
+        var.type = variant.type
+        var.arches = set(variant.arches)
+
+        if var.type == "layered-product":
+            var.release.name = variant.release_name
+            var.release.short = variant.release_short
+            var.release.version = variant.release_version
+            var.release.is_layered = True
+            var.release.type = ci.release.type
+
+        for arch in variant.arches:
+            # paths: binaries
+            var.paths.os_tree[arch] = relative_path(
+                compose.paths.compose.os_tree(
+                    arch=arch, variant=variant, create_dir=False
+                ).rstrip("/")
+                + "/",
+                compose.paths.compose.topdir().rstrip("/") + "/",
+            ).rstrip("/")
+            var.paths.repository[arch] = relative_path(
+                compose.paths.compose.repository(
+                    arch=arch, variant=variant, create_dir=False
+                ).rstrip("/")
+                + "/",
+                compose.paths.compose.topdir().rstrip("/") + "/",
+            ).rstrip("/")
+            var.paths.packages[arch] = relative_path(
+                compose.paths.compose.packages(
+                    arch=arch, variant=variant, create_dir=False
+                ).rstrip("/")
+                + "/",
+                compose.paths.compose.topdir().rstrip("/") + "/",
+            ).rstrip("/")
+            iso_dir = (
+                compose.paths.compose.iso_dir(
+                    arch=arch, variant=variant, create_dir=False
+                )
+                or ""
+            )
+            if iso_dir and os.path.isdir(
+                os.path.join(compose.paths.compose.topdir(), iso_dir)
+            ):
+                var.paths.isos[arch] = relative_path(
+                    iso_dir, compose.paths.compose.topdir().rstrip("/") + "/"
+                ).rstrip("/")
+            image_dir = compose.paths.compose.image_dir(variant=variant) or ""
+            if image_dir:
+                image_dir = image_dir % {"arch": arch}
+                if os.path.isdir(image_dir):
+                    var.paths.images[arch] = relative_path(
+                        image_dir, compose.paths.compose.topdir().rstrip("/") + "/"
+                    ).rstrip("/")
+            jigdo_dir = (
+                compose.paths.compose.jigdo_dir(
+                    arch=arch, variant=variant, create_dir=False
+                )
+                or ""
+            )
+            if jigdo_dir and os.path.isdir(
+                os.path.join(compose.paths.compose.topdir(), jigdo_dir)
+            ):
+                var.paths.jigdos[arch] = relative_path(
+                    jigdo_dir, compose.paths.compose.topdir().rstrip("/") + "/"
+                ).rstrip("/")
+
+            # paths: sources
+            var.paths.source_tree[arch] = relative_path(
+                compose.paths.compose.os_tree(
+                    arch="source", variant=variant, create_dir=False
+                ).rstrip("/")
+                + "/",
+                compose.paths.compose.topdir().rstrip("/") + "/",
+            ).rstrip("/")
+            var.paths.source_repository[arch] = relative_path(
+                compose.paths.compose.repository(
+                    arch="source", variant=variant, create_dir=False
+                ).rstrip("/")
+                + "/",
+                compose.paths.compose.topdir().rstrip("/") + "/",
+            ).rstrip("/")
+            var.paths.source_packages[arch] = relative_path(
+                compose.paths.compose.packages(
+                    arch="source", variant=variant, create_dir=False
+                ).rstrip("/")
+                + "/",
+                compose.paths.compose.topdir().rstrip("/") + "/",
+            ).rstrip("/")
+            source_iso_dir = (
+                compose.paths.compose.iso_dir(
+                    arch="source", variant=variant, create_dir=False
+                )
+                or ""
+            )
+            if source_iso_dir and os.path.isdir(
+                os.path.join(compose.paths.compose.topdir(), source_iso_dir)
+            ):
+                var.paths.source_isos[arch] = relative_path(
+                    source_iso_dir, compose.paths.compose.topdir().rstrip("/") + "/"
+                ).rstrip("/")
+            source_jigdo_dir = (
+                compose.paths.compose.jigdo_dir(
+                    arch="source", variant=variant, create_dir=False
+                )
+                or ""
+            )
+            if source_jigdo_dir and os.path.isdir(
+                os.path.join(compose.paths.compose.topdir(), source_jigdo_dir)
+            ):
+                var.paths.source_jigdos[arch] = relative_path(
+                    source_jigdo_dir, compose.paths.compose.topdir().rstrip("/") + "/"
+                ).rstrip("/")
+
+            # paths: debug
+            var.paths.debug_tree[arch] = relative_path(
+                compose.paths.compose.debug_tree(
+                    arch=arch, variant=variant, create_dir=False
+                ).rstrip("/")
+                + "/",
+                compose.paths.compose.topdir().rstrip("/") + "/",
+            ).rstrip("/")
+            var.paths.debug_repository[arch] = relative_path(
+                compose.paths.compose.debug_repository(
+                    arch=arch, variant=variant, create_dir=False
+                ).rstrip("/")
+                + "/",
+                compose.paths.compose.topdir().rstrip("/") + "/",
+            ).rstrip("/")
+            var.paths.debug_packages[arch] = relative_path(
+                compose.paths.compose.debug_packages(
+                    arch=arch, variant=variant, create_dir=False
+                ).rstrip("/")
+                + "/",
+                compose.paths.compose.topdir().rstrip("/") + "/",
+            ).rstrip("/")
+            """
+            # XXX: not supported (yet?)
+            debug_iso_dir = (
+                compose.paths.compose.debug_iso_dir(arch=arch, variant=variant) or ""
+            )
+            if debug_iso_dir:
+                var.debug_iso_dir[arch] = relative_path(
+                    debug_iso_dir, compose.paths.compose.topdir().rstrip("/") + "/"
+                ).rstrip("/")
+            debug_jigdo_dir = (
+                compose.paths.compose.debug_jigdo_dir(arch=arch, variant=variant) or ""
+            )
+            if debug_jigdo_dir:
+                var.debug_jigdo_dir[arch] = relative_path(
+                    debug_jigdo_dir, compose.paths.compose.topdir().rstrip("/") + "/"
+                ).rstrip("/")
+            """
+
+        for v in variant.get_variants(recursive=False):
+            x = dump_variant(v, parent=variant)
+            if x is not None:
+                var.add(x)
+        return var
+
+    for variant_id in sorted(compose.variants):
+        variant = compose.variants[variant_id]
+        v = dump_variant(variant)
+        if v is not None:
+            ci.variants.add(v)
+    return ci
+
+
+def write_compose_info(compose):
+    ci = compose_to_composeinfo(compose)
+
+    msg = "Writing composeinfo"
+    compose.log_info("[BEGIN] %s" % msg)
+
+    path = compose.paths.compose.metadata("composeinfo.json")
+    # make a copy of composeinfo and modify the copy
+    # if any path in variant paths doesn't exist or just an empty
+    # dir, set it to None, then it won't be dumped.
+    ci_copy = copy.deepcopy(ci)
+    for variant in ci_copy.variants.variants.values():
+        for field in variant.paths._fields:
+            field_paths = getattr(variant.paths, field)
+            for arch, dirpath in field_paths.items():
+                dirpath = os.path.join(compose.paths.compose.topdir(), dirpath)
+                if not os.path.isdir(dirpath):
+                    # If the directory does not exist, do not include the path
+                    # in metadata.
+                    field_paths[arch] = None
+    ci_copy.dump(path)
+
+    compose.log_info("[DONE ] %s" % msg)
+
+
+def write_tree_info(compose, arch, variant, timestamp=None, bi=None):
+    if variant.type in ("addon",) or variant.is_empty:
+        return
+
+    if not timestamp:
+        timestamp = int(time.time())
+    else:
+        timestamp = int(timestamp)
+
+    os_tree = (
+        compose.paths.compose.os_tree(arch=arch, variant=variant).rstrip("/") + "/"
+    )
+
+    ti = productmd.treeinfo.TreeInfo()
+    # load from buildinstall .treeinfo
+
+    if variant.type == "layered-product":
+        # we need to make sure the layered product behaves as it was composed separately
+
+        # release
+        # TODO: read from variants.xml
+        ti.release.name = variant.release_name
+        ti.release.version = variant.release_version
+        ti.release.short = variant.release_short
+        ti.release.is_layered = True
+        ti.release.type = compose.conf["release_type"].lower()
+
+        # base product
+        ti.base_product.name = compose.conf["release_name"]
+        if "." in compose.conf["release_version"]:
+            # remove minor version if present
+            ti.base_product.version = get_major_version(compose.conf["release_version"])
+        else:
+            ti.base_product.version = compose.conf["release_version"]
+        ti.base_product.short = compose.conf["release_short"]
+    else:
+        # release
+        ti.release.name = compose.conf["release_name"]
+        ti.release.version = compose.conf.get(
+            "treeinfo_version", compose.conf["release_version"]
+        )
+        ti.release.short = compose.conf["release_short"]
+        ti.release.is_layered = (
+            True if compose.conf.get("base_product_name", "") else False
+        )
+        ti.release.type = compose.conf["release_type"].lower()
+
+        # base product
+        if ti.release.is_layered:
+            ti.base_product.name = compose.conf["base_product_name"]
+            ti.base_product.version = compose.conf["base_product_version"]
+            ti.base_product.short = compose.conf["base_product_short"]
+
+    # tree
+    ti.tree.arch = arch
+    ti.tree.build_timestamp = timestamp
+    # ti.platforms
+
+    # main variant
+    var = productmd.treeinfo.Variant(ti)
+    if variant.type == "layered-product":
+        var.id = variant.parent.id
+        var.uid = variant.parent.uid
+        var.name = variant.parent.name
+        var.type = "variant"
+    else:
+        # remove dashes from variant ID, rely on productmd verification
+        var.id = variant.id.replace("-", "")
+        var.uid = variant.uid
+        var.name = variant.name
+        var.type = variant.type
+
+    var.paths.packages = (
+        relative_path(
+            compose.paths.compose.packages(
+                arch=arch, variant=variant, create_dir=False
+            ).rstrip("/")
+            + "/",
+            os_tree,
+        ).rstrip("/")
+        or "."
+    )
+    var.paths.repository = (
+        relative_path(
+            compose.paths.compose.repository(
+                arch=arch, variant=variant, create_dir=False
+            ).rstrip("/")
+            + "/",
+            os_tree,
+        ).rstrip("/")
+        or "."
+    )
+
+    ti.variants.add(var)
+
+    repomd_path = os.path.join(var.paths.repository, "repodata", "repomd.xml")
+    createrepo_checksum = compose.conf["createrepo_checksum"]
+    if os.path.isfile(repomd_path):
+        ti.checksums.add(repomd_path, createrepo_checksum, root_dir=os_tree)
+
+    for i in variant.get_variants(types=["addon"], arch=arch):
+        addon = productmd.treeinfo.Variant(ti)
+        addon.id = i.id
+        addon.uid = i.uid
+        addon.name = i.name
+        addon.type = i.type
+        compose.log_debug(
+            "variant '%s' inserting addon uid '%s' type '%s'"
+            % (variant, addon.uid, addon.type)
+        )
+
+        os_tree = compose.paths.compose.os_tree(arch=arch, variant=i).rstrip("/") + "/"
+        addon.paths.packages = (
+            relative_path(
+                compose.paths.compose.packages(
+                    arch=arch, variant=i, create_dir=False
+                ).rstrip("/")
+                + "/",
+                os_tree,
+            ).rstrip("/")
+            or "."
+        )
+        addon.paths.repository = (
+            relative_path(
+                compose.paths.compose.repository(
+                    arch=arch, variant=i, create_dir=False
+                ).rstrip("/")
+                + "/",
+                os_tree,
+            ).rstrip("/")
+            or "."
+        )
+        var.add(addon)
+
+        repomd_path = os.path.join(addon.paths.repository, "repodata", "repomd.xml")
+        if os.path.isfile(repomd_path):
+            ti.checksums.add(repomd_path, createrepo_checksum, root_dir=os_tree)
+
+    class LoraxProduct(productmd.treeinfo.Release):
+        def _validate_short(self):
+            # HACK: set self.short so .treeinfo produced by lorax can be read
+            if not self.short:
+                self.short = compose.conf["release_short"]
+
+    class LoraxTreeInfo(productmd.treeinfo.TreeInfo):
+        def __init__(self, *args, **kwargs):
+            super(LoraxTreeInfo, self).__init__(*args, **kwargs)
+            self.release = LoraxProduct(self)
+
+    # images
+    if variant.type == "variant" and bi.succeeded(variant, arch):
+        os_tree = compose.paths.compose.os_tree(arch, variant)
+
+        # clone all but 'general' sections from buildinstall .treeinfo
+
+        bi_dir = compose.paths.work.buildinstall_dir(arch)
+        if compose.conf.get("buildinstall_method") == "lorax":
+            # The .treeinfo file produced by lorax is nested in variant
+            # subdirectory. Legacy buildinstall runs once per arch, so there is
+            # only one file.
+            bi_dir = os.path.join(bi_dir, variant.uid)
+        bi_treeinfo = os.path.join(bi_dir, ".treeinfo")
+
+        if os.path.exists(bi_treeinfo):
+            bi_ti = LoraxTreeInfo()
+            bi_ti.load(bi_treeinfo)
+
+            # stage2 - mainimage
+            if bi_ti.stage2.mainimage:
+                ti.stage2.mainimage = bi_ti.stage2.mainimage
+                ti.checksums.add(
+                    ti.stage2.mainimage, createrepo_checksum, root_dir=os_tree
+                )
+
+            # stage2 - instimage
+            if bi_ti.stage2.instimage:
+                ti.stage2.instimage = bi_ti.stage2.instimage
+                ti.checksums.add(
+                    ti.stage2.instimage, createrepo_checksum, root_dir=os_tree
+                )
+
+            # images
+            for platform in bi_ti.images.images:
+                ti.images.images[platform] = {}
+                ti.tree.platforms.add(platform)
+                for image, path in bi_ti.images.images[platform].items():
+                    if not path:
+                        # The .treeinfo file contains an image without a path.
+                        # We can't add that.
+                        continue
+                    ti.images.images[platform][image] = path
+                    ti.checksums.add(path, createrepo_checksum, root_dir=os_tree)
+
+    path = os.path.join(
+        compose.paths.compose.os_tree(arch=arch, variant=variant), ".treeinfo"
+    )
+    compose.log_info("Writing treeinfo: %s" % path)
+    ti.dump(path)
+
+
+def populate_extra_files_metadata(
+    metadata, variant, arch, topdir, files, checksum_types, relative_root=None
+):
+    """
+    :param metadata: an instance of productmd.extra_files.ExtraFiles to
+                     populate with the current files
+    :param Variant variant: under which variant should the files be listed
+    :param str arch: under which arch should the files be listed
+    :param topdir: directory where files are located
+    :param files: list of file paths relative to topdir
+    :param checksum_types: list of checksums to compute
+    :param relative_root: ancestor directory of topdir, this will be removed
+                          from paths written to local metadata file
+    """
+    for copied_file in files:
+        full_path = os.path.join(topdir, copied_file)
+        size = os.path.getsize(full_path)
+        try:
+            checksums = compute_file_checksums(full_path, checksum_types)
+        except IOError as exc:
+            raise RuntimeError(
+                "Failed to calculate checksum for %s: %s" % (full_path, exc)
+            )
+
+        if relative_root:
+            copied_file = os.path.relpath(full_path, relative_root)
+        metadata.add(variant.uid, arch, copied_file, size, checksums)
+
+    strip_prefix = (
+        (os.path.relpath(topdir, relative_root) + "/") if relative_root else ""
+    )
+    with open(os.path.join(topdir, "extra_files.json"), "w") as f:
+        metadata.dump_for_tree(f, variant.uid, arch, strip_prefix)

pungi/module_util.py

@@ -0,0 +1,118 @@
+# -*- coding: utf-8 -*-
+
+
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; version 2 of the License.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Library General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, see <https://gnu.org/licenses/>.
+
+import glob
+import os
+
+try:
+    import gi
+
+    gi.require_version("Modulemd", "2.0")  # noqa
+    from gi.repository import Modulemd
+except (ImportError, ValueError):
+    Modulemd = None
+
+
+def iter_module_defaults(path):
+    """Given a path to a directory with yaml files, yield each module default
+    in there as a pair (module_name, ModuleDefaults instance).
+    """
+    # It is really tempting to merge all the module indexes into a single one
+    # and work with it. However that does not allow for detecting conflicting
+    # defaults. That should not happen in practice, but better safe than sorry.
+    # Once libmodulemd can report the error, this code can be simplifed by a
+    # lot. It was implemented in
+    # https://github.com/fedora-modularity/libmodulemd/commit/3087e4a5c38a331041fec9b6b8f1a372f9ffe64d
+    # and released in 2.6.0, but 2.8.0 added the need to merge overrides and
+    # that breaks this use case again.
+    for file in glob.glob(os.path.join(path, "*.yaml")):
+        index = Modulemd.ModuleIndex()
+        index.update_from_file(file, strict=False)
+        for module_name in index.get_module_names():
+            yield module_name, index.get_module(module_name).get_defaults()
+
+
+def get_module_obsoletes_idx(path, mod_list):
+    """Given a path to a directory with yaml files, return Index with
+    merged all obsoletes.
+    """
+
+    merger = Modulemd.ModuleIndexMerger.new()
+    md_idxs = []
+
+    # associate_index does NOT copy it's argument (nor increases a
+    # reference counter on the object). It only stores a pointer.
+    for file in glob.glob(os.path.join(path, "*.yaml")):
+        index = Modulemd.ModuleIndex()
+        index.update_from_file(file, strict=False)
+        mod_name = index.get_module_names()[0]
+
+        if mod_name and (mod_name in mod_list or not mod_list):
+            md_idxs.append(index)
+            merger.associate_index(md_idxs[-1], 0)
+
+    merged_idx = merger.resolve()
+
+    return merged_idx
+
+
+def collect_module_defaults(
+    defaults_dir, modules_to_load=None, mod_index=None, overrides_dir=None
+):
+    """Load module defaults into index.
+
+    If `modules_to_load` is passed in, it should be a set of module names. Only
+    defaults for these modules will be loaded.
+
+    If `mod_index` is passed in, it will be updated and returned. If it was
+    not, a new ModuleIndex will be created and returned
+    """
+    mod_index = mod_index or Modulemd.ModuleIndex()
+
+    temp_index = Modulemd.ModuleIndex.new()
+    temp_index.update_from_defaults_directory(
+        defaults_dir, overrides_path=overrides_dir, strict=False
+    )
+
+    for module_name in temp_index.get_module_names():
+        defaults = temp_index.get_module(module_name).get_defaults()
+
+        if not modules_to_load or module_name in modules_to_load:
+            mod_index.add_defaults(defaults)
+
+    return mod_index
+
+
+def collect_module_obsoletes(obsoletes_dir, modules_to_load, mod_index=None):
+    """Load module obsoletes into index.
+
+    This works in a similar fashion as collect_module_defaults except it
+    merges indexes together instead of adding them during iteration.
+
+    Additionally if modules_to_load is not empty returned Index will include
+    only obsoletes for those modules.
+    """
+
+    obsoletes_index = get_module_obsoletes_idx(obsoletes_dir, modules_to_load)
+
+    # Merge Obsoletes with Modules Index.
+    if mod_index:
+        merger = Modulemd.ModuleIndexMerger.new()
+        merger.associate_index(mod_index, 0)
+        merger.associate_index(obsoletes_index, 0)
+        merged_idx = merger.resolve()
+        obsoletes_index = merged_idx
+
+    return obsoletes_index

pungi/multilib_dnf.py

@@ -0,0 +1,67 @@
+# -*- coding: utf-8 -*-
+
+
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; version 2 of the License.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Library General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, see <https://gnu.org/licenses/>.
+
+from multilib import multilib
+
+
+class Multilib(object):
+    """This class decides whether a package should be multilib.
+
+    To use it, create an instance and call the ``is_multilib`` method on it.
+    The blacklist and whitelist in constructor should be sets of package names.
+
+    It may be more convenient to create the instance with the ``from_globs``
+    method that accepts a DNF sach and an iterable of globs that will be used
+    to find package names.
+    """
+
+    def __init__(self, methods, blacklist, whitelist):
+        self.methods = {}
+        self.blacklist = blacklist
+        self.whitelist = whitelist
+
+        self.all_methods = {
+            "none": multilib.NoMultilibMethod(None),
+            "all": multilib.AllMultilibMethod(None),
+            "devel": multilib.DevelMultilibMethod(None),
+            "runtime": multilib.RuntimeMultilibMethod(None),
+        }
+
+        for method in methods:
+            self.methods[method] = self.all_methods[method]
+
+    @classmethod
+    def from_globs(cls, sack, methods, blacklist=None, whitelist=None):
+        """Create a Multilib instance with expanded blacklist and whitelist."""
+        return cls(
+            methods,
+            _expand_list(sack, blacklist or []),
+            _expand_list(sack, whitelist or []),
+        )
+
+    def is_multilib(self, pkg):
+        if pkg.name in self.blacklist:
+            return False
+        if pkg.name in self.whitelist:
+            return "whitelist"
+        for method, cls in self.methods.items():
+            if cls.select(pkg):
+                return method
+        return False
+
+
+def _expand_list(sack, patterns):
+    """Find all package names that match any of the provided patterns."""
+    return set(pkg.name for pkg in sack.query().filter(name__glob=list(patterns)))

pungi/multilib_yum.py

@@ -0,0 +1,295 @@
+# -*- coding: utf-8 -*-
+
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; version 2 of the License.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Library General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, see <https://gnu.org/licenses/>.
+
+
+import re
+import fnmatch
+
+import pungi.pathmatch
+import pungi.gather
+import pungi.util
+
+
+LINE_PATTERN_RE = re.compile(r"^\s*(?P<line>[^#]+)(:?\s+(?P<comment>#.*))?$")
+RUNTIME_PATTERN_SPLIT_RE = re.compile(
+    r"^\s*(?P<path>[^\s]+)\s+(?P<pattern>[^\s]+)(:?\s+(?P<comment>#.*))?$"
+)
+SONAME_PATTERN_RE = re.compile(r"^(.+\.so\.[a-zA-Z0-9_\.]+).*$")
+
+
+def read_lines(lines):
+    result = []
+    for i in lines:
+        i = i.strip()
+
+        if not i:
+            continue
+
+        # skip comments
+        if i.startswith("#"):
+            continue
+
+        match = LINE_PATTERN_RE.match(i)
+        if match is None:
+            raise ValueError("Couldn't parse line: %s" % i)
+        gd = match.groupdict()
+        result.append(gd["line"])
+    return result
+
+
+def read_lines_from_file(path):
+    lines = open(path, "r").readlines()
+    lines = read_lines(lines)
+    return lines
+
+
+def read_runtime_patterns(lines):
+    result = []
+    for i in read_lines(lines):
+        match = RUNTIME_PATTERN_SPLIT_RE.match(i)
+        if match is None:
+            raise ValueError("Couldn't parse pattern: %s" % i)
+        gd = match.groupdict()
+        result.append((gd["path"], gd["pattern"]))
+    return result
+
+
+def read_runtime_patterns_from_file(path):
+    lines = open(path, "r").readlines()
+    return read_runtime_patterns(lines)
+
+
+def expand_runtime_patterns(patterns):
+    pm = pungi.pathmatch.PathMatch()
+    for path, pattern in patterns:
+        for root in ("", "/opt/*/*/root"):
+            # include Software Collections: /opt/<vendor>/<scl_name>/root/...
+            if "$LIBDIR" in path:
+                for lib_dir in ("/lib", "/lib64", "/usr/lib", "/usr/lib64"):
+                    path_pattern = path.replace("$LIBDIR", lib_dir)
+                    path_pattern = "%s/%s" % (root, path_pattern.lstrip("/"))
+                    pm[path_pattern] = (path_pattern, pattern)
+            else:
+                path_pattern = "%s/%s" % (root, path.lstrip("/"))
+                pm[path_pattern] = (path_pattern, pattern)
+    return pm
+
+
+class MultilibMethodBase(object):
+    """a base class for multilib methods"""
+
+    name = "base"
+
+    def __init__(self, config_path):
+        self.config_path = config_path
+
+    def select(self, po):
+        raise NotImplementedError
+
+    def skip(self, po):
+        if (
+            pungi.gather.is_noarch(po)
+            or pungi.gather.is_source(po)
+            or pungi.util.pkg_is_debug(po)
+        ):
+            return True
+        return False
+
+    def is_kernel(self, po):
+        for p_name, p_flag, (p_e, p_v, p_r) in po.provides:
+            if p_name == "kernel":
+                return True
+        return False
+
+    def is_kernel_devel(self, po):
+        for p_name, p_flag, (p_e, p_v, p_r) in po.provides:
+            if p_name == "kernel-devel":
+                return True
+        return False
+
+    def is_kernel_or_kernel_devel(self, po):
+        for p_name, p_flag, (p_e, p_v, p_r) in po.provides:
+            if p_name in ("kernel", "kernel-devel"):
+                return True
+        return False
+
+
+class NoneMultilibMethod(MultilibMethodBase):
+    """multilib disabled"""
+
+    name = "none"
+
+    def select(self, po):
+        return False
+
+
+class AllMultilibMethod(MultilibMethodBase):
+    """all packages are multilib"""
+
+    name = "all"
+
+    def select(self, po):
+        if self.skip(po):
+            return False
+        return True
+
+
+class RuntimeMultilibMethod(MultilibMethodBase):
+    """pre-defined paths to libs"""
+
+    name = "runtime"
+
+    def __init__(self, *args, **kwargs):
+        super(RuntimeMultilibMethod, self).__init__(*args, **kwargs)
+        self.blacklist = read_lines_from_file(
+            self.config_path + "runtime-blacklist.conf"
+        )
+        self.whitelist = read_lines_from_file(
+            self.config_path + "runtime-whitelist.conf"
+        )
+        self.patterns = expand_runtime_patterns(
+            read_runtime_patterns_from_file(self.config_path + "runtime-patterns.conf")
+        )
+
+    def select(self, po):
+        if self.skip(po):
+            return False
+        if po.name in self.blacklist:
+            return False
+        if po.name in self.whitelist:
+            return True
+        if self.is_kernel(po):
+            return False
+
+        # gather all *.so.* provides from the RPM header
+        provides = set()
+        for i in po.provides:
+            match = SONAME_PATTERN_RE.match(i[0])
+            if match is not None:
+                provides.add(match.group(1))
+
+        for path in po.returnFileEntries() + po.returnFileEntries("ghost"):
+            dirname, filename = path.rsplit("/", 1)
+            dirname = dirname.rstrip("/")
+
+            patterns = self.patterns[dirname]
+            if not patterns:
+                continue
+            for dir_pattern, file_pattern in patterns:
+                if file_pattern == "-":
+                    return True
+                if fnmatch.fnmatch(filename, file_pattern):
+                    if ".so.*" in file_pattern:
+                        if filename in provides:
+                            # return only if the lib is provided in RPM header
+                            # (some libs may be private, hence not exposed in Provides)
+                            return True
+                    else:
+                        return True
+        return False
+
+
+class KernelMultilibMethod(MultilibMethodBase):
+    """kernel and kernel-devel"""
+
+    name = "kernel"
+
+    def __init__(self, *args, **kwargs):
+        super(KernelMultilibMethod, self).__init__(*args, **kwargs)
+
+    def select(self, po):
+        if self.is_kernel_or_kernel_devel(po):
+            return True
+        return False
+
+
+class YabootMultilibMethod(MultilibMethodBase):
+    """yaboot on ppc"""
+
+    name = "yaboot"
+
+    def __init__(self, *args, **kwargs):
+        super(YabootMultilibMethod, self).__init__(*args, **kwargs)
+
+    def select(self, po):
+        if po.arch in ["ppc"]:
+            if po.name.startswith("yaboot"):
+                return True
+        return False
+
+
+class DevelMultilibMethod(MultilibMethodBase):
+    """all -devel and -static packages"""
+
+    name = "devel"
+
+    def __init__(self, *args, **kwargs):
+        super(DevelMultilibMethod, self).__init__(*args, **kwargs)
+        self.blacklist = read_lines_from_file(self.config_path + "devel-blacklist.conf")
+        self.whitelist = read_lines_from_file(self.config_path + "devel-whitelist.conf")
+
+    def select(self, po):
+        if self.skip(po):
+            return False
+        if po.name in self.blacklist:
+            return False
+        if po.name in self.whitelist:
+            return True
+        if self.is_kernel_devel(po):
+            return False
+        # HACK: exclude ghc*
+        if po.name.startswith("ghc-"):
+            return False
+        if po.name.endswith("-devel"):
+            return True
+        if po.name.endswith("-static"):
+            return True
+        for p_name, p_flag, (p_e, p_v, p_r) in po.provides:
+            if p_name.endswith("-devel"):
+                return True
+            if p_name.endswith("-static"):
+                return True
+        return False
+
+
+DEFAULT_METHODS = ["devel", "runtime"]
+METHOD_MAP = {}
+
+
+def init(config_path="/usr/share/pungi/multilib/"):
+    global METHOD_MAP
+
+    if not config_path.endswith("/"):
+        config_path += "/"
+
+    for cls in (
+        AllMultilibMethod,
+        DevelMultilibMethod,
+        KernelMultilibMethod,
+        NoneMultilibMethod,
+        RuntimeMultilibMethod,
+        YabootMultilibMethod,
+    ):
+        method = cls(config_path)
+        METHOD_MAP[method.name] = method
+
+
+def po_is_multilib(po, methods):
+    for method_name in methods:
+        if not method_name:
+            continue
+        method = METHOD_MAP[method_name]
+        if method.select(po):
+            return method_name
+    return None

pungi/notifier.py

@@ -0,0 +1,112 @@
+# -*- coding: utf-8 -*-
+
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; version 2 of the License.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Library General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, see <https://gnu.org/licenses/>.
+
+from datetime import datetime
+import json
+import os
+import threading
+
+import pungi.util
+
+from kobo import shortcuts
+
+
+class PungiNotifier(object):
+    """Wrapper around an external script for sending messages.
+
+    If no script is configured, the messages are just silently ignored. If the
+    script fails, a warning will be logged, but the compose process will not be
+    interrupted.
+    """
+
+    def __init__(self, cmds):
+        self.cmds = cmds
+        self.lock = threading.Lock()
+        self.compose = None
+
+    def _update_args(self, data):
+        """Add compose related information to the data."""
+        if not self.compose:
+            return
+        data.setdefault("compose_id", self.compose.compose_id)
+
+        # Publish where in the world this compose will end up living
+        location = pungi.util.translate_path(
+            self.compose, self.compose.paths.compose.topdir()
+        )
+        data.setdefault("location", location)
+
+        # Add information about the compose itself.
+        data.setdefault("compose_date", self.compose.compose_date)
+        data.setdefault("compose_type", self.compose.compose_type)
+        data.setdefault("compose_respin", self.compose.compose_respin)
+        data.setdefault("compose_label", self.compose.compose_label)
+        data.setdefault("compose_path", self.compose.topdir)
+        data.setdefault("release_short", self.compose.conf["release_short"])
+        data.setdefault("release_name", self.compose.conf["release_name"])
+        data.setdefault("release_version", self.compose.conf["release_version"])
+        data.setdefault("release_type", self.compose.conf["release_type"].lower())
+        data.setdefault("release_is_layered", False)
+
+        if self.compose.conf.get("base_product_name", ""):
+            data["release_is_layered"] = True
+            data["base_product_name"] = self.compose.conf["base_product_name"]
+            data["base_product_version"] = self.compose.conf["base_product_version"]
+            data["base_product_short"] = self.compose.conf["base_product_short"]
+            data["base_product_type"] = self.compose.conf["base_product_type"].lower()
+
+    def send(self, msg, workdir=None, **kwargs):
+        """Send a message.
+
+        The actual meaning of ``msg`` depends on what the notification script
+        will be doing. The keyword arguments will be JSON-encoded and passed on
+        to standard input of the notification process.
+
+        Unless you specify it manually, a ``compose_id`` key with appropriate
+        value will be automatically added.
+        """
+        if not self.cmds:
+            return
+
+        self._update_args(kwargs)
+
+        with self.lock:
+            for cmd in self.cmds:
+                self._run_script(cmd, msg, workdir, kwargs)
+
+    def _run_script(self, cmd, msg, workdir, kwargs):
+        """Run a single notification script with proper logging."""
+        logfile = None
+        if self.compose:
+            self.compose.log_debug("Notification: %r %r, %r" % (cmd, msg, kwargs))
+            logfile = os.path.join(
+                self.compose.paths.log.topdir(),
+                "notifications",
+                "notification-%s.log" % datetime.utcnow().strftime("%Y-%m-%d_%H-%M-%S"),
+            )
+            pungi.util.makedirs(os.path.dirname(logfile))
+
+        ret, _ = shortcuts.run(
+            (cmd, msg),
+            stdin_data=json.dumps(kwargs),
+            can_fail=True,
+            workdir=workdir,
+            return_stdout=False,
+            show_cmd=True,
+            universal_newlines=True,
+            logfile=logfile,
+        )
+        if ret != 0:
+            if self.compose:
+                self.compose.log_warning("Failed to invoke notification script.")

pungi/ostree/__init__.py

@@ -0,0 +1,202 @@
+# -*- coding: utf-8 -*-
+
+
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; version 2 of the License.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Library General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, see <https://gnu.org/licenses/>.
+
+
+import argparse
+import logging
+
+from .tree import Tree
+from .installer import Installer
+from .container import Container
+
+
+def main(args=None):
+    parser = argparse.ArgumentParser()
+    subparser = parser.add_subparsers(help="Sub commands")
+
+    treep = subparser.add_parser("tree", help="Compose OSTree repository")
+    treep.set_defaults(_class=Tree, func="run")
+    treep.add_argument(
+        "--repo",
+        metavar="PATH",
+        required=True,
+        help="where to put the OSTree repo (required)",
+    )
+    treep.add_argument(
+        "--treefile",
+        metavar="FILE",
+        required=True,
+        help="treefile for rpm-ostree (required)",
+    )
+    treep.add_argument(
+        "--log-dir",
+        metavar="DIR",
+        required=True,
+        help="where to log output and commitid (required). \
+                             Note: commitid file will be written to this dir",
+    )
+    treep.add_argument(
+        "--extra-config", metavar="FILE", help="JSON file contains extra configurations"
+    )
+    treep.add_argument(
+        "--version",
+        metavar="VERSION",
+        help="version string to be added as versioning metadata",
+    )
+    treep.add_argument(
+        "--update-summary", action="store_true", help="update summary metadata"
+    )
+    treep.add_argument(
+        "--ostree-ref", metavar="PATH", help="override ref value from treefile"
+    )
+    treep.add_argument(
+        "--force-new-commit",
+        action="store_true",
+        help="do not use rpm-ostree's built-in change detection",
+    )
+    treep.add_argument(
+        "--unified-core",
+        action="store_true",
+        help="use unified core mode in rpm-ostree",
+    )
+
+    container = subparser.add_parser(
+        "container", help="Compose OSTree native container"
+    )
+    container.set_defaults(_class=Container, func="run")
+    container.add_argument(
+        "--name",
+        required=True,
+        help="the name of the the OCI archive (required)",
+    )
+    container.add_argument(
+        "--path",
+        required=True,
+        help="where to output the OCI archive (required)",
+    )
+    container.add_argument(
+        "--treefile",
+        metavar="FILE",
+        required=True,
+        help="treefile for rpm-ostree (required)",
+    )
+    container.add_argument(
+        "--log-dir",
+        metavar="DIR",
+        required=True,
+        help="where to log output (required).",
+    )
+    container.add_argument(
+        "--extra-config", metavar="FILE", help="JSON file contains extra configurations"
+    )
+    container.add_argument(
+        "-v",
+        "--version",
+        metavar="VERSION",
+        required=True,
+        help="version identifier (required)",
+    )
+
+    installerp = subparser.add_parser(
+        "installer", help="Create an OSTree installer image"
+    )
+    installerp.set_defaults(_class=Installer, func="run")
+    installerp.add_argument(
+        "-p",
+        "--product",
+        metavar="PRODUCT",
+        required=True,
+        help="product name (required)",
+    )
+    installerp.add_argument(
+        "-v",
+        "--version",
+        metavar="VERSION",
+        required=True,
+        help="version identifier (required)",
+    )
+    installerp.add_argument(
+        "-r",
+        "--release",
+        metavar="RELEASE",
+        required=True,
+        help="release information (required)",
+    )
+    installerp.add_argument(
+        "-s",
+        "--source",
+        metavar="REPOSITORY",
+        required=True,
+        action="append",
+        help="source repository (required)",
+    )
+    installerp.add_argument(
+        "-o",
+        "--output",
+        metavar="DIR",
+        required=True,
+        help="path to image output directory (required)",
+    )
+    installerp.add_argument("--log-dir", metavar="DIR", help="path to log directory")
+    installerp.add_argument("--volid", metavar="VOLID", help="volume id")
+    installerp.add_argument("--variant", metavar="VARIANT", help="variant name")
+    installerp.add_argument("--rootfs-size", metavar="SIZE")
+    installerp.add_argument("--nomacboot", action="store_true", default=False)
+    installerp.add_argument("--noupgrade", action="store_true", default=False)
+    installerp.add_argument("--isfinal", action="store_true", default=False)
+
+    installerp.add_argument(
+        "--installpkgs",
+        metavar="PACKAGE",
+        action="append",
+        help="package glob to install before runtime-install.tmpl",
+    )
+    installerp.add_argument(
+        "--add-template",
+        metavar="FILE",
+        action="append",
+        help="Additional template for runtime image",
+    )
+    installerp.add_argument(
+        "--add-template-var",
+        metavar="ADD_TEMPLATE_VARS",
+        action="append",
+        help="Set variable for runtime image template",
+    )
+    installerp.add_argument(
+        "--add-arch-template",
+        metavar="FILE",
+        action="append",
+        help="Additional template for architecture-specific image",
+    )
+    installerp.add_argument(
+        "--add-arch-template-var",
+        metavar="ADD_ARCH_TEMPLATE_VARS",
+        action="append",
+        help="Set variable for architecture-specific image",
+    )
+
+    installerp.add_argument(
+        "--extra-config", metavar="FILE", help="JSON file contains extra configurations"
+    )
+
+    args = parser.parse_args(args)
+
+    logging.basicConfig(format="%(message)s", level=logging.DEBUG)
+
+    _class = args._class()
+    _class.set_args(args)
+    func = getattr(_class, args.func)
+    func()

pungi/ostree/base.py

@@ -0,0 +1,19 @@
+# -*- coding: utf-8 -*-
+
+
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; version 2 of the License.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Library General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, see <https://gnu.org/licenses/>.
+
+
+class OSTree(object):
+    def set_args(self, args):
+        self.args = args

pungi/ostree/container.py

@@ -0,0 +1,86 @@
+# -*- coding: utf-8 -*-
+
+
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; version 2 of the License.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Library General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, see <https://gnu.org/licenses/>.
+
+
+import os
+import json
+import six
+from six.moves import shlex_quote
+
+
+from .base import OSTree
+from .utils import tweak_treeconf
+
+
+def emit(cmd):
+    """Print line of shell code into the stream."""
+    if isinstance(cmd, six.string_types):
+        print(cmd)
+    else:
+        print(" ".join([shlex_quote(x) for x in cmd]))
+
+
+class Container(OSTree):
+    def _make_container(self):
+        """Compose OSTree Container Native image"""
+        stamp_file = os.path.join(self.logdir, "%s.stamp" % self.name)
+        cmd = [
+            "rpm-ostree",
+            "compose",
+            "image",
+            # Always initialize for now
+            "--initialize",
+            # Touch the file if a new commit was created. This can help us tell
+            # if the commitid file is missing because no commit was created or
+            # because something went wrong.
+            "--touch-if-changed=%s" % stamp_file,
+            self.treefile,
+        ]
+        fullpath = os.path.join(self.path, "%s.ociarchive" % self.name)
+        cmd.append(fullpath)
+
+        # Set the umask to be more permissive so directories get group write
+        # permissions. See https://pagure.io/releng/issue/8811#comment-629051
+        emit("umask 0002")
+        emit(cmd)
+
+    def run(self):
+        self.name = self.args.name
+        self.path = self.args.path
+        self.treefile = self.args.treefile
+        self.logdir = self.args.log_dir
+        self.extra_config = self.args.extra_config
+
+        if self.extra_config:
+            self.extra_config = json.load(open(self.extra_config, "r"))
+            repos = self.extra_config.get("repo", [])
+            keep_original_sources = self.extra_config.get(
+                "keep_original_sources", False
+            )
+        else:
+            # missing extra_config mustn't affect tweak_treeconf call
+            repos = []
+            keep_original_sources = True
+
+        update_dict = {"automatic-version-prefix": self.args.version}
+
+        self.treefile = tweak_treeconf(
+            self.treefile,
+            source_repos=repos,
+            keep_original_sources=keep_original_sources,
+            update_dict=update_dict,
+        )
+
+        self._make_container()

pungi/ostree/installer.py

@@ -0,0 +1,77 @@
+# -*- coding: utf-8 -*-
+
+
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; version 2 of the License.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Library General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, see <https://gnu.org/licenses/>.
+
+
+import json
+from kobo import shortcuts
+
+from .base import OSTree
+from ..wrappers import lorax
+
+
+class Installer(OSTree):
+    def _merge_config(self, config):
+        self.installpkgs.extend(config.get("installpkgs", []))
+        self.add_template.extend(config.get("add_template", []))
+        self.add_template_var.extend(config.get("add_template_var"))
+        self.add_arch_template.extend(config.get("add_arch_template", []))
+        self.add_arch_template_var.extend(config.get("add_arch_template_var", []))
+
+    def run(self):
+        self.product = self.args.product
+        self.version = self.args.version
+        self.release = self.args.release
+        self.sources = self.args.source
+        self.output = self.args.output
+
+        self.logdir = self.args.log_dir
+        self.volid = self.args.volid
+        self.variant = self.args.variant
+        self.rootfs_size = self.args.rootfs_size
+        self.nomacboot = self.args.nomacboot
+        self.noupgrade = self.args.noupgrade
+        self.isfinal = self.args.isfinal
+
+        self.installpkgs = self.args.installpkgs or []
+        self.add_template = self.args.add_template or []
+        self.add_template_var = self.args.add_template_var or []
+        self.add_arch_template = self.args.add_arch_template or []
+        self.add_arch_template_var = self.args.add_arch_template_var or []
+
+        self.extra_config = self.args.extra_config
+        if self.extra_config:
+            self.extra_config = json.load(open(self.extra_config, "r"))
+            self._merge_config(self.extra_config)
+
+        lorax_wrapper = lorax.LoraxWrapper()
+        cmd = lorax_wrapper.get_lorax_cmd(
+            self.product,
+            self.version,
+            self.release,
+            self.sources,
+            self.output,
+            variant=self.variant,
+            nomacboot=self.nomacboot,
+            volid=self.volid,
+            buildinstallpackages=self.installpkgs,
+            add_template=self.add_template,
+            add_template_var=self.add_template_var,
+            add_arch_template=self.add_arch_template,
+            add_arch_template_var=self.add_arch_template_var,
+            rootfs_size=self.rootfs_size,
+            is_final=self.isfinal,
+            log_dir=self.logdir,
+        )
+        shortcuts.run(cmd)

pungi/ostree/tree.py

@@ -0,0 +1,158 @@
+# -*- coding: utf-8 -*-
+
+
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; version 2 of the License.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Library General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, see <https://gnu.org/licenses/>.
+
+
+import os
+import json
+from kobo import shortcuts
+
+from pungi.util import makedirs
+from .base import OSTree
+from .utils import (
+    make_log_file,
+    tweak_treeconf,
+    get_ref_from_treefile,
+    get_commitid_from_commitid_file,
+)
+
+
+class Tree(OSTree):
+    def _make_tree(self):
+        """Compose OSTree tree"""
+        log_file = make_log_file(self.logdir, "create-ostree-repo")
+        cmd = [
+            "rpm-ostree",
+            "compose",
+            "tree",
+            "--repo=%s" % self.repo,
+            "--write-commitid-to=%s" % self.commitid_file,
+            # Touch the file if a new commit was created. This can help us tell
+            # if the commitid file is missing because no commit was created or
+            # because something went wrong.
+            "--touch-if-changed=%s.stamp" % self.commitid_file,
+        ]
+        if self.unified_core:
+            # See https://github.com/coreos/rpm-ostree/issues/729
+            cmd.append("--unified-core")
+        if self.version:
+            # Add versioning metadata
+            cmd.append("--add-metadata-string=version=%s" % self.version)
+        # Note renamed from rpm-ostree --force-nocache since it's a better
+        # name; more clearly describes what we're doing here.
+        if self.force_new_commit:
+            cmd.append("--force-nocache")
+        cmd.append(self.treefile)
+
+        # Set the umask to be more permissive so directories get group write
+        # permissions. See https://pagure.io/releng/issue/8811#comment-629051
+        oldumask = os.umask(0o0002)
+        try:
+            shortcuts.run(
+                cmd,
+                show_cmd=True,
+                stdout=True,
+                logfile=log_file,
+                universal_newlines=True,
+            )
+        finally:
+            os.umask(oldumask)
+
+    def _update_summary(self):
+        """Update summary metadata"""
+        log_file = make_log_file(self.logdir, "ostree-summary")
+        shortcuts.run(
+            ["ostree", "summary", "-u", "--repo=%s" % self.repo],
+            show_cmd=True,
+            stdout=True,
+            logfile=log_file,
+            universal_newlines=True,
+        )
+
+    def _update_ref(self):
+        """
+        Update the ref.
+
+        '--write-commitid-to' is specified when compose the tree, so we need
+        to update the ref by ourselves. ref is retrieved from treefile and
+        commitid is retrieved from the committid file.
+        """
+        tag_ref = True
+        if self.extra_config:
+            tag_ref = self.extra_config.get("tag_ref", True)
+        if not tag_ref:
+            print("Not updating ref as configured")
+            return
+        ref = get_ref_from_treefile(self.treefile)
+        commitid = get_commitid_from_commitid_file(self.commitid_file)
+        print("Ref: %r, Commit ID: %r" % (ref, commitid))
+        if ref and commitid:
+            print("Updating ref")
+            # Let's write the tag out ourselves
+            heads_dir = os.path.join(self.repo, "refs", "heads")
+            if not os.path.exists(heads_dir):
+                raise RuntimeError("Refs/heads did not exist in ostree repo")
+
+            ref_path = os.path.join(heads_dir, ref)
+            # Set the umask to be more permissive so directories get group write
+            # permissions. See https://pagure.io/releng/issue/8811#comment-629051
+            oldumask = os.umask(0o0002)
+            try:
+                makedirs(os.path.dirname(ref_path))
+            finally:
+                os.umask(oldumask)
+            with open(ref_path, "w") as f:
+                f.write(commitid + "\n")
+
+    def run(self):
+        self.repo = self.args.repo
+        self.treefile = self.args.treefile
+        self.version = self.args.version
+        self.logdir = self.args.log_dir
+        self.update_summary = self.args.update_summary
+        self.extra_config = self.args.extra_config
+        self.ostree_ref = self.args.ostree_ref
+        self.force_new_commit = self.args.force_new_commit
+        self.unified_core = self.args.unified_core
+
+        if self.extra_config or self.ostree_ref:
+            if self.extra_config:
+                self.extra_config = json.load(open(self.extra_config, "r"))
+                repos = self.extra_config.get("repo", [])
+                keep_original_sources = self.extra_config.get(
+                    "keep_original_sources", False
+                )
+            else:
+                # missing extra_config mustn't affect tweak_treeconf call
+                repos = []
+                keep_original_sources = True
+
+            update_dict = {}
+            if self.ostree_ref:
+                # override ref value in treefile
+                update_dict["ref"] = self.ostree_ref
+
+            self.treefile = tweak_treeconf(
+                self.treefile,
+                source_repos=repos,
+                keep_original_sources=keep_original_sources,
+                update_dict=update_dict,
+            )
+
+        self.commitid_file = make_log_file(self.logdir, "commitid")
+
+        self._make_tree()
+        self._update_ref()
+        if self.update_summary:
+            self._update_summary()

pungi/ostree/utils.py

@@ -0,0 +1,126 @@
+# -*- coding: utf-8 -*-
+
+
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; version 2 of the License.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Library General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, see <https://gnu.org/licenses/>.
+
+
+import json
+import logging
+import os
+import shutil
+
+import yaml
+
+from pungi.arch_utils import getBaseArch
+from pungi.util import makedirs
+
+
+def make_log_file(log_dir, filename):
+    """Return path to log file with given name, if log_dir is set."""
+    if not log_dir:
+        return None
+    makedirs(log_dir)
+    return os.path.join(log_dir, "%s.log" % filename)
+
+
+def get_ref_from_treefile(treefile, arch=None, logger=None):
+    """
+    Return ref name by parsing the tree config file. Replacing ${basearch} with
+    the basearch of the architecture we are running on or of the passed in arch.
+    """
+    logger = logger or logging.getLogger(__name__)
+    if os.path.isfile(treefile):
+        with open(treefile, "r") as f:
+            try:
+                # rpm-ostree now supports YAML
+                #   https://github.com/projectatomic/rpm-ostree/pull/1377
+                if treefile.endswith(".yaml"):
+                    parsed = yaml.safe_load(f)
+                else:
+                    parsed = json.load(f)
+                return parsed["ref"].replace("${basearch}", getBaseArch(arch))
+            except Exception as e:
+                logger.error("Unable to get ref from treefile: %s" % e)
+    else:
+        logger.error("Unable to open treefile")
+    return None
+
+
+def get_commitid_from_commitid_file(commitid_file):
+    """Return commit id which is read from the commitid file"""
+    if not os.path.exists(commitid_file + ".stamp"):
+        # The stamp does not exist, so no new commit.
+        return None
+    with open(commitid_file, "r") as f:
+        return f.read().replace("\n", "")
+
+
+def tweak_treeconf(
+    treeconf, source_repos=None, keep_original_sources=False, update_dict=None
+):
+    """
+    Update tree config file by adding new repos, and remove existing repos
+    from the tree config file if 'keep_original_sources' is not enabled.
+    Additionally, other values can be passed to method by 'update_dict' parameter to
+    update treefile content.
+    """
+
+    # backup the old tree config
+    shutil.copy2(treeconf, "{0}.bak".format(treeconf))
+
+    treeconf_dir = os.path.dirname(treeconf)
+    with open(treeconf, "r") as f:
+        # rpm-ostree now supports YAML, but we'll end up converting it to JSON.
+        # https://github.com/projectatomic/rpm-ostree/pull/1377
+        if treeconf.endswith(".yaml"):
+            treeconf_content = yaml.safe_load(f)
+            treeconf = treeconf.replace(".yaml", ".json")
+        else:
+            treeconf_content = json.load(f)
+
+    repos = []
+    if source_repos:
+        # Sort to ensure reliable ordering
+        source_repos = sorted(source_repos, key=lambda x: x["name"])
+        # Now, since pungi includes timestamps in the repo names which
+        # currently defeats rpm-ostree's change detection, let's just
+        # use repos named 'repo-<number>'.
+        # https://pagure.io/pungi/issue/811
+        with open("{0}/pungi.repo".format(treeconf_dir), "w") as f:
+            for i, repo in enumerate(source_repos):
+                name = "repo-{0}".format(i)
+                f.write("[%s]\n" % name)
+                f.write("name=%s\n" % name)
+                f.write("baseurl=%s\n" % repo["baseurl"])
+                exclude = repo.get("exclude", None)
+                if exclude:
+                    f.write("exclude=%s\n" % exclude)
+                gpgcheck = "1" if repo.get("gpgcheck", False) else "0"
+                f.write("gpgcheck=%s\n" % gpgcheck)
+
+                repos.append(name)
+
+    original_repos = treeconf_content.get("repos", [])
+    if keep_original_sources:
+        treeconf_content["repos"] = original_repos + repos
+    else:
+        treeconf_content["repos"] = repos
+
+    # update content with config values from dictionary (for example 'ref')
+    if isinstance(update_dict, dict):
+        treeconf_content.update(update_dict)
+
+    # update tree config to add new repos
+    with open(treeconf, "w") as f:
+        json.dump(treeconf_content, f, indent=4)
+    return treeconf

pungi/pathmatch.py

@@ -0,0 +1,73 @@
+# -*- coding: utf-8 -*-
+
+
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; version 2 of the License.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Library General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, see <https://gnu.org/licenses/>.
+
+
+import fnmatch
+
+
+def head_tail_split(name):
+    name_split = name.strip("/").split("/", 1)
+    if len(name_split) == 2:
+        head = name_split[0]
+        tail = name_split[1].strip("/")
+    else:
+        head, tail = name_split[0], None
+    return head, tail
+
+
+class PathMatch(object):
+    def __init__(self, parent=None, desc=None):
+        self._patterns = {}
+        self._final_patterns = {}
+        self._values = []
+
+    def __setitem__(self, name, value):
+        head, tail = head_tail_split(name)
+
+        if tail is not None:
+            # recursion
+            if head not in self._patterns:
+                self._patterns[head] = PathMatch(parent=self, desc=head)
+            self._patterns[head][tail] = value
+        else:
+            if head not in self._final_patterns:
+                self._final_patterns[head] = PathMatch(parent=self, desc=head)
+            if value not in self._final_patterns[head]._values:
+                self._final_patterns[head]._values.append(value)
+
+    def __getitem__(self, name):
+        result = []
+        head, tail = head_tail_split(name)
+        for pattern in self._patterns:
+            if fnmatch.fnmatch(head, pattern):
+                if tail is None:
+                    values = self._patterns[pattern]._values
+                else:
+                    values = self._patterns[pattern][tail]
+                for value in values:
+                    if value not in result:
+                        result.append(value)
+
+        for pattern in self._final_patterns:
+            if tail is None:
+                x = head
+            else:
+                x = "%s/%s" % (head, tail)
+            if fnmatch.fnmatch(x, pattern):
+                values = self._final_patterns[pattern]._values
+                for value in values:
+                    if value not in result:
+                        result.append(value)
+        return result

pungi/paths.py

@@ -0,0 +1,841 @@
+# -*- coding: utf-8 -*-
+
+
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; version 2 of the License.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Library General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, see <https://gnu.org/licenses/>.
+
+
+__all__ = ("Paths",)
+
+
+import errno
+import os
+
+from kobo.shortcuts import relative_path
+from pungi.util import makedirs, find_old_compose
+
+
+class Paths(object):
+    def __init__(self, compose):
+        self._compose = compose
+        paths_module_name = compose.conf.get("paths_module")
+        if paths_module_name:
+            # custom paths
+            paths_module = __import__(
+                paths_module_name,
+                globals(),
+                locals(),
+                ["LogPaths", "WorkPaths", "ComposePaths"],
+            )
+            self.compose = paths_module.ComposePaths(compose)
+            self.log = paths_module.LogPaths(compose)
+            self.work = paths_module.WorkPaths(compose)
+        else:
+            # default paths
+            self.compose = ComposePaths(compose)
+            self.log = LogPaths(compose)
+            self.work = WorkPaths(compose)
+        # self.metadata ?
+
+    def get_old_compose_topdir(self, **kwargs):
+        """
+        Finds old compose using the `find_old_compose` function and returns
+        the path to it. The `kwargs` are passed to `find_old_compose`.
+        """
+        is_layered = self._compose.ci_base.release.is_layered
+        return find_old_compose(
+            self._compose.old_composes,
+            self._compose.ci_base.release.short,
+            self._compose.ci_base.release.version,
+            self._compose.ci_base.release.type_suffix,
+            self._compose.ci_base.base_product.short if is_layered else None,
+            self._compose.ci_base.base_product.version if is_layered else None,
+            **kwargs
+        )
+
+    def old_compose_path(self, path, **kwargs):
+        """
+        Translates `path` to the topdir of old compose.
+
+        :param str path: Path to translate.
+        :param kwargs: The kwargs passed to `find_old_compose` function.
+        :return: None if old compose cannot be used or if `path` does not exist
+            in the old compose topdir. Otherwise path translated to old_compose
+            topdir.
+
+        Example:
+            old_repo_dir = compose.old_compose_path(
+                compose.paths.work.pkgset_repo(pkgset.name, arch="global"))
+        """
+        old_compose_topdir = self.get_old_compose_topdir(**kwargs)
+        if not old_compose_topdir:
+            return None
+
+        rel_path = relative_path(path, self._compose.topdir.rstrip("/") + "/")
+        old_path = os.path.join(old_compose_topdir, rel_path)
+        if not os.path.exists(old_path):
+            return None
+        return old_path
+
+
+class LogPaths(object):
+    def __init__(self, compose):
+        self.compose = compose
+
+    def topdir(self, arch=None, create_dir=True):
+        """
+        Examples:
+            log/global
+            log/x86_64
+        """
+        arch = arch or "global"
+        path = os.path.join(self.compose.topdir, "logs", arch)
+        if create_dir:
+            makedirs(path)
+        return path
+
+    def koji_tasks_dir(self, create_dir=True):
+        """
+        Examples:
+            logs/global/koji-tasks
+        """
+        path = os.path.join(self.topdir(create_dir=create_dir), "koji-tasks")
+        if create_dir:
+            makedirs(path)
+        return path
+
+    def log_file(self, arch, log_name, create_dir=True, ext=None):
+        ext = ext or "log"
+        arch = arch or "global"
+        if log_name.endswith(".log"):
+            log_name = log_name[:-4]
+        return os.path.join(
+            self.topdir(arch, create_dir=create_dir), "%s.%s.%s" % (log_name, arch, ext)
+        )
+
+
+class WorkPaths(object):
+    def __init__(self, compose):
+        self.compose = compose
+
+    def topdir(self, arch=None, create_dir=True):
+        """
+        Examples:
+            work/global
+            work/x86_64
+        """
+        arch = arch or "global"
+        path = os.path.join(self.compose.topdir, "work", arch)
+        if create_dir:
+            makedirs(path)
+        return path
+
+    def variants_file(self, arch=None, create_dir=True):
+        """
+        Examples:
+            work/global/variants.xml
+        """
+        arch = "global"
+        path = os.path.join(self.topdir(arch, create_dir=create_dir), "variants.xml")
+        return path
+
+    def comps(self, arch=None, variant=None, create_dir=True):
+        """
+        Examples:
+            work/x86_64/comps/comps-86_64.xml
+            work/x86_64/comps/comps-Server.x86_64.xml
+        """
+        arch = arch or "global"
+        if variant is None:
+            file_name = "comps-%s.xml" % arch
+        else:
+            file_name = "comps-%s.%s.xml" % (variant.uid, arch)
+        path = os.path.join(self.topdir(arch, create_dir=create_dir), "comps")
+        if create_dir:
+            makedirs(path)
+        path = os.path.join(path, file_name)
+        return path
+
+    def gather_result(self, arch=None, variant=None, create_dir=True):
+        """
+        Examples:
+            work/x86_64/gather_result/x86_64.result
+            work/x86_64/gather_result/Server.x86_64.result
+        """
+        arch = arch or "global"
+        file_name = ""
+        if variant:
+            file_name += variant.uid + "."
+        file_name += arch + "."
+        file_name += "result"
+        path = os.path.join(self.topdir(arch, create_dir=create_dir), "gather_result")
+        if create_dir:
+            makedirs(path)
+        path = os.path.join(path, file_name)
+        return path
+
+    def pungi_conf(self, arch=None, variant=None, create_dir=True, source_name=None):
+        """
+        Examples:
+            work/x86_64/pungi/x86_64.conf
+            work/x86_64/pungi/Server.x86_64.conf
+        """
+        arch = arch or "global"
+        file_name = ""
+        if variant:
+            file_name += variant.uid + "."
+        file_name += arch + "."
+        if source_name:
+            file_name += source_name + "."
+        file_name += "conf"
+        path = os.path.join(self.topdir(arch, create_dir=create_dir), "pungi")
+        if create_dir:
+            makedirs(path)
+        path = os.path.join(path, file_name)
+        return path
+
+    def fus_conf(self, arch, variant, iteration, create_dir=True):
+        """
+        Examples:
+            work/x86_64/fus/Server-solvables.x86_64.conf
+        """
+        file_name = "%s-solvables-%d.%s.conf" % (variant.uid, iteration, arch)
+        path = os.path.join(self.topdir(arch, create_dir=create_dir), "fus")
+        if create_dir:
+            makedirs(path)
+        return os.path.join(path, file_name)
+
+    def pungi_log(self, arch=None, variant=None, create_dir=True, source_name=None):
+        """
+        Examples:
+            work/x86_64/pungi/x86_64.log
+            work/x86_64/pungi/Server.x86_64.log
+        """
+        path = self.pungi_conf(arch, variant, create_dir=create_dir)
+        path = path[:-5]
+        if source_name:
+            path += "." + source_name
+        return path + ".log"
+
+    def pungi_cache_dir(self, arch, variant=None, create_dir=True):
+        """
+        Examples:
+            work/global/pungi-cache
+        """
+        # WARNING: Using the same cache dir with repos of the same names
+        # may lead to a race condition.
+        # We should use per arch variant cache dirs to workaround this.
+        path = os.path.join(self.topdir(arch, create_dir=create_dir), "pungi-cache")
+        if variant:
+            path = os.path.join(path, variant.uid)
+        if create_dir:
+            makedirs(path)
+        return path
+
+    def _repo(self, type, arch=None, variant=None, create_dir=True):
+        arch = arch or "global"
+        path = os.path.join(self.topdir(arch, create_dir=create_dir), "%s_repo" % type)
+        if variant:
+            path += "_" + variant.uid
+        if create_dir:
+            makedirs(path)
+        return path
+
+    def comps_repo(self, arch=None, variant=None, create_dir=True):
+        """
+        Examples:
+            work/x86_64/comps_repo_Server
+            work/global/comps_repo
+        """
+        return self._repo("comps", arch, variant, create_dir=create_dir)
+
+    def pkgset_repo(self, pkgset_name, arch=None, create_dir=True):
+        """
+        Examples:
+            work/x86_64/repo/f30-compose
+            work/global/repo/f30-compose
+        """
+        arch = arch or "global"
+        path = os.path.join(
+            self.topdir(arch, create_dir=create_dir), "repo", pkgset_name
+        )
+        if create_dir:
+            makedirs(path)
+        return path
+
+    def lookaside_repo(self, arch, variant, create_dir=True):
+        """
+        Examples:
+            work/x86_64/Server/lookaside_repo
+        """
+        path = os.path.join(
+            self.topdir(arch, create_dir=create_dir), variant.uid, "lookaside_repo"
+        )
+        if create_dir:
+            makedirs(path)
+        return path
+
+    def package_list(
+        self, arch=None, variant=None, pkgset=None, pkg_type=None, create_dir=True
+    ):
+        """
+        Examples:
+            work/x86_64/package_list/x86_64.conf
+            work/x86_64/package_list/Server.x86_64.conf
+            work/x86_64/package_list/Server.x86_64.rpm.conf
+        """
+        arch = arch or "global"
+        if variant is not None:
+            file_name = "%s.%s" % (variant, arch)
+        else:
+            file_name = "%s" % arch
+        if pkgset:
+            file_name += "." + pkgset.name
+        if pkg_type is not None:
+            file_name += ".%s" % pkg_type
+        file_name += ".conf"
+        path = os.path.join(self.topdir(arch, create_dir=create_dir), "package_list")
+        if create_dir:
+            makedirs(path)
+        path = os.path.join(path, file_name)
+        return path
+
+    def lookaside_package_list(self, arch, variant, create_dir=True):
+        """
+        Examples:
+            work/x86_64/package_list/Server.x86_64.lookaside.conf
+        """
+        return self.package_list(
+            arch, variant, pkg_type="lookaside", create_dir=create_dir
+        )
+
+    def pungi_download_dir(self, arch, create_dir=True):
+        """
+        Examples:
+            work/x86_64/pungi_download
+        """
+        path = os.path.join(self.topdir(arch, create_dir=create_dir), "pungi_download")
+        if create_dir:
+            makedirs(path)
+        return path
+
+    def buildinstall_dir(
+        self, arch, create_dir=True, allow_topdir_override=False, variant=None
+    ):
+        """
+        :param bool allow_topdir_override: When True, the
+            "buildinstall_topdir" will be used (if set) instead of real
+            "topdir".
+        Examples:
+            work/x86_64/buildinstall
+        """
+        if arch == "global":
+            raise RuntimeError("Global buildinstall dir makes no sense.")
+
+        buildinstall_topdir = self.compose.conf.get("buildinstall_topdir", "")
+        if allow_topdir_override and buildinstall_topdir:
+            topdir_basename = os.path.basename(self.compose.topdir)
+            path = os.path.join(
+                buildinstall_topdir, "buildinstall-%s" % topdir_basename, arch
+            )
+        else:
+            path = os.path.join(
+                self.topdir(arch, create_dir=create_dir), "buildinstall"
+            )
+
+        if variant:
+            path = os.path.join(path, variant.uid)
+        return path
+
+    def extra_files_dir(self, arch, variant, create_dir=True):
+        """
+        Examples:
+            work/x86_64/Server/extra-files
+        """
+        if arch == "global":
+            raise RuntimeError("Global extra files dir makes no sense.")
+        path = os.path.join(
+            self.topdir(arch, create_dir=create_dir), variant.uid, "extra-files"
+        )
+        if create_dir:
+            makedirs(path)
+        return path
+
+    def extra_iso_extra_files_dir(self, arch, variant, create_dir=True):
+        """
+        Examples:
+            work/x86_64/Server/extra-iso-extra-files
+        """
+        if arch == "global":
+            raise RuntimeError("Global extra files dir makes no sense.")
+        path = os.path.join(
+            self.topdir(arch, create_dir=create_dir),
+            variant.uid,
+            "extra-iso-extra-files",
+        )
+        if create_dir:
+            makedirs(path)
+        return path
+
+    def iso_staging_dir(self, arch, variant, filename, create_dir=True):
+        """
+        Examples:
+            work/x86_64/Server/iso-staging-dir/file.iso/
+        """
+        path = os.path.join(
+            self.topdir(arch, create_dir=create_dir),
+            variant.uid,
+            "iso-staging-dir",
+            filename,
+        )
+        if create_dir:
+            makedirs(path)
+        return path
+
+    def repo_package_list(self, arch, variant, pkg_type=None, create_dir=True):
+        """
+        Examples:
+            work/x86_64/repo_package_list/Server.x86_64.rpm.conf
+        """
+        file_name = "%s.%s" % (variant.uid, arch)
+        if pkg_type is not None:
+            file_name += ".%s" % pkg_type
+        file_name += ".conf"
+        path = os.path.join(
+            self.topdir(arch, create_dir=create_dir), "repo_package_list"
+        )
+        if create_dir:
+            makedirs(path)
+        path = os.path.join(path, file_name)
+        return path
+
+    def iso_dir(self, arch, filename, create_dir=True):
+        """
+        Examples:
+            work/x86_64/iso/Project-1.0-20151203.0-Client-x86_64-dvd1.iso
+        """
+        path = os.path.join(self.topdir(arch, create_dir=create_dir), "iso", filename)
+        if create_dir:
+            makedirs(path)
+        return path
+
+    def tmp_dir(self, arch=None, variant=None, create_dir=True):
+        """
+        Examples:
+            work/global/tmp
+            work/x86_64/tmp
+            work/x86_64/tmp-Server
+        """
+        dir_name = "tmp"
+        if variant:
+            dir_name += "-%s" % variant.uid
+        path = os.path.join(self.topdir(arch=arch, create_dir=create_dir), dir_name)
+        if create_dir:
+            makedirs(path)
+        return path
+
+    def product_id(self, arch, variant, create_dir=True):
+        """
+        Examples:
+            work/x86_64/product_id/productid-Server.x86_64.pem/productid
+        """
+        # file_name = "%s.%s.pem" % (variant, arch)
+        # HACK: modifyrepo doesn't handle renames -> $dir/productid
+        file_name = "productid"
+        path = os.path.join(
+            self.topdir(arch, create_dir=create_dir),
+            "product_id",
+            "%s.%s.pem" % (variant, arch),
+        )
+        if create_dir:
+            makedirs(path)
+        path = os.path.join(path, file_name)
+        return path
+
+    def image_build_dir(self, variant, create_dir=True):
+        """
+        @param variant
+        @param create_dir=True
+
+        Examples:
+            work/image-build/Server
+        """
+        path = os.path.join(
+            self.topdir("image-build", create_dir=create_dir), variant.uid
+        )
+        if create_dir:
+            makedirs(path)
+        return path
+
+    def image_build_conf(
+        self, variant, image_name, image_type, arches=None, create_dir=True
+    ):
+        """
+        @param variant
+        @param image-name
+        @param image-type (e.g docker)
+        @param arches
+        @param create_dir=True
+
+        Examples:
+            work/image-build/Server/docker_rhel-server-docker.cfg
+            work/image-build/Server/docker_rhel-server-docker_x86_64.cfg
+            work/image-build/Server/docker_rhel-server-docker_x86_64-ppc64le.cfg
+        """
+        path = os.path.join(
+            self.image_build_dir(variant), "%s_%s" % (image_type, image_name)
+        )
+        if arches is not None:
+            path = "%s_%s" % (path, "-".join(list(arches)))
+        path = "%s.cfg" % path
+        return path
+
+    def module_defaults_dir(self, create_dir=True):
+        """
+        Example:
+            work/global/module_defaults
+        """
+        path = os.path.join(self.topdir(create_dir=create_dir), "module_defaults")
+        if create_dir:
+            makedirs(path)
+        return path
+
+    def module_obsoletes_dir(self, create_dir=True):
+        """
+        Example:
+            work/global/module_obsoletes
+        """
+        path = os.path.join(self.topdir(create_dir=create_dir), "module_obsoletes")
+        if create_dir:
+            makedirs(path)
+        return path
+
+    def pkgset_file_cache(self, pkgset_name):
+        """
+        Returns the path to file in which the cached version of
+        PackageSetBase.file_cache should be stored.
+
+        Example:
+            work/global/pkgset_f33-compose_file_cache.pickle
+        """
+        filename = "pkgset_%s_file_cache.pickle" % pkgset_name
+        return os.path.join(self.topdir(arch="global"), filename)
+
+    def pkgset_reuse_file(self, pkgset_name):
+        """
+        Example:
+            work/global/pkgset_f30-compose_reuse.pickle
+        """
+        filename = "pkgset_%s_reuse.pickle" % pkgset_name
+        return os.path.join(self.topdir(arch="global", create_dir=False), filename)
+
+
+class ComposePaths(object):
+    def __init__(self, compose):
+        self.compose = compose
+        # TODO: TREES?
+
+    def topdir(self, arch=None, variant=None, create_dir=True, relative=False):
+        """
+        Examples:
+            compose
+            compose/Server/x86_64
+        """
+        if bool(arch) != bool(variant):
+            raise TypeError("topdir(): either none or 2 arguments are expected")
+
+        path = ""
+        if not relative:
+            path = os.path.join(self.compose.topdir, "compose")
+
+        if arch or variant:
+            if variant.type == "addon":
+                return self.topdir(
+                    arch, variant.parent, create_dir=create_dir, relative=relative
+                )
+            path = os.path.join(path, variant.uid, arch)
+        if create_dir and not relative:
+            makedirs(path)
+        return path
+
+    def tree_dir(self, arch, variant, create_dir=True, relative=False):
+        """
+        Examples:
+            compose/Server/x86_64/os
+            compose/Server-optional/x86_64/os
+        """
+        if arch == "src":
+            arch = "source"
+
+        if arch == "source":
+            tree_dir = "tree"
+        else:
+            # use 'os' dir due to historical reasons
+            tree_dir = "os"
+
+        path = os.path.join(
+            self.topdir(arch, variant, create_dir=create_dir, relative=relative),
+            tree_dir,
+        )
+        if create_dir and not relative:
+            makedirs(path)
+        return path
+
+    def os_tree(self, arch, variant, create_dir=True, relative=False):
+        return self.tree_dir(arch, variant, create_dir=create_dir, relative=relative)
+
+    def repository(self, arch, variant, create_dir=True, relative=False):
+        """
+        Examples:
+            compose/Server/x86_64/os
+            compose/Server/x86_64/addons/LoadBalancer
+        """
+        if variant.type == "addon":
+            path = self.packages(
+                arch, variant, create_dir=create_dir, relative=relative
+            )
+        else:
+            path = self.tree_dir(
+                arch, variant, create_dir=create_dir, relative=relative
+            )
+        if create_dir and not relative:
+            makedirs(path)
+        return path
+
+    def packages(self, arch, variant, create_dir=True, relative=False):
+        """
+        Examples:
+            compose/Server/x86_64/os/Packages
+            compose/Server/x86_64/os/addons/LoadBalancer
+            compose/Server-optional/x86_64/os/Packages
+        """
+        if variant.type == "addon":
+            path = os.path.join(
+                self.tree_dir(arch, variant, create_dir=create_dir, relative=relative),
+                "addons",
+                variant.id,
+            )
+        else:
+            path = os.path.join(
+                self.tree_dir(arch, variant, create_dir=create_dir, relative=relative),
+                "Packages",
+            )
+        if create_dir and not relative:
+            makedirs(path)
+        return path
+
+    def debug_topdir(self, arch, variant, create_dir=True, relative=False):
+        """
+        Examples:
+            compose/Server/x86_64/debug
+            compose/Server-optional/x86_64/debug
+        """
+        path = os.path.join(
+            self.topdir(arch, variant, create_dir=create_dir, relative=relative),
+            "debug",
+        )
+        if create_dir and not relative:
+            makedirs(path)
+        return path
+
+    def debug_tree(self, arch, variant, create_dir=True, relative=False):
+        """
+        Examples:
+            compose/Server/x86_64/debug/tree
+            compose/Server-optional/x86_64/debug/tree
+        """
+        path = os.path.join(
+            self.debug_topdir(arch, variant, create_dir=create_dir, relative=relative),
+            "tree",
+        )
+        if create_dir and not relative:
+            makedirs(path)
+        return path
+
+    def debug_packages(self, arch, variant, create_dir=True, relative=False):
+        """
+        Examples:
+            compose/Server/x86_64/debug/tree/Packages
+            compose/Server/x86_64/debug/tree/addons/LoadBalancer
+            compose/Server-optional/x86_64/debug/tree/Packages
+        """
+        if arch in ("source", "src"):
+            return None
+        if variant.type == "addon":
+            path = os.path.join(
+                self.debug_tree(
+                    arch, variant, create_dir=create_dir, relative=relative
+                ),
+                "addons",
+                variant.id,
+            )
+        else:
+            path = os.path.join(
+                self.debug_tree(
+                    arch, variant, create_dir=create_dir, relative=relative
+                ),
+                "Packages",
+            )
+        if create_dir and not relative:
+            makedirs(path)
+        return path
+
+    def debug_repository(self, arch, variant, create_dir=True, relative=False):
+        """
+        Examples:
+            compose/Server/x86_64/debug/tree
+            compose/Server/x86_64/debug/tree/addons/LoadBalancer
+            compose/Server-optional/x86_64/debug/tree
+        """
+        if arch in ("source", "src"):
+            return None
+        if variant.type == "addon":
+            path = os.path.join(
+                self.debug_tree(
+                    arch, variant, create_dir=create_dir, relative=relative
+                ),
+                "addons",
+                variant.id,
+            )
+        else:
+            path = self.debug_tree(
+                arch, variant, create_dir=create_dir, relative=relative
+            )
+        if create_dir and not relative:
+            makedirs(path)
+        return path
+
+    def iso_dir(self, arch, variant, symlink_to=None, create_dir=True, relative=False):
+        """
+        Examples:
+            compose/Server/x86_64/iso
+            None
+        """
+        if variant.type == "addon":
+            return None
+        if variant.type == "optional":
+            if not self.compose.conf.get("create_optional_isos", False):
+                return None
+        if arch == "src":
+            arch = "source"
+        path = os.path.join(
+            self.topdir(arch, variant, create_dir=create_dir, relative=relative), "iso"
+        )
+
+        if symlink_to:
+            # TODO: create_dir
+            topdir = self.compose.topdir.rstrip("/") + "/"
+            relative_dir = path[len(topdir) :]
+            target_dir = os.path.join(symlink_to, self.compose.compose_id, relative_dir)
+            if create_dir and not relative:
+                makedirs(target_dir)
+            try:
+                os.symlink(target_dir, path)
+            except OSError as ex:
+                if ex.errno != errno.EEXIST:
+                    raise
+                msg = "Symlink pointing to '%s' expected: %s" % (target_dir, path)
+                if not os.path.islink(path):
+                    raise RuntimeError(msg)
+                if os.path.abspath(os.readlink(path)) != target_dir:
+                    raise RuntimeError(msg)
+        else:
+            if create_dir and not relative:
+                makedirs(path)
+        return path
+
+    def iso_path(
+        self, arch, variant, filename, symlink_to=None, create_dir=True, relative=False
+    ):
+        """
+        Examples:
+            compose/Server/x86_64/iso/rhel-7.0-20120127.0-Server-x86_64-dvd1.iso
+            None
+        """
+        path = self.iso_dir(
+            arch,
+            variant,
+            symlink_to=symlink_to,
+            create_dir=create_dir,
+            relative=relative,
+        )
+        if path is None:
+            return None
+
+        return os.path.join(path, filename)
+
+    def image_dir(self, variant, symlink_to=None, relative=False):
+        """
+        The arch is listed as literal '%(arch)s'
+        Examples:
+            compose/Server/%(arch)s/images
+            None
+        @param variant
+        @param symlink_to=None
+        @param relative=False
+        """
+        path = os.path.join(
+            self.topdir("%(arch)s", variant, create_dir=False, relative=relative),
+            "images",
+        )
+        if symlink_to:
+            topdir = self.compose.topdir.rstrip("/") + "/"
+            relative_dir = path[len(topdir) :]
+            target_dir = os.path.join(symlink_to, self.compose.compose_id, relative_dir)
+            try:
+                os.symlink(target_dir, path)
+            except OSError as ex:
+                if ex.errno != errno.EEXIST:
+                    raise
+                msg = "Symlink pointing to '%s' expected: %s" % (target_dir, path)
+                if not os.path.islink(path):
+                    raise RuntimeError(msg)
+                if os.path.abspath(os.readlink(path)) != target_dir:
+                    raise RuntimeError(msg)
+        return path
+
+    def jigdo_dir(self, arch, variant, create_dir=True, relative=False):
+        """
+        Examples:
+            compose/Server/x86_64/jigdo
+            None
+        """
+        if variant.type == "addon":
+            return None
+        if variant.type == "optional":
+            if not self.compose.conf.get("create_optional_isos", False):
+                return None
+        if arch == "src":
+            arch = "source"
+        path = os.path.join(
+            self.topdir(arch, variant, create_dir=create_dir, relative=relative),
+            "jigdo",
+        )
+
+        if create_dir and not relative:
+            makedirs(path)
+        return path
+
+    def metadata(self, file_name=None, create_dir=True, relative=False):
+        """
+        Examples:
+            compose/metadata
+            compose/metadata/rpms.json
+        """
+        path = os.path.join(
+            self.topdir(create_dir=create_dir, relative=relative), "metadata"
+        )
+        if create_dir and not relative:
+            makedirs(path)
+        if file_name:
+            path = os.path.join(path, file_name)
+        return path

pungi/phases/__init__.py

@@ -0,0 +1,44 @@
+# -*- coding: utf-8 -*-
+
+
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; version 2 of the License.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Library General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, see <https://gnu.org/licenses/>.
+
+import sys
+
+# phases in runtime order
+from .init import InitPhase  # noqa
+from .weaver import WeaverPhase  # noqa
+from .pkgset import PkgsetPhase  # noqa
+from .gather import GatherPhase  # noqa
+from .createrepo import CreaterepoPhase  # noqa
+from .buildinstall import BuildinstallPhase  # noqa
+from .extra_files import ExtraFilesPhase  # noqa
+from .createiso import CreateisoPhase  # noqa
+from .extra_isos import ExtraIsosPhase  # noqa
+from .image_build import ImageBuildPhase  # noqa
+from .image_container import ImageContainerPhase  # noqa
+from .kiwibuild import KiwiBuildPhase  # noqa
+from .osbuild import OSBuildPhase  # noqa
+from .repoclosure import RepoclosurePhase  # noqa
+from .test import TestPhase  # noqa
+from .image_checksum import ImageChecksumPhase  # noqa
+from .livemedia_phase import LiveMediaPhase  # noqa
+from .ostree import OSTreePhase  # noqa
+from .ostree_installer import OstreeInstallerPhase  # noqa
+from .ostree_container import OSTreeContainerPhase  # noqa
+from .osbs import OSBSPhase  # noqa
+from .phases_metadata import gather_phases_metadata  # noqa
+
+
+this_module = sys.modules[__name__]
+PHASES_NAMES = gather_phases_metadata(this_module)

pungi/phases/base.py

@@ -0,0 +1,224 @@
+# -*- coding: utf-8 -*-
+
+
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; version 2 of the License.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Library General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, see <https://gnu.org/licenses/>.
+
+import logging
+import math
+import time
+
+from pungi import util
+
+
+class PhaseBase(object):
+    def __init__(self, compose):
+        self.compose = compose
+        self.msg = "---------- PHASE: %s ----------" % self.name.upper()
+        self.finished = False
+        self._skipped = False
+
+        # A set of config patterns that were actually used. Starts as None, and
+        # when config is queried the variable turns into a set of patterns.
+        self.used_patterns = None
+
+    def validate(self):
+        pass
+
+    def conf_assert_str(self, name):
+        missing = []
+        invalid = []
+        if name not in self.compose.conf:
+            missing.append(name)
+        elif not isinstance(self.compose.conf[name], str):
+            invalid.append(name, type(self.compose.conf[name]), str)
+        return missing, invalid
+
+    def skip(self):
+        if self._skipped:
+            return True
+        if self.compose.just_phases and self.name not in self.compose.just_phases:
+            return True
+        if self.name in self.compose.skip_phases:
+            return True
+        if self.name in self.compose.conf["skip_phases"]:
+            return True
+        return False
+
+    def start(self):
+        self._skipped = self.skip()
+        if self._skipped:
+            self.compose.log_warning("[SKIP ] %s" % self.msg)
+            self.finished = True
+            return
+        self._start_time = time.time()
+        self.compose.log_info("[BEGIN] %s" % self.msg)
+        self.compose.notifier.send("phase-start", phase_name=self.name)
+        self.run()
+
+    def get_config_block(self, variant, arch=None):
+        """In config for current phase, find a block corresponding to given
+        variant and arch. The arch should be given if and only if the config
+        uses variant/arch mapping.
+        """
+        self.used_patterns = self.used_patterns or set()
+        if arch is not None:
+            return util.get_arch_variant_data(
+                self.compose.conf, self.name, arch, variant, keys=self.used_patterns
+            )
+        else:
+            return util.get_variant_data(
+                self.compose.conf, self.name, variant, keys=self.used_patterns
+            )
+
+    def get_all_patterns(self):
+        """Get all variant patterns from config file for this phase."""
+        if isinstance(self.compose.conf.get(self.name), dict):
+            return set(self.compose.conf.get(self.name, {}).keys())
+        else:
+            return set(x[0] for x in self.compose.conf.get(self.name, []))
+
+    def report_unused_patterns(self):
+        """Log warning about unused parts of the config.
+
+        This is not technically an error, but can help debug when something
+        expected is missing.
+        """
+        all_patterns = self.get_all_patterns()
+        unused_patterns = all_patterns - self.used_patterns
+        if unused_patterns:
+            self.compose.log_warning(
+                "[%s] Patterns in config do not match any variant: %s"
+                % (self.name.upper(), ", ".join(sorted(unused_patterns)))
+            )
+            self.compose.log_info(
+                "Note that variants can be excluded in configuration file"
+            )
+
+    def stop(self):
+        if self.finished:
+            return
+        if hasattr(self, "pool"):
+            self.pool.stop()
+        self.finished = True
+        self.compose.log_info("[DONE ] %s" % self.msg)
+
+        if hasattr(self, "_start_time"):
+            self.compose.log_info(
+                "PHASE %s took %d seconds"
+                % (self.name.upper(), math.ceil(time.time() - self._start_time))
+            )
+
+        if self.used_patterns is not None:
+            # We only want to report this if the config was actually queried.
+            self.report_unused_patterns()
+        self.compose.notifier.send("phase-stop", phase_name=self.name)
+
+    def run(self):
+        raise NotImplementedError
+
+
+class ConfigGuardedPhase(PhaseBase):
+    """A phase that is skipped unless config option is set."""
+
+    def skip(self):
+        if super(ConfigGuardedPhase, self).skip():
+            return True
+        if not self.compose.conf.get(self.name):
+            self.compose.log_info(
+                "Config section '%s' was not found. Skipping." % self.name
+            )
+            return True
+        return False
+
+
+class ImageConfigMixin(object):
+    """
+    A mixin for phase that needs to access image related settings: ksurl,
+    version, target and release.
+
+    First, it checks config object given as argument, then it checks
+    phase-level configuration and finally falls back to global configuration.
+    """
+
+    def __init__(self, *args, **kwargs):
+        super(ImageConfigMixin, self).__init__(*args, **kwargs)
+
+    def get_config(self, cfg, opt):
+        return cfg.get(
+            opt,
+            self.compose.conf.get(
+                "%s_%s" % (self.name, opt), self.compose.conf.get("global_%s" % opt)
+            ),
+        )
+
+    def get_version(self, cfg):
+        """
+        Get version from configuration hierarchy or fall back to release
+        version.
+        """
+        return (
+            util.version_generator(self.compose, self.get_config(cfg, "version"))
+            or self.get_config(cfg, "version")
+            or self.compose.image_version
+        )
+
+    def get_release(self, cfg):
+        """
+        If release is set to a magic string (or explicitly to None -
+        deprecated), replace it with a generated value. Uses configuration
+        passed as argument, phase specific settings and global settings.
+        """
+        for key, conf in [
+            ("release", cfg),
+            ("%s_release" % self.name, self.compose.conf),
+            ("global_release", self.compose.conf),
+        ]:
+            if key in conf:
+                return (
+                    util.version_generator(self.compose, conf[key])
+                    or self.compose.image_release
+                )
+        return None
+
+    def get_ksurl(self, cfg):
+        """
+        Get ksurl from `cfg`. If not present, fall back to phase defined one or
+        global one.
+        """
+        return (
+            cfg.get("ksurl")
+            or self.compose.conf.get("%s_ksurl" % self.name)
+            or self.compose.conf.get("global_ksurl")
+        )
+
+
+class PhaseLoggerMixin(object):
+    """
+    A mixin that can extend a phase with a new logging logger that copy
+    handlers from compose, but with different formatter that includes phase name.
+    """
+
+    def __init__(self, *args, **kwargs):
+        super(PhaseLoggerMixin, self).__init__(*args, **kwargs)
+        self.logger = None
+        if self.compose._logger and self.compose._logger.handlers:
+            self.logger = logging.getLogger(self.name.upper())
+            self.logger.setLevel(logging.DEBUG)
+            format = "%(asctime)s [%(name)-16s] [%(levelname)-8s] %(message)s"
+            import copy
+
+            for handler in self.compose._logger.handlers:
+                hl = copy.copy(handler)
+                hl.setFormatter(logging.Formatter(format, datefmt="%Y-%m-%d %H:%M:%S"))
+                hl.setLevel(logging.DEBUG)
+                self.logger.addHandler(hl)

pungi/phases/buildinstall.py

@@ -0,0 +1,950 @@
+# -*- coding: utf-8 -*-
+
+
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; version 2 of the License.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Library General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, see <https://gnu.org/licenses/>.
+
+
+import errno
+import os
+import time
+import shutil
+import re
+from six.moves import cPickle as pickle
+from copy import copy
+
+from kobo.threads import ThreadPool, WorkerThread
+from kobo.shortcuts import run, force_list
+import kobo.rpmlib
+from productmd.images import Image
+from six.moves import shlex_quote
+
+from pungi.arch import get_valid_arches
+from pungi.util import get_volid, get_arch_variant_data
+from pungi.util import get_file_size, get_mtime, failable, makedirs
+from pungi.util import copy_all, translate_path
+from pungi.wrappers.lorax import LoraxWrapper
+from pungi.wrappers import iso
+from pungi.wrappers.scm import get_file
+from pungi.wrappers.scm import get_file_from_scm
+from pungi.wrappers import kojiwrapper
+from pungi.phases.base import PhaseBase
+from pungi.runroot import Runroot, download_and_extract_archive
+
+
+class BuildinstallPhase(PhaseBase):
+    name = "buildinstall"
+
+    def __init__(self, compose, pkgset_phase=None):
+        PhaseBase.__init__(self, compose)
+        self.pool = ThreadPool(logger=self.compose._logger)
+        # A set of (variant_uid, arch) pairs that completed successfully. This
+        # is needed to skip copying files for failed tasks.
+        self.pool.finished_tasks = set()
+        # A set of (variant_uid, arch) pairs that were reused from previous
+        # compose.
+        self.pool.reused_tasks = set()
+        self.buildinstall_method = self.compose.conf.get("buildinstall_method")
+        self.lorax_use_koji_plugin = self.compose.conf.get("lorax_use_koji_plugin")
+        self.used_lorax = self.buildinstall_method == "lorax"
+        self.pkgset_phase = pkgset_phase
+
+        self.warned_skipped = False
+
+    def skip(self):
+        if PhaseBase.skip(self):
+            return True
+        if not self.compose.conf.get("buildinstall_method"):
+            if not self.warned_skipped:
+                msg = "Not a bootable product. Skipping buildinstall."
+                self.compose.log_debug(msg)
+                self.warned_skipped = True
+            return True
+        return False
+
+    def _get_lorax_cmd(
+        self,
+        repo_baseurl,
+        output_dir,
+        variant,
+        arch,
+        buildarch,
+        volid,
+        final_output_dir,
+    ):
+        noupgrade = True
+        bugurl = None
+        nomacboot = True
+        add_template = []
+        add_arch_template = []
+        add_template_var = []
+        add_arch_template_var = []
+        dracut_args = []
+        rootfs_size = None
+        skip_branding = False
+        squashfs_only = False
+        configuration_file = None
+        configuration_file_source = None
+        version = self.compose.conf.get(
+            "treeinfo_version", self.compose.conf["release_version"]
+        )
+        for data in get_arch_variant_data(
+            self.compose.conf, "lorax_options", arch, variant
+        ):
+            if not data.get("noupgrade", True):
+                noupgrade = False
+            if data.get("bugurl"):
+                bugurl = data.get("bugurl")
+            if not data.get("nomacboot", True):
+                nomacboot = False
+            if "rootfs_size" in data:
+                rootfs_size = data.get("rootfs_size")
+            add_template.extend(data.get("add_template", []))
+            add_arch_template.extend(data.get("add_arch_template", []))
+            add_template_var.extend(data.get("add_template_var", []))
+            add_arch_template_var.extend(data.get("add_arch_template_var", []))
+            dracut_args.extend(data.get("dracut_args", []))
+            skip_branding = data.get("skip_branding", False)
+            configuration_file_source = data.get("configuration_file")
+            squashfs_only = data.get("squashfs_only", False)
+            if "version" in data:
+                version = data["version"]
+        output_dir = os.path.join(output_dir, variant.uid)
+        output_topdir = output_dir
+
+        # The paths module will modify the filename (by inserting arch). But we
+        # only care about the directory anyway.
+        log_dir = _get_log_dir(self.compose, variant, arch)
+        # Place the lorax.conf as specified by
+        # the configuration_file parameter of lorax_options to the log directory.
+        if configuration_file_source:
+            configuration_file_destination = os.path.join(log_dir, "lorax.conf")
+            # Obtain lorax.conf for the buildInstall phase
+            get_file(
+                configuration_file_source,
+                configuration_file_destination,
+                compose=self.compose,
+            )
+            configuration_file = configuration_file_destination
+
+        repos = repo_baseurl[:]
+        repos.extend(
+            get_arch_variant_data(
+                self.compose.conf, "lorax_extra_sources", arch, variant
+            )
+        )
+        if self.compose.has_comps:
+            comps_repo = self.compose.paths.work.comps_repo(arch, variant)
+            if final_output_dir != output_dir or self.lorax_use_koji_plugin:
+                comps_repo = translate_path(self.compose, comps_repo)
+            repos.append(comps_repo)
+
+        if self.lorax_use_koji_plugin:
+            return {
+                "product": self.compose.conf["release_name"],
+                "version": version,
+                "release": version,
+                "sources": force_list(repos),
+                "variant": variant.uid,
+                "installpkgs": variant.buildinstallpackages,
+                "isfinal": self.compose.supported,
+                "buildarch": buildarch,
+                "volid": volid,
+                "nomacboot": nomacboot,
+                "bugurl": bugurl,
+                "add-template": add_template,
+                "add-arch-template": add_arch_template,
+                "add-template-var": add_template_var,
+                "add-arch-template-var": add_arch_template_var,
+                "noupgrade": noupgrade,
+                "rootfs-size": rootfs_size,
+                "dracut-args": dracut_args,
+                "skip_branding": skip_branding,
+                "squashfs_only": squashfs_only,
+                "configuration_file": configuration_file,
+            }
+        else:
+            # If the buildinstall_topdir is set, it means Koji is used for
+            # buildinstall phase and the filesystem with Koji is read-only.
+            # In that case, we have to write logs to buildinstall_topdir and
+            # later copy them back to our local log directory.
+            if self.compose.conf.get("buildinstall_topdir", None):
+                output_dir = os.path.join(output_dir, "results")
+
+            lorax = LoraxWrapper()
+            lorax_cmd = lorax.get_lorax_cmd(
+                self.compose.conf["release_name"],
+                version,
+                version,
+                repos,
+                output_dir,
+                variant=variant.uid,
+                buildinstallpackages=variant.buildinstallpackages,
+                is_final=self.compose.supported,
+                buildarch=buildarch,
+                volid=volid,
+                nomacboot=nomacboot,
+                bugurl=bugurl,
+                add_template=add_template,
+                add_arch_template=add_arch_template,
+                add_template_var=add_template_var,
+                add_arch_template_var=add_arch_template_var,
+                noupgrade=noupgrade,
+                rootfs_size=rootfs_size,
+                log_dir=log_dir,
+                dracut_args=dracut_args,
+                skip_branding=skip_branding,
+                squashfs_only=squashfs_only,
+                configuration_file=configuration_file,
+            )
+            return "rm -rf %s && %s" % (
+                shlex_quote(output_topdir),
+                " ".join([shlex_quote(x) for x in lorax_cmd]),
+            )
+
+    def get_repos(self, arch):
+        repos = []
+        for pkgset in self.pkgset_phase.package_sets:
+            repos.append(pkgset.paths[arch])
+        return repos
+
+    def run(self):
+        disc_type = self.compose.conf["disc_types"].get("dvd", "dvd")
+
+        # Prepare kickstart file for final images.
+        self.pool.kickstart_file = get_kickstart_file(self.compose)
+
+        for arch in self.compose.get_arches():
+            commands = []
+
+            output_dir = self.compose.paths.work.buildinstall_dir(
+                arch, allow_topdir_override=True
+            )
+            final_output_dir = self.compose.paths.work.buildinstall_dir(
+                arch, allow_topdir_override=False
+            )
+            makedirs(final_output_dir)
+            repo_baseurls = self.get_repos(arch)
+            if final_output_dir != output_dir or self.lorax_use_koji_plugin:
+                repo_baseurls = [translate_path(self.compose, r) for r in repo_baseurls]
+
+            if self.buildinstall_method == "lorax":
+                buildarch = get_valid_arches(arch)[0]
+                for variant in self.compose.get_variants(arch=arch, types=["variant"]):
+                    if variant.is_empty:
+                        continue
+
+                    skip = get_arch_variant_data(
+                        self.compose.conf, "buildinstall_skip", arch, variant
+                    )
+                    if skip == [True]:
+                        self.compose.log_info(
+                            "Skipping buildinstall for %s.%s due to config option"
+                            % (variant, arch)
+                        )
+                        continue
+
+                    volid = get_volid(
+                        self.compose, arch, variant=variant, disc_type=disc_type
+                    )
+                    commands.append(
+                        (
+                            variant,
+                            self._get_lorax_cmd(
+                                repo_baseurls,
+                                output_dir,
+                                variant,
+                                arch,
+                                buildarch,
+                                volid,
+                                final_output_dir,
+                            ),
+                        )
+                    )
+            else:
+                raise ValueError(
+                    "Unsupported buildinstall method: %s" % self.buildinstall_method
+                )
+
+            for variant, cmd in commands:
+                self.pool.add(BuildinstallThread(self.pool))
+                self.pool.queue_put(
+                    (self.compose, arch, variant, cmd, self.pkgset_phase)
+                )
+
+        self.pool.start()
+
+    def succeeded(self, variant, arch):
+        # If the phase is skipped, we can treat it as successful. Either there
+        # will be no output, or it's a debug run of compose where anything can
+        # happen.
+        return (
+            super(BuildinstallPhase, self).skip()
+            or (variant.uid if self.used_lorax else None, arch)
+            in self.pool.finished_tasks
+        )
+
+    def reused(self, variant, arch):
+        """
+        Check if buildinstall phase reused previous results for given variant
+        and arch. If the phase is skipped, the results will be considered
+        reused as well.
+        """
+        return (
+            super(BuildinstallPhase, self).skip()
+            or (variant.uid if self.used_lorax else None, arch)
+            in self.pool.reused_tasks
+        )
+
+
+def get_kickstart_file(compose):
+    scm_dict = compose.conf.get("buildinstall_kickstart")
+    if not scm_dict:
+        compose.log_debug("Path to ks.cfg (buildinstall_kickstart) not specified.")
+        return
+
+    msg = "Getting ks.cfg"
+    kickstart_path = os.path.join(compose.paths.work.topdir(arch="global"), "ks.cfg")
+    if os.path.exists(kickstart_path):
+        compose.log_warning("[SKIP ] %s" % msg)
+        return kickstart_path
+
+    compose.log_info("[BEGIN] %s" % msg)
+    if isinstance(scm_dict, dict):
+        kickstart_name = os.path.basename(scm_dict["file"])
+        if scm_dict["scm"] == "file":
+            scm_dict["file"] = os.path.join(compose.config_dir, scm_dict["file"])
+    else:
+        kickstart_name = os.path.basename(scm_dict)
+        scm_dict = os.path.join(compose.config_dir, scm_dict)
+
+    tmp_dir = compose.mkdtemp(prefix="buildinstall_kickstart_")
+    get_file_from_scm(scm_dict, tmp_dir, compose=compose)
+    src = os.path.join(tmp_dir, kickstart_name)
+    shutil.copy2(src, kickstart_path)
+    compose.log_info("[DONE ] %s" % msg)
+    return kickstart_path
+
+
+BOOT_CONFIGS = [
+    "isolinux/isolinux.cfg",
+    "etc/yaboot.conf",
+    "ppc/ppc64/yaboot.conf",
+    "EFI/BOOT/BOOTX64.conf",
+    "EFI/BOOT/grub.cfg",
+]
+BOOT_IMAGES = [
+    "images/efiboot.img",
+]
+
+
+def tweak_configs(path, volid, ks_file, configs=BOOT_CONFIGS, logger=None):
+    """
+    Put escaped volume ID and possibly kickstart file into the boot
+    configuration files.
+    :returns: list of paths to modified config files
+    """
+    volid_escaped = volid.replace(" ", r"\x20").replace("\\", "\\\\")
+    volid_escaped_2 = volid_escaped.replace("\\", "\\\\")
+    found_configs = []
+    for config in configs:
+        config_path = os.path.join(path, config)
+        if not os.path.exists(config_path):
+            continue
+
+        with open(config_path, "r") as f:
+            data = original_data = f.read()
+        os.unlink(config_path)  # break hadlink by removing file writing a new one
+
+        # double-escape volid in yaboot.conf
+        new_volid = volid_escaped_2 if "yaboot" in config else volid_escaped
+
+        ks = (" inst.ks=hd:LABEL=%s:/ks.cfg" % new_volid) if ks_file else ""
+
+        # pre-f18
+        data = re.sub(r":CDLABEL=[^ \n]*", r":CDLABEL=%s%s" % (new_volid, ks), data)
+        # f18+
+        data = re.sub(r":LABEL=[^ \n]*", r":LABEL=%s%s" % (new_volid, ks), data)
+        data = re.sub(r"(search .* -l) '[^'\n]*'", r"\1 '%s'" % volid, data)
+
+        with open(config_path, "w") as f:
+            f.write(data)
+
+        if data != original_data:
+            found_configs.append(config)
+            if logger:
+                # Generally lorax should create file with correct volume id
+                # already. If we don't have a kickstart, this function should
+                # be a no-op.
+                logger.info("Boot config %s changed" % config_path)
+
+    return found_configs
+
+
+# HACK: this is a hack!
+# * it's quite trivial to replace volids
+# * it's not easy to replace menu titles
+# * we probably need to get this into lorax
+def tweak_buildinstall(
+    compose, src, dst, arch, variant, label, volid, kickstart_file=None
+):
+    tmp_dir = compose.mkdtemp(prefix="tweak_buildinstall_")
+
+    # verify src
+    if not os.path.isdir(src):
+        raise OSError(errno.ENOENT, "Directory does not exist: %s" % src)
+
+    # create dst
+    try:
+        os.makedirs(dst)
+    except OSError as ex:
+        if ex.errno != errno.EEXIST:
+            raise
+
+    # copy src to temp
+    # TODO: place temp on the same device as buildinstall dir so we can hardlink
+    cmd = "cp -dRv --preserve=mode,links,timestamps --remove-destination %s/* %s/" % (
+        shlex_quote(src),
+        shlex_quote(tmp_dir),
+    )
+    run(cmd)
+
+    found_configs = tweak_configs(
+        tmp_dir, volid, kickstart_file, logger=compose._logger
+    )
+    if kickstart_file and found_configs:
+        shutil.copy2(kickstart_file, os.path.join(dst, "ks.cfg"))
+
+    images = [os.path.join(tmp_dir, img) for img in BOOT_IMAGES]
+    if found_configs:
+        for image in images:
+            if not os.path.isfile(image):
+                continue
+
+            with iso.mount(
+                image,
+                logger=compose._logger,
+                use_guestmount=compose.conf.get("buildinstall_use_guestmount"),
+            ) as mount_tmp_dir:
+                for config in found_configs:
+                    # Put each modified config file into the image (overwriting the
+                    # original).
+                    config_path = os.path.join(tmp_dir, config)
+                    config_in_image = os.path.join(mount_tmp_dir, config)
+
+                    if os.path.isfile(config_in_image):
+                        cmd = [
+                            "cp",
+                            "-v",
+                            "--remove-destination",
+                            config_path,
+                            config_in_image,
+                        ]
+                        run(cmd)
+
+    # HACK: make buildinstall files world readable
+    run("chmod -R a+rX %s" % shlex_quote(tmp_dir))
+
+    # copy temp to dst
+    cmd = "cp -dRv --preserve=mode,links,timestamps --remove-destination %s/* %s/" % (
+        shlex_quote(tmp_dir),
+        shlex_quote(dst),
+    )
+    run(cmd)
+
+    shutil.rmtree(tmp_dir)
+
+
+def link_boot_iso(compose, arch, variant, can_fail):
+    if arch == "src":
+        return
+
+    disc_type = compose.conf["disc_types"].get("boot", "boot")
+
+    symlink_isos_to = compose.conf.get("symlink_isos_to")
+    os_tree = compose.paths.compose.os_tree(arch, variant)
+    # TODO: find in treeinfo?
+    boot_iso_path = os.path.join(os_tree, "images", "boot.iso")
+    if not os.path.isfile(boot_iso_path):
+        return
+
+    msg = "Linking boot.iso (arch: %s, variant: %s)" % (arch, variant)
+    filename = compose.get_image_name(
+        arch, variant, disc_type=disc_type, disc_num=None, suffix=".iso"
+    )
+    new_boot_iso_path = compose.paths.compose.iso_path(
+        arch, variant, filename, symlink_to=symlink_isos_to
+    )
+    new_boot_iso_relative_path = compose.paths.compose.iso_path(
+        arch, variant, filename, relative=True
+    )
+    if os.path.exists(new_boot_iso_path):
+        # TODO: log
+        compose.log_warning("[SKIP ] %s" % msg)
+        return
+
+    compose.log_info("[BEGIN] %s" % msg)
+    # Try to hardlink, and copy if that fails
+    try:
+        os.link(boot_iso_path, new_boot_iso_path)
+    except OSError:
+        shutil.copy2(boot_iso_path, new_boot_iso_path)
+
+    implant_md5 = iso.get_implanted_md5(new_boot_iso_path)
+    iso_name = os.path.basename(new_boot_iso_path)
+    iso_dir = os.path.dirname(new_boot_iso_path)
+
+    # create iso manifest
+    run(iso.get_manifest_cmd(iso_name), workdir=iso_dir)
+
+    img = Image(compose.im)
+    img.path = new_boot_iso_relative_path
+    img.mtime = get_mtime(new_boot_iso_path)
+    img.size = get_file_size(new_boot_iso_path)
+    img.arch = arch
+    img.type = "boot"
+    img.format = "iso"
+    img.disc_number = 1
+    img.disc_count = 1
+    img.bootable = True
+    img.subvariant = variant.uid
+    img.implant_md5 = implant_md5
+    setattr(img, "can_fail", can_fail)
+    setattr(img, "deliverable", "buildinstall")
+    try:
+        img.volume_id = iso.get_volume_id(
+            new_boot_iso_path,
+            compose.conf.get("createiso_use_xorrisofs"),
+        )
+    except RuntimeError:
+        pass
+    # In this phase we should add to compose only the images that
+    # will be used only as netinstall.
+    # On this step lorax generates environment
+    # for creating isos and create them.
+    # On step `extra_isos` we overwrite the not needed iso `boot Minimal` by
+    # new iso. It already contains necessary packages from incldued variants.
+    if variant.uid in compose.conf['netinstall_variants']:
+        compose.im.add(variant.uid, arch, img)
+    compose.log_info("[DONE ] %s" % msg)
+
+
+class BuildinstallThread(WorkerThread):
+    def process(self, item, num):
+        # The variant is None unless lorax is used as buildinstall method.
+        compose, arch, variant, cmd, pkgset_phase = item
+        can_fail = compose.can_fail(variant, arch, "buildinstall")
+        with failable(compose, can_fail, variant, arch, "buildinstall"):
+            try:
+                self.worker(compose, arch, variant, cmd, pkgset_phase, num)
+            except RuntimeError:
+                self._print_depsolve_error(compose, arch, variant)
+                raise
+
+    def _print_depsolve_error(self, compose, arch, variant):
+        try:
+            log_file = os.path.join(_get_log_dir(compose, variant, arch), "pylorax.log")
+            with open(log_file) as f:
+                matched = False
+                for line in f:
+                    if re.match("Dependency check failed", line):
+                        matched = True
+                    if matched:
+                        compose.log_error(line.rstrip())
+        except Exception:
+            pass
+
+    def _generate_buildinstall_metadata(
+        self, compose, arch, variant, cmd, buildroot_rpms, pkgset_phase
+    ):
+        """
+        Generate buildinstall.metadata dict.
+
+        :param Compose compose: Current compose.
+        :param str arch: Current architecture.
+        :param Variant variant: Compose variant.
+        :param list cmd: List of command line arguments passed to buildinstall task.
+        :param list buildroot_rpms: List of NVRAs of all RPMs installed in the
+            buildinstall task's buildroot.
+        :param PkgsetPhase pkgset_phase: Package set phase instance.
+        :return: The buildinstall.metadata dict.
+        """
+        # Load the list of packages installed in the boot.iso.
+        # The list of installed packages is logged by Lorax in the "pkglists"
+        # directory. There is one file for each installed RPM and the name
+        # of the file is the name of the RPM.
+        # We need to resolve the name of each RPM back to its NVRA.
+        installed_rpms = []
+        log_fname = "buildinstall-%s-logs/dummy" % variant.uid
+        log_dir = os.path.dirname(compose.paths.log.log_file(arch, log_fname, False))
+        pkglists_dir = os.path.join(log_dir, "pkglists")
+        if os.path.exists(pkglists_dir):
+            for pkg_name in os.listdir(pkglists_dir):
+                for pkgset in pkgset_phase.package_sets:
+                    global_pkgset = pkgset["global"]
+                    # We actually do not care from which package_set the RPM
+                    # came from or if there are multiple versions/release of
+                    # the single RPM in more packages sets. We simply include
+                    # all RPMs with this name in the metadata.
+                    # Later when deciding if the buildinstall phase results
+                    # can be reused, we check that all the RPMs with this name
+                    # are still the same in old/new compose.
+                    for rpm_path, rpm_obj in global_pkgset.file_cache.items():
+                        if rpm_obj.name == pkg_name:
+                            installed_rpms.append(rpm_path)
+
+        # Store the metadata in `buildinstall.metadata`.
+        metadata = {
+            "cmd": cmd,
+            "buildroot_rpms": sorted(buildroot_rpms),
+            "installed_rpms": sorted(installed_rpms),
+        }
+        return metadata
+
+    def _write_buildinstall_metadata(
+        self, compose, arch, variant, cmd, buildroot_rpms, pkgset_phase
+    ):
+        """
+        Write buildinstall.metadata file containing all the information about
+        buildinstall phase input and environment.
+
+        This file is later used to decide whether old buildinstall results can
+        be reused instead of generating them again.
+
+        :param Compose compose: Current compose.
+        :param str arch: Current architecture.
+        :param Variant variant: Compose variant.
+        :param list cmd: List of command line arguments passed to buildinstall task.
+        :param list buildroot_rpms: List of NVRAs of all RPMs installed in the
+            buildinstall task's buildroot.
+        :param PkgsetPhase pkgset_phase: Package set phase instance.
+        """
+        # Generate the list of `*-RPMs` log file.
+        log_filename = ("buildinstall-%s" % variant.uid) if variant else "buildinstall"
+        log_file = compose.paths.log.log_file(arch, log_filename + "-RPMs")
+        with open(log_file, "w") as f:
+            f.write("\n".join(buildroot_rpms))
+
+        # Write buildinstall.metadata only if particular variant is defined.
+        # The `variant` is `None` only if old "buildinstall" method is used.
+        if not variant:
+            return
+
+        metadata = self._generate_buildinstall_metadata(
+            compose, arch, variant, cmd, buildroot_rpms, pkgset_phase
+        )
+
+        log_fname = "buildinstall-%s-logs/dummy" % variant.uid
+        log_dir = os.path.dirname(compose.paths.log.log_file(arch, log_fname))
+        metadata_path = os.path.join(log_dir, "buildinstall.metadata")
+        with open(metadata_path, "wb") as f:
+            pickle.dump(metadata, f, protocol=pickle.HIGHEST_PROTOCOL)
+
+    def _load_old_buildinstall_metadata(self, compose, arch, variant):
+        """
+        Helper method to load "buildinstall.metadata" from old compose.
+
+        :param Compose compose: Current compose.
+        :param str arch: Current architecture.
+        :param Variant variant: Compose variant.
+        """
+        if not variant:
+            return None
+
+        log_fname = "buildinstall-%s-logs/dummy" % variant.uid
+        metadata = os.path.join(
+            os.path.dirname(compose.paths.log.log_file(arch, log_fname)),
+            "buildinstall.metadata",
+        )
+        old_metadata = compose.paths.old_compose_path(metadata)
+        if not old_metadata:
+            return None
+
+        compose.log_info("Loading old BUILDINSTALL phase metadata: %s", old_metadata)
+        try:
+            with open(old_metadata, "rb") as f:
+                old_result = pickle.load(f)
+                return old_result
+        except Exception as e:
+            compose.log_debug(
+                "Failed to load old BUILDINSTALL phase metadata %s : %s"
+                % (old_metadata, str(e))
+            )
+            return None
+
+    def _reuse_old_buildinstall_result(self, compose, arch, variant, cmd, pkgset_phase):
+        """
+        Try to reuse old buildinstall results.
+
+        :param Compose compose: Current compose.
+        :param str arch: Current architecture.
+        :param Variant variant: Compose variant.
+        :param list cmd: List of command line arguments passed to buildinstall task.
+        :param list buildroot_rpms: List of NVRAs of all RPMs installed in the
+            buildinstall task's buildroot.
+        :param PkgsetPhase pkgset_phase: Package set phase instance.
+        :return: True if old buildinstall phase results have been reused.
+        """
+        log_msg = "Cannot reuse old BUILDINSTALL phase results - %s"
+
+        if not compose.conf["buildinstall_allow_reuse"]:
+            compose.log_info(log_msg % "reuse of old buildinstall results is disabled.")
+            return
+
+        # Load the old buildinstall.metadata.
+        old_metadata = self._load_old_buildinstall_metadata(compose, arch, variant)
+        if old_metadata is None:
+            compose.log_info(log_msg % "no old BUILDINSTALL metadata.")
+            return
+
+        # For now try to reuse only if pungi_buildinstall plugin is used.
+        # This is the easiest approach, because we later need to filter out
+        # some parts of `cmd` and for pungi_buildinstall, the `cmd` is a dict
+        # which makes this easy.
+        if not isinstance(old_metadata["cmd"], dict) or not isinstance(cmd, dict):
+            compose.log_info(log_msg % "pungi_buildinstall plugin is not used.")
+            return
+
+        # Filter out "outputdir" and "sources" because they change every time.
+        # The "sources" are not important, because we check the buildinstall
+        # input on RPM level.
+        cmd_copy = copy(cmd)
+        for key in ["outputdir", "sources"]:
+            cmd_copy.pop(key, None)
+            old_metadata["cmd"].pop(key, None)
+
+        # Do not reuse if command line arguments are not the same.
+        if old_metadata["cmd"] != cmd_copy:
+            compose.log_info(log_msg % "lorax command line arguments differ.")
+            return
+
+        # Check that the RPMs installed in the old boot.iso exists in the very
+        # same versions/releases in this compose.
+        for rpm_path in old_metadata["installed_rpms"]:
+            found = False
+            for pkgset in pkgset_phase.package_sets:
+                global_pkgset = pkgset["global"]
+                if rpm_path in global_pkgset.file_cache:
+                    found = True
+                    break
+            if not found:
+                compose.log_info(
+                    log_msg % "RPM %s does not exist in new compose." % rpm_path
+                )
+                return
+
+        # Ask Koji for all the RPMs in the `runroot_tag` and check that
+        # those installed in the old buildinstall buildroot are still in the
+        # very same versions/releases.
+        koji_wrapper = kojiwrapper.KojiWrapper(compose)
+        rpms = koji_wrapper.koji_proxy.listTaggedRPMS(
+            compose.conf.get("runroot_tag"), inherit=True, latest=True
+        )[0]
+        rpm_nvras = set()
+        for rpm in rpms:
+            rpm_nvras.add(kobo.rpmlib.make_nvra(rpm, add_rpm=False, force_epoch=False))
+        for old_nvra in old_metadata["buildroot_rpms"]:
+            if old_nvra not in rpm_nvras:
+                compose.log_info(
+                    log_msg % "RPM %s does not exist in new buildroot." % old_nvra
+                )
+                return
+
+        # We can reuse the old buildinstall results!
+        compose.log_info("Reusing old BUILDINSTALL phase output")
+
+        # Copy old buildinstall output to this this compose.
+        final_output_dir = compose.paths.work.buildinstall_dir(arch, variant=variant)
+        old_final_output_dir = compose.paths.old_compose_path(final_output_dir)
+        copy_all(old_final_output_dir, final_output_dir)
+
+        # Copy old buildinstall logs to this compose.
+        log_fname = "buildinstall-%s-logs/dummy" % variant.uid
+        final_log_dir = os.path.dirname(compose.paths.log.log_file(arch, log_fname))
+        old_final_log_dir = compose.paths.old_compose_path(final_log_dir)
+        if not os.path.exists(final_log_dir):
+            makedirs(final_log_dir)
+        copy_all(old_final_log_dir, final_log_dir)
+
+        # Write the buildinstall metadata so next compose can reuse this compose.
+        self._write_buildinstall_metadata(
+            compose, arch, variant, cmd, old_metadata["buildroot_rpms"], pkgset_phase
+        )
+
+        return True
+
+    def worker(self, compose, arch, variant, cmd, pkgset_phase, num):
+        buildinstall_method = compose.conf["buildinstall_method"]
+        lorax_use_koji_plugin = compose.conf["lorax_use_koji_plugin"]
+        log_filename = ("buildinstall-%s" % variant.uid) if variant else "buildinstall"
+        log_file = compose.paths.log.log_file(arch, log_filename)
+
+        msg = "Running buildinstall for arch %s, variant %s" % (arch, variant)
+
+        output_dir = compose.paths.work.buildinstall_dir(
+            arch, allow_topdir_override=True, variant=variant
+        )
+        final_output_dir = compose.paths.work.buildinstall_dir(arch, variant=variant)
+
+        if (
+            os.path.isdir(output_dir)
+            and os.listdir(output_dir)
+            or os.path.isdir(final_output_dir)
+            and os.listdir(final_output_dir)
+        ):
+            # output dir is *not* empty -> SKIP
+            self.pool.log_warning(
+                "[SKIP ] Buildinstall for arch %s, variant %s" % (arch, variant)
+            )
+            return
+
+        self.pool.log_info("[BEGIN] %s" % msg)
+
+        # Get list of packages which are needed in runroot.
+        packages = []
+        chown_paths = [output_dir]
+        if buildinstall_method == "lorax":
+            packages += ["lorax"]
+            chown_paths.append(_get_log_dir(compose, variant, arch))
+        packages += get_arch_variant_data(
+            compose.conf, "buildinstall_packages", arch, variant
+        )
+        if self._reuse_old_buildinstall_result(
+            compose, arch, variant, cmd, pkgset_phase
+        ):
+            self.copy_files(compose, variant, arch)
+            self.pool.finished_tasks.add((variant.uid if variant else None, arch))
+            self.pool.reused_tasks.add((variant.uid if variant else None, arch))
+            self.pool.log_info("[DONE ] %s" % msg)
+            return
+
+        # This should avoid a possible race condition with multiple processes
+        # trying to get a kerberos ticket at the same time.
+        # Kerberos authentication failed:
+        #   Permission denied in replay cache code (-1765328215)
+        time.sleep(num * 3)
+
+        # Start the runroot task.
+        runroot = Runroot(compose, phase="buildinstall")
+        task_id = None
+        if buildinstall_method == "lorax" and lorax_use_koji_plugin:
+            task_id = runroot.run_pungi_buildinstall(
+                cmd,
+                log_file=log_file,
+                arch=arch,
+                packages=packages,
+                weight=compose.conf["runroot_weights"].get("buildinstall"),
+            )
+        else:
+            try:
+                lorax_log_dir = _get_log_dir(compose, variant, arch)
+            except Exception:
+                lorax_log_dir = None
+            runroot.run(
+                cmd,
+                log_file=log_file,
+                arch=arch,
+                packages=packages,
+                mounts=[compose.topdir],
+                weight=compose.conf["runroot_weights"].get("buildinstall"),
+                chown_paths=chown_paths,
+                log_dir=lorax_log_dir,
+            )
+
+        if final_output_dir != output_dir:
+            if not os.path.exists(final_output_dir):
+                makedirs(final_output_dir)
+            results_dir = os.path.join(output_dir, "results")
+            copy_all(results_dir, final_output_dir)
+
+            # Get the log_dir into which we should copy the resulting log files.
+            log_fname = "buildinstall-%s-logs/dummy" % variant.uid
+            final_log_dir = os.path.dirname(compose.paths.log.log_file(arch, log_fname))
+            if not os.path.exists(final_log_dir):
+                makedirs(final_log_dir)
+            log_dir = os.path.join(output_dir, "logs")
+            copy_all(log_dir, final_log_dir)
+        elif lorax_use_koji_plugin:
+            # If Koji pungi-buildinstall is used, then the buildinstall results
+            # are attached as outputs to the Koji task. Download and unpack
+            # them to the correct location.
+            download_and_extract_archive(
+                compose, task_id, "results.tar.gz", final_output_dir
+            )
+
+            # Download the logs into proper location too.
+            log_fname = "buildinstall-%s-logs/dummy" % variant.uid
+            final_log_dir = os.path.dirname(compose.paths.log.log_file(arch, log_fname))
+            download_and_extract_archive(compose, task_id, "logs.tar.gz", final_log_dir)
+
+        rpms = runroot.get_buildroot_rpms()
+        self._write_buildinstall_metadata(
+            compose, arch, variant, cmd, rpms, pkgset_phase
+        )
+
+        self.copy_files(compose, variant, arch)
+
+        self.pool.finished_tasks.add((variant.uid if variant else None, arch))
+
+        self.pool.log_info("[DONE ] %s" % msg)
+
+    def copy_files(self, compose, variant, arch):
+        disc_type = compose.conf["disc_types"].get("dvd", "dvd")
+
+        buildinstall_dir = compose.paths.work.buildinstall_dir(arch)
+
+        # Lorax runs per-variant, so we need to tweak the source path
+        # to include variant.
+        if variant:
+            buildinstall_dir = os.path.join(buildinstall_dir, variant.uid)
+
+        # Find all relevant variants if lorax is not used.
+        variants = (
+            [variant]
+            if variant
+            else compose.get_variants(arch=arch, types=["self", "variant"])
+        )
+        for var in variants:
+            os_tree = compose.paths.compose.os_tree(arch, var)
+            # TODO: label is not used
+            label = ""
+            volid = get_volid(compose, arch, var, disc_type=disc_type)
+            can_fail = compose.can_fail(var, arch, "buildinstall")
+            tweak_buildinstall(
+                compose,
+                buildinstall_dir,
+                os_tree,
+                arch,
+                var.uid,
+                label,
+                volid,
+                self.pool.kickstart_file,
+            )
+            link_boot_iso(compose, arch, var, can_fail)
+
+
+def _get_log_dir(compose, variant, arch):
+    """Find directory where to store lorax logs in. If it's inside the compose,
+    create the directory.
+    """
+    if compose.conf.get("buildinstall_topdir"):
+        log_dir = compose.paths.work.buildinstall_dir(
+            arch, allow_topdir_override=True, variant=variant
+        )
+        return os.path.join(log_dir, "logs")
+
+    # The paths module will modify the filename (by inserting arch). But we
+    # only care about the directory anyway.
+    log_filename = "buildinstall-%s-logs/dummy" % variant.uid
+    log_dir = os.path.dirname(compose.paths.log.log_file(arch, log_filename))
+    makedirs(log_dir)
+    return log_dir

pungi/phases/createiso.py

@@ -0,0 +1,932 @@
+# -*- coding: utf-8 -*-
+
+
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; version 2 of the License.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Library General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, see <https://gnu.org/licenses/>.
+
+
+import itertools
+import os
+import random
+import shutil
+import stat
+import json
+
+import productmd.treeinfo
+from productmd.images import Image
+from kobo.threads import ThreadPool, WorkerThread
+from kobo.shortcuts import run, relative_path, compute_file_checksums
+from six.moves import shlex_quote
+
+from pungi.wrappers import iso
+from pungi.wrappers.createrepo import CreaterepoWrapper
+from pungi.wrappers import kojiwrapper
+from pungi.phases.base import PhaseBase, PhaseLoggerMixin
+from pungi.util import (
+    makedirs,
+    get_volid,
+    get_arch_variant_data,
+    failable,
+    get_file_size,
+    get_mtime,
+    read_json_file,
+)
+from pungi.media_split import MediaSplitter, convert_media_size
+from pungi.compose_metadata.discinfo import read_discinfo, write_discinfo
+from pungi.runroot import Runroot
+
+from .. import createiso
+
+
+class CreateisoPhase(PhaseLoggerMixin, PhaseBase):
+    name = "createiso"
+
+    def __init__(self, compose, buildinstall_phase):
+        super(CreateisoPhase, self).__init__(compose)
+        self.pool = ThreadPool(logger=self.logger)
+        self.bi = buildinstall_phase
+
+    def _find_rpms(self, path):
+        """Check if there are some RPMs in the path."""
+        for _, _, files in os.walk(path):
+            for fn in files:
+                if fn.endswith(".rpm"):
+                    return True
+        return False
+
+    def _is_bootable(self, variant, arch):
+        if arch == "src":
+            return False
+        if variant.type != "variant":
+            return False
+        skip = get_arch_variant_data(
+            self.compose.conf, "buildinstall_skip", arch, variant
+        )
+        if skip == [True]:
+            # Buildinstall is skipped for this tree. Can't create a bootable ISO.
+            return False
+        return bool(self.compose.conf.get("buildinstall_method", ""))
+
+    def _metadata_path(self, variant, arch, disc_num, disc_count):
+        return self.compose.paths.log.log_file(
+            arch,
+            "createiso-%s-%d-%d" % (variant.uid, disc_num, disc_count),
+            ext="json",
+        )
+
+    def save_reuse_metadata(self, cmd, variant, arch, opts):
+        """Save metadata for future composes to verify if the compose can be reused."""
+        metadata = {
+            "cmd": cmd,
+            "opts": opts._asdict(),
+        }
+
+        metadata_path = self._metadata_path(
+            variant, arch, cmd["disc_num"], cmd["disc_count"]
+        )
+        with open(metadata_path, "w") as f:
+            json.dump(metadata, f, indent=2)
+        return metadata
+
+    def _load_old_metadata(self, cmd, variant, arch):
+        metadata_path = self._metadata_path(
+            variant, arch, cmd["disc_num"], cmd["disc_count"]
+        )
+        old_path = self.compose.paths.old_compose_path(metadata_path)
+        self.logger.info(
+            "Loading old metadata for %s.%s from: %s", variant, arch, old_path
+        )
+        try:
+            return read_json_file(old_path)
+        except Exception:
+            return None
+
+    def perform_reuse(self, cmd, variant, arch, opts, iso_path):
+        """
+        Copy all related files from old compose to the new one. As a last step
+        add the new image to metadata.
+        """
+        linker = OldFileLinker(self.logger)
+        old_file_name = os.path.basename(iso_path)
+        current_file_name = os.path.basename(cmd["iso_path"])
+        try:
+            # Hardlink ISO and manifest
+            for suffix in ("", ".manifest"):
+                linker.link(iso_path + suffix, cmd["iso_path"] + suffix)
+            # Copy log files
+            # The log file name includes filename of the image, so we need to
+            # find old file with the old name, and rename it to the new name.
+            log_file = self.compose.paths.log.log_file(
+                arch, "createiso-%s" % current_file_name
+            )
+            old_log_file = self.compose.paths.old_compose_path(
+                self.compose.paths.log.log_file(arch, "createiso-%s" % old_file_name)
+            )
+            linker.link(old_log_file, log_file)
+            # Copy jigdo files
+            if opts.jigdo_dir:
+                old_jigdo_dir = self.compose.paths.old_compose_path(opts.jigdo_dir)
+                for suffix in (".template", ".jigdo"):
+                    linker.link(
+                        os.path.join(old_jigdo_dir, old_file_name) + suffix,
+                        os.path.join(opts.jigdo_dir, current_file_name) + suffix,
+                    )
+        except Exception:
+            # A problem happened while linking some file, let's clean up
+            # everything.
+            linker.abort()
+            raise
+        # Add image to manifest
+        add_iso_to_metadata(
+            self.compose,
+            variant,
+            arch,
+            cmd["iso_path"],
+            bootable=cmd["bootable"],
+            disc_num=cmd["disc_num"],
+            disc_count=cmd["disc_count"],
+        )
+        if self.compose.notifier:
+            self.compose.notifier.send(
+                "createiso-imagedone",
+                file=cmd["iso_path"],
+                arch=arch,
+                variant=str(variant),
+            )
+
+    def try_reuse(self, cmd, variant, arch, opts):
+        """Try to reuse image from previous compose.
+
+        :returns bool: True if reuse was successful, False otherwise
+        """
+        if not self.compose.conf["createiso_allow_reuse"]:
+            return
+
+        log_msg = "Cannot reuse ISO for %s.%s" % (variant, arch)
+        current_metadata = self.save_reuse_metadata(cmd, variant, arch, opts)
+
+        if opts.buildinstall_method and not self.bi.reused(variant, arch):
+            # If buildinstall phase was not reused for some reason, we can not
+            # reuse any bootable image. If a package change caused rebuild of
+            # boot.iso, we would catch it here too, but there could be a
+            # configuration change in lorax template which would remain
+            # undetected.
+            self.logger.info("%s - boot configuration changed", log_msg)
+            return False
+
+        # Check old compose configuration: extra_files and product_ids can be
+        # reflected on ISO.
+        old_config = self.compose.load_old_compose_config()
+        if not old_config:
+            self.logger.info("%s - no config for old compose", log_msg)
+            return False
+
+        # Disable reuse if unsigned packages are allowed. The older compose
+        # could have unsigned packages, and those may have been signed since
+        # then. We want to regenerate the ISO to have signatures.
+        if None in self.compose.conf["sigkeys"]:
+            self.logger.info("%s - unsigned packages are allowed", log_msg)
+            return False
+
+        # Convert current configuration to JSON and back to encode it similarly
+        # to the old one
+        config = json.loads(json.dumps(self.compose.conf))
+        for opt in self.compose.conf:
+            # Skip a selection of options: these affect what packages can be
+            # included, which we explicitly check later on.
+            config_whitelist = set(
+                [
+                    "gather_lookaside_repos",
+                    "pkgset_koji_builds",
+                    "pkgset_koji_scratch_tasks",
+                    "pkgset_koji_module_builds",
+                ]
+            )
+            # Skip irrelevant options
+            config_whitelist.update(["osbs", "osbuild"])
+            if opt in config_whitelist:
+                continue
+
+            if old_config.get(opt) != config.get(opt):
+                self.logger.info("%s - option %s differs", log_msg, opt)
+                return False
+
+        old_metadata = self._load_old_metadata(cmd, variant, arch)
+        if not old_metadata:
+            self.logger.info("%s - no old metadata found", log_msg)
+            return False
+
+        # Test if volume ID matches - volid can be generated dynamically based on
+        # other values, and could change even if nothing else is different.
+        if current_metadata["opts"]["volid"] != old_metadata["opts"]["volid"]:
+            self.logger.info("%s - volume ID differs", log_msg)
+            return False
+
+        # Compare packages on the ISO.
+        if compare_packages(
+            old_metadata["opts"]["graft_points"],
+            current_metadata["opts"]["graft_points"],
+        ):
+            self.logger.info("%s - packages differ", log_msg)
+            return False
+
+        try:
+            self.perform_reuse(
+                cmd,
+                variant,
+                arch,
+                opts,
+                old_metadata["cmd"]["iso_path"],
+            )
+            return True
+        except Exception as exc:
+            self.compose.log_error(
+                "Error while reusing ISO for %s.%s: %s", variant, arch, exc
+            )
+            self.compose.traceback("createiso-reuse-%s-%s" % (variant, arch))
+            return False
+
+    def run(self):
+        symlink_isos_to = self.compose.conf.get("symlink_isos_to")
+        disc_type = self.compose.conf["disc_types"].get("dvd", "dvd")
+        deliverables = []
+
+        commands = []
+        for variant in self.compose.get_variants(
+            types=["variant", "layered-product", "optional"]
+        ):
+            if variant.is_empty:
+                continue
+            for arch in variant.arches + ["src"]:
+                skip_iso = get_arch_variant_data(
+                    self.compose.conf, "createiso_skip", arch, variant
+                )
+                if skip_iso == [True]:
+                    self.logger.info(
+                        "Skipping createiso for %s.%s due to config option"
+                        % (variant, arch)
+                    )
+                    continue
+
+                volid = get_volid(self.compose, arch, variant, disc_type=disc_type)
+                os_tree = self.compose.paths.compose.os_tree(arch, variant)
+
+                iso_dir = self.compose.paths.compose.iso_dir(
+                    arch, variant, symlink_to=symlink_isos_to
+                )
+                if not iso_dir:
+                    continue
+
+                if not self._find_rpms(os_tree):
+                    self.logger.warning(
+                        "No RPMs found for %s.%s, skipping ISO" % (variant.uid, arch)
+                    )
+                    continue
+
+                bootable = self._is_bootable(variant, arch)
+
+                if bootable and not self.bi.succeeded(variant, arch):
+                    self.logger.warning(
+                        "ISO should be bootable, but buildinstall failed. "
+                        "Skipping for %s.%s" % (variant, arch)
+                    )
+                    continue
+
+                split_iso_data = split_iso(
+                    self.compose, arch, variant, no_split=bootable, logger=self.logger
+                )
+                disc_count = len(split_iso_data)
+
+                for disc_num, iso_data in enumerate(split_iso_data):
+                    disc_num += 1
+
+                    filename = self.compose.get_image_name(
+                        arch, variant, disc_type=disc_type, disc_num=disc_num
+                    )
+                    iso_path = self.compose.paths.compose.iso_path(
+                        arch, variant, filename, symlink_to=symlink_isos_to
+                    )
+                    if os.path.isfile(iso_path):
+                        self.logger.warning(
+                            "Skipping mkisofs, image already exists: %s", iso_path
+                        )
+                        continue
+                    deliverables.append(iso_path)
+
+                    graft_points = prepare_iso(
+                        self.compose,
+                        arch,
+                        variant,
+                        disc_num=disc_num,
+                        disc_count=disc_count,
+                        split_iso_data=iso_data,
+                    )
+
+                    cmd = {
+                        "iso_path": iso_path,
+                        "bootable": bootable,
+                        "cmd": [],
+                        "label": "",  # currently not used
+                        "disc_num": disc_num,
+                        "disc_count": disc_count,
+                    }
+
+                    if os.path.islink(iso_dir):
+                        cmd["mount"] = os.path.abspath(
+                            os.path.join(os.path.dirname(iso_dir), os.readlink(iso_dir))
+                        )
+
+                    opts = createiso.CreateIsoOpts(
+                        output_dir=iso_dir,
+                        iso_name=filename,
+                        volid=volid,
+                        graft_points=graft_points,
+                        arch=arch,
+                        supported=self.compose.supported,
+                        hfs_compat=self.compose.conf["iso_hfs_ppc64le_compatible"],
+                        use_xorrisofs=self.compose.conf.get("createiso_use_xorrisofs"),
+                        iso_level=get_iso_level_config(self.compose, variant, arch),
+                    )
+
+                    if bootable:
+                        opts = opts._replace(
+                            buildinstall_method=self.compose.conf[
+                                "buildinstall_method"
+                            ],
+                            boot_iso=os.path.join(os_tree, "images", "boot.iso"),
+                        )
+
+                    if self.compose.conf["create_jigdo"]:
+                        jigdo_dir = self.compose.paths.compose.jigdo_dir(arch, variant)
+                        opts = opts._replace(jigdo_dir=jigdo_dir, os_tree=os_tree)
+
+                    # Try to reuse
+                    if self.try_reuse(cmd, variant, arch, opts):
+                        # Reuse was successful, go to next ISO
+                        continue
+
+                    script_dir = self.compose.paths.work.tmp_dir(arch, variant)
+                    opts = opts._replace(script_dir=script_dir)
+                    script_file = os.path.join(script_dir, "createiso-%s.sh" % filename)
+                    with open(script_file, "w") as f:
+                        createiso.write_script(opts, f)
+                    cmd["cmd"] = ["bash", script_file]
+                    commands.append((cmd, variant, arch))
+
+        if self.compose.notifier:
+            self.compose.notifier.send("createiso-targets", deliverables=deliverables)
+
+        for cmd, variant, arch in commands:
+            self.pool.add(CreateIsoThread(self.pool))
+            self.pool.queue_put((self.compose, cmd, variant, arch))
+
+        self.pool.start()
+
+
+def read_packages(graft_points):
+    """Read packages that were listed in given graft points file.
+
+    Only files under Packages directory are considered. Particularly this
+    excludes .discinfo, .treeinfo and media.repo as well as repodata and
+    any extra files.
+
+    Extra files are easier to check by configuration (same name doesn't
+    imply same content). Repodata depend entirely on included packages (and
+    possibly product id certificate), but are affected by current time
+    which can change checksum despite data being the same.
+    """
+    with open(graft_points) as f:
+        return set(
+            line.split("=", 1)[0]
+            for line in f
+            if line.startswith("Packages/") or "/Packages/" in line
+        )
+
+
+def compare_packages(old_graft_points, new_graft_points):
+    """Read packages from the two files and compare them.
+
+    :returns bool: True if there are differences, False otherwise
+    """
+    old_files = read_packages(old_graft_points)
+    new_files = read_packages(new_graft_points)
+    return old_files != new_files
+
+
+class CreateIsoThread(WorkerThread):
+    def fail(self, compose, cmd, variant, arch):
+        self.pool.log_error("CreateISO failed, removing ISO: %s" % cmd["iso_path"])
+        try:
+            # remove incomplete ISO
+            os.unlink(cmd["iso_path"])
+            # TODO: remove jigdo & template
+        except OSError:
+            pass
+        if compose.notifier:
+            compose.notifier.send(
+                "createiso-imagefail",
+                file=cmd["iso_path"],
+                arch=arch,
+                variant=str(variant),
+            )
+
+    def process(self, item, num):
+        compose, cmd, variant, arch = item
+        can_fail = compose.can_fail(variant, arch, "iso")
+        with failable(
+            compose, can_fail, variant, arch, "iso", logger=self.pool._logger
+        ):
+            self.worker(compose, cmd, variant, arch, num)
+
+    def worker(self, compose, cmd, variant, arch, num):
+        mounts = [compose.topdir]
+        if "mount" in cmd:
+            mounts.append(cmd["mount"])
+
+        bootable = cmd["bootable"]
+        log_file = compose.paths.log.log_file(
+            arch, "createiso-%s" % os.path.basename(cmd["iso_path"])
+        )
+
+        msg = "Creating ISO (arch: %s, variant: %s): %s" % (
+            arch,
+            variant,
+            os.path.basename(cmd["iso_path"]),
+        )
+        self.pool.log_info("[BEGIN] %s" % msg)
+
+        try:
+            run_createiso_command(
+                num,
+                compose,
+                bootable,
+                arch,
+                cmd["cmd"],
+                mounts,
+                log_file,
+                cmd["iso_path"],
+            )
+        except Exception:
+            self.fail(compose, cmd, variant, arch)
+            raise
+
+        add_iso_to_metadata(
+            compose,
+            variant,
+            arch,
+            cmd["iso_path"],
+            cmd["bootable"],
+            cmd["disc_num"],
+            cmd["disc_count"],
+        )
+
+        # Delete staging directory if present.
+        staging_dir = compose.paths.work.iso_staging_dir(
+            arch, variant, filename=os.path.basename(cmd["iso_path"]), create_dir=False
+        )
+        if os.path.exists(staging_dir):
+            try:
+                shutil.rmtree(staging_dir)
+            except Exception as e:
+                self.pool.log_warning(
+                    "Failed to clean up staging dir: %s %s" % (staging_dir, str(e))
+                )
+
+        self.pool.log_info("[DONE ] %s" % msg)
+        if compose.notifier:
+            compose.notifier.send(
+                "createiso-imagedone",
+                file=cmd["iso_path"],
+                arch=arch,
+                variant=str(variant),
+            )
+
+
+def add_iso_to_metadata(
+    compose,
+    variant,
+    arch,
+    iso_path,
+    bootable,
+    disc_num=1,
+    disc_count=1,
+    additional_variants=None,
+):
+    img = Image(compose.im)
+    img.path = iso_path.replace(compose.paths.compose.topdir(), "").lstrip("/")
+    img.mtime = get_mtime(iso_path)
+    img.size = get_file_size(iso_path)
+    img.arch = arch
+    # XXX: HARDCODED
+    img.type = "dvd"
+    img.format = "iso"
+    img.disc_number = disc_num
+    img.disc_count = disc_count
+    img.bootable = bootable
+    img.subvariant = variant.uid
+    img.implant_md5 = iso.get_implanted_md5(iso_path, logger=compose._logger)
+    if additional_variants:
+        img.unified = True
+        img.additional_variants = additional_variants
+    setattr(img, "can_fail", compose.can_fail(variant, arch, "iso"))
+    setattr(img, "deliverable", "iso")
+    try:
+        img.volume_id = iso.get_volume_id(
+            iso_path,
+            compose.conf.get("createiso_use_xorrisofs"),
+        )
+    except RuntimeError:
+        pass
+    if arch == "src":
+        for variant_arch in variant.arches:
+            compose.im.add(variant.uid, variant_arch, img)
+    else:
+        compose.im.add(variant.uid, arch, img)
+    return img
+
+
+def run_createiso_command(
+    num, compose, bootable, arch, cmd, mounts, log_file, iso_path
+):
+    packages = [
+        "coreutils",
+        "xorriso" if compose.conf.get("createiso_use_xorrisofs") else "genisoimage",
+        "isomd5sum",
+    ]
+    if compose.conf["create_jigdo"]:
+        packages.append("jigdo")
+    if bootable:
+        extra_packages = {
+            "lorax": ["lorax", "which"],
+        }
+        packages.extend(extra_packages[compose.conf["buildinstall_method"]])
+
+    runroot = Runroot(compose, phase="createiso")
+
+    build_arch = arch
+    if runroot.runroot_method == "koji" and not bootable:
+        runroot_tag = compose.conf["runroot_tag"]
+        koji_wrapper = kojiwrapper.KojiWrapper(compose)
+        koji_proxy = koji_wrapper.koji_proxy
+        tag_info = koji_proxy.getTag(runroot_tag)
+        if not tag_info:
+            raise RuntimeError('Tag "%s" does not exist.' % runroot_tag)
+        tag_arches = tag_info["arches"].split(" ")
+
+        if "x86_64" in tag_arches:
+            # assign non-bootable images to x86_64 if possible
+            build_arch = "x86_64"
+        elif build_arch == "src":
+            # pick random arch from available runroot tag arches
+            build_arch = random.choice(tag_arches)
+
+    runroot.run(
+        cmd,
+        log_file=log_file,
+        arch=build_arch,
+        packages=packages,
+        mounts=mounts,
+        weight=compose.conf["runroot_weights"].get("createiso"),
+    )
+
+    if bootable and compose.conf.get("createiso_use_xorrisofs"):
+        fix_treeinfo_checksums(compose, iso_path, arch)
+
+
+def fix_treeinfo_checksums(compose, iso_path, arch):
+    """It is possible for the ISO to contain a .treefile with incorrect
+    checksums. By modifying the ISO (adding files) some of the images may
+    change.
+
+    This function fixes that after the fact by looking for incorrect checksums,
+    recalculating them and updating the .treeinfo file. Since the size of the
+    file doesn't change, this seems to not change any images.
+    """
+    modified = False
+    with iso.mount(iso_path, compose._logger) as mountpoint:
+        ti = productmd.TreeInfo()
+        ti.load(os.path.join(mountpoint, ".treeinfo"))
+        for image, (type_, expected) in ti.checksums.checksums.items():
+            checksums = compute_file_checksums(os.path.join(mountpoint, image), [type_])
+            actual = checksums[type_]
+            if actual == expected:
+                # Everything fine here, skip to next image.
+                continue
+
+            compose.log_debug("%s: %s: checksum mismatch", iso_path, image)
+            # Update treeinfo with correct checksum
+            ti.checksums.checksums[image] = (type_, actual)
+            modified = True
+
+    if not modified:
+        compose.log_debug("%s: All checksums match, nothing to do.", iso_path)
+        return
+
+    try:
+        tmpdir = compose.mkdtemp(arch, prefix="fix-checksum-")
+        # Write modified .treeinfo
+        ti_path = os.path.join(tmpdir, ".treeinfo")
+        compose.log_debug("Storing modified .treeinfo in %s", ti_path)
+        ti.dump(ti_path)
+        # Write a modified DVD into a temporary path, that is atomically moved
+        # over the original file.
+        fixed_path = os.path.join(tmpdir, "fixed-checksum-dvd.iso")
+        cmd = ["xorriso"]
+        cmd.extend(
+            itertools.chain.from_iterable(
+                iso.xorriso_commands(arch, iso_path, fixed_path)
+            )
+        )
+        cmd.extend(["-map", ti_path, ".treeinfo"])
+        run(
+            cmd,
+            logfile=compose.paths.log.log_file(
+                arch, "checksum-fix_generate_%s" % os.path.basename(iso_path)
+            ),
+        )
+        # The modified ISO no longer has implanted MD5, so that needs to be
+        # fixed again.
+        compose.log_debug("Implanting new MD5 to %s", fixed_path)
+        run(
+            iso.get_implantisomd5_cmd(fixed_path, compose.supported),
+            logfile=compose.paths.log.log_file(
+                arch, "checksum-fix_implantisomd5_%s" % os.path.basename(iso_path)
+            ),
+        )
+        # All done, move the updated image to the final location.
+        compose.log_debug("Updating %s", iso_path)
+        os.rename(fixed_path, iso_path)
+    finally:
+        shutil.rmtree(tmpdir)
+
+
+def split_iso(compose, arch, variant, no_split=False, logger=None):
+    """
+    Split contents of the os/ directory for given tree into chunks fitting on ISO.
+
+    All files from the directory are taken except for possible boot.iso image.
+    Files added in extra_files phase are put on all disks.
+
+    If `no_split` is set, we will pretend that the media is practically
+    infinite so that everything goes on single disc. A warning is printed if
+    the size is bigger than configured.
+    """
+    if not logger:
+        logger = compose._logger
+    media_size = compose.conf["iso_size"]
+    media_reserve = compose.conf["split_iso_reserve"]
+    split_size = convert_media_size(media_size) - convert_media_size(media_reserve)
+    real_size = None if no_split else split_size
+
+    ms = MediaSplitter(real_size, compose, logger=logger)
+
+    os_tree = compose.paths.compose.os_tree(arch, variant)
+    extra_files_dir = compose.paths.work.extra_files_dir(arch, variant)
+
+    # scan extra files to mark them "sticky" -> they'll be on all media after split
+    extra_files = set(["media.repo"])
+    for root, dirs, files in os.walk(extra_files_dir):
+        for fn in files:
+            path = os.path.join(root, fn)
+            rel_path = relative_path(path, extra_files_dir.rstrip("/") + "/")
+            extra_files.add(rel_path)
+
+    packages = []
+    all_files = []
+    all_files_ignore = []
+
+    ti = productmd.treeinfo.TreeInfo()
+    ti.load(os.path.join(os_tree, ".treeinfo"))
+    boot_iso_rpath = ti.images.images.get(arch, {}).get("boot.iso", None)
+    if boot_iso_rpath:
+        all_files_ignore.append(boot_iso_rpath)
+    if all_files_ignore:
+        logger.debug("split_iso all_files_ignore = %s" % ", ".join(all_files_ignore))
+
+    for root, dirs, files in os.walk(os_tree):
+        for dn in dirs[:]:
+            repo_dir = os.path.join(root, dn)
+            if repo_dir == os.path.join(
+                compose.paths.compose.repository(arch, variant), "repodata"
+            ):
+                dirs.remove(dn)
+
+        for fn in files:
+            path = os.path.join(root, fn)
+            rel_path = relative_path(path, os_tree.rstrip("/") + "/")
+            sticky = rel_path in extra_files
+            if rel_path in all_files_ignore:
+                logger.info("split_iso: Skipping %s" % rel_path)
+                continue
+            if root.startswith(compose.paths.compose.packages(arch, variant)):
+                packages.append((path, os.path.getsize(path), sticky))
+            else:
+                all_files.append((path, os.path.getsize(path), sticky))
+
+    for path, size, sticky in all_files + packages:
+        ms.add_file(path, size, sticky)
+
+    logger.debug("Splitting media for %s.%s:" % (variant.uid, arch))
+    result = ms.split()
+    if no_split and result[0]["size"] > split_size:
+        logger.warning(
+            "ISO for %s.%s does not fit on single media! It is %s bytes too big. "
+            "(Total size: %s B)"
+            % (variant.uid, arch, result[0]["size"] - split_size, result[0]["size"])
+        )
+    return result
+
+
+def prepare_iso(
+    compose, arch, variant, disc_num=1, disc_count=None, split_iso_data=None
+):
+    tree_dir = compose.paths.compose.os_tree(arch, variant)
+    filename = compose.get_image_name(arch, variant, disc_num=disc_num)
+    iso_dir = compose.paths.work.iso_dir(arch, filename)
+
+    # modify treeinfo
+    ti_path = os.path.join(tree_dir, ".treeinfo")
+    ti = load_and_tweak_treeinfo(ti_path, disc_num, disc_count)
+
+    copy_boot_images(tree_dir, iso_dir)
+
+    if disc_count > 1:
+        # remove repodata/repomd.xml from checksums, create a new one later
+        if "repodata/repomd.xml" in ti.checksums.checksums:
+            del ti.checksums.checksums["repodata/repomd.xml"]
+
+        # rebuild repodata
+        createrepo_c = compose.conf["createrepo_c"]
+        createrepo_checksum = compose.conf["createrepo_checksum"]
+        repo = CreaterepoWrapper(createrepo_c=createrepo_c)
+
+        file_list = "%s-file-list" % iso_dir
+        packages_dir = compose.paths.compose.packages(arch, variant)
+        file_list_content = []
+        for i in split_iso_data["files"]:
+            if not i.endswith(".rpm"):
+                continue
+            if not i.startswith(packages_dir):
+                continue
+            rel_path = relative_path(i, tree_dir.rstrip("/") + "/")
+            file_list_content.append(rel_path)
+
+        if file_list_content:
+            # write modified repodata only if there are packages available
+            run("cp -a %s/repodata %s/" % (shlex_quote(tree_dir), shlex_quote(iso_dir)))
+            with open(file_list, "w") as f:
+                f.write("\n".join(file_list_content))
+            cmd = repo.get_createrepo_cmd(
+                tree_dir,
+                update=True,
+                database=True,
+                skip_stat=True,
+                pkglist=file_list,
+                outputdir=iso_dir,
+                workers=compose.conf["createrepo_num_workers"],
+                checksum=createrepo_checksum,
+            )
+            run(cmd)
+            # add repodata/repomd.xml back to checksums
+            ti.checksums.add(
+                "repodata/repomd.xml", createrepo_checksum, root_dir=iso_dir
+            )
+
+    new_ti_path = os.path.join(iso_dir, ".treeinfo")
+    ti.dump(new_ti_path)
+
+    # modify discinfo
+    di_path = os.path.join(tree_dir, ".discinfo")
+    data = read_discinfo(di_path)
+    data["disc_numbers"] = [disc_num]
+    new_di_path = os.path.join(iso_dir, ".discinfo")
+    write_discinfo(new_di_path, **data)
+
+    if not disc_count or disc_count == 1:
+        data = iso.get_graft_points(compose.paths.compose.topdir(), [tree_dir, iso_dir])
+    else:
+        data = iso.get_graft_points(
+            compose.paths.compose.topdir(),
+            [iso._paths_from_list(tree_dir, split_iso_data["files"]), iso_dir],
+        )
+
+    if compose.conf["createiso_break_hardlinks"]:
+        compose.log_debug(
+            "Breaking hardlinks for ISO %s for %s.%s" % (filename, variant, arch)
+        )
+        break_hardlinks(
+            data, compose.paths.work.iso_staging_dir(arch, variant, filename)
+        )
+        # Create hardlinks for files with duplicate contents.
+        compose.log_debug(
+            "Creating hardlinks for ISO %s for %s.%s" % (filename, variant, arch)
+        )
+        create_hardlinks(
+            compose.paths.work.iso_staging_dir(arch, variant, filename),
+            log_file=compose.paths.log.log_file(
+                arch, "iso-hardlink-%s.log" % variant.uid
+            ),
+        )
+
+    # TODO: /content /graft-points
+    gp = "%s-graft-points" % iso_dir
+    iso.write_graft_points(gp, data, exclude=["*/lost+found", "*/boot.iso"])
+    return gp
+
+
+def load_and_tweak_treeinfo(ti_path, disc_num=1, disc_count=1):
+    """Treeinfo on the media should not contain any reference to boot.iso and
+    it should also have a valid [media] section.
+    """
+    ti = productmd.treeinfo.TreeInfo()
+    ti.load(ti_path)
+    ti.media.totaldiscs = disc_count or 1
+    ti.media.discnum = disc_num
+
+    # remove boot.iso from all sections
+    paths = set()
+    for platform in ti.images.images:
+        if "boot.iso" in ti.images.images[platform]:
+            paths.add(ti.images.images[platform].pop("boot.iso"))
+
+    # remove boot.iso from checksums
+    for i in paths:
+        if i in ti.checksums.checksums.keys():
+            del ti.checksums.checksums[i]
+
+    return ti
+
+
+def copy_boot_images(src, dest):
+    """When mkisofs is called it tries to modify isolinux/isolinux.bin and
+    images/boot.img. Therefore we need to make copies of them.
+    """
+    for i in ("isolinux/isolinux.bin", "images/boot.img"):
+        src_path = os.path.join(src, i)
+        dst_path = os.path.join(dest, i)
+        if os.path.exists(src_path):
+            makedirs(os.path.dirname(dst_path))
+            shutil.copy2(src_path, dst_path)
+
+
+def break_hardlinks(graft_points, staging_dir):
+    """Iterate over graft points and copy any file that has more than 1
+    hardlink into the staging directory. Replace the entry in the dict.
+    """
+    for f in graft_points:
+        info = os.stat(graft_points[f])
+        if stat.S_ISREG(info.st_mode) and info.st_nlink > 1:
+            dest_path = os.path.join(staging_dir, graft_points[f].lstrip("/"))
+            makedirs(os.path.dirname(dest_path))
+            shutil.copy2(graft_points[f], dest_path)
+            graft_points[f] = dest_path
+
+
+def create_hardlinks(staging_dir, log_file):
+    """Create hardlinks within the staging directory.
+    Should happen after break_hardlinks()
+    """
+    cmd = ["/usr/sbin/hardlink", "-c", "-vv", staging_dir]
+    run(cmd, logfile=log_file, show_cmd=True)
+
+
+class OldFileLinker(object):
+    """
+    A wrapper around os.link that remembers which files were linked and can
+    clean them up.
+    """
+
+    def __init__(self, logger):
+        self.logger = logger
+        self.linked_files = []
+
+    def link(self, src, dst):
+        self.logger.debug("Hardlinking %s to %s", src, dst)
+        os.link(src, dst)
+        self.linked_files.append(dst)
+
+    def abort(self):
+        """Clean up all files created by this instance."""
+        for f in self.linked_files:
+            os.unlink(f)
+
+
+def get_iso_level_config(compose, variant, arch):
+    """
+    Get configured ISO level for this variant and architecture.
+    """
+    level = compose.conf.get("iso_level")
+    if isinstance(level, list):
+        level = None
+        for c in get_arch_variant_data(compose.conf, "iso_level", arch, variant):
+            level = c
+    return level

pungi/phases/createrepo.py

@@ -0,0 +1,496 @@
+# -*- coding: utf-8 -*-
+
+
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; version 2 of the License.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Library General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, see <https://gnu.org/licenses/>.
+
+
+__all__ = ("create_variant_repo",)
+
+import copy
+import errno
+import glob
+import os
+import shutil
+import threading
+import xml.dom.minidom
+
+import productmd.modules
+import productmd.rpms
+from kobo.shortcuts import relative_path, run
+from kobo.threads import ThreadPool, WorkerThread
+
+from ..module_util import Modulemd, collect_module_defaults, collect_module_obsoletes
+from ..util import (
+    get_arch_variant_data,
+    read_single_module_stream_from_file,
+    temp_dir,
+)
+from ..wrappers.createrepo import CreaterepoWrapper
+from ..wrappers.scm import get_dir_from_scm
+from .base import PhaseBase
+
+CACHE_TOPDIR = "/var/cache/pungi/createrepo_c/"
+createrepo_lock = threading.Lock()
+createrepo_dirs = set()
+
+
+class CreaterepoPhase(PhaseBase):
+    name = "createrepo"
+
+    def __init__(self, compose, pkgset_phase=None):
+        PhaseBase.__init__(self, compose)
+        self.pool = ThreadPool(logger=self.compose._logger)
+        self.modules_metadata = ModulesMetadata(compose)
+        self.pkgset_phase = pkgset_phase
+
+    def validate(self):
+        errors = []
+
+        if not self.compose.old_composes and self.compose.conf.get("createrepo_deltas"):
+            errors.append("Can not generate deltas without old compose")
+
+        if errors:
+            raise ValueError("\n".join(errors))
+
+    def run(self):
+        get_productids_from_scm(self.compose)
+        reference_pkgset = None
+        if self.pkgset_phase and self.pkgset_phase.package_sets:
+            reference_pkgset = self.pkgset_phase.package_sets[-1]
+        for i in range(self.compose.conf["createrepo_num_threads"]):
+            self.pool.add(
+                CreaterepoThread(self.pool, reference_pkgset, self.modules_metadata)
+            )
+
+        for variant in self.compose.get_variants():
+            if variant.is_empty:
+                continue
+
+            if variant.uid in self.compose.conf.get("createrepo_extra_modulemd", {}):
+                # Clone extra modulemd repository if it's configured.
+                get_dir_from_scm(
+                    self.compose.conf["createrepo_extra_modulemd"][variant.uid],
+                    self.compose.paths.work.tmp_dir(variant=variant, create_dir=False),
+                    compose=self.compose,
+                )
+
+            self.pool.queue_put((self.compose, None, variant, "srpm"))
+            for arch in variant.arches:
+                self.pool.queue_put((self.compose, arch, variant, "rpm"))
+                self.pool.queue_put((self.compose, arch, variant, "debuginfo"))
+
+        self.pool.start()
+
+    def stop(self):
+        super(CreaterepoPhase, self).stop()
+        self.modules_metadata.write_modules_metadata()
+
+
+def create_variant_repo(
+    compose, arch, variant, pkg_type, pkgset, modules_metadata=None
+):
+    types = {
+        "rpm": (
+            "binary",
+            lambda **kwargs: compose.paths.compose.repository(
+                arch=arch, variant=variant, **kwargs
+            ),
+        ),
+        "srpm": (
+            "source",
+            lambda **kwargs: compose.paths.compose.repository(
+                arch="src", variant=variant, **kwargs
+            ),
+        ),
+        "debuginfo": (
+            "debug",
+            lambda **kwargs: compose.paths.compose.debug_repository(
+                arch=arch, variant=variant, **kwargs
+            ),
+        ),
+    }
+
+    if variant.is_empty or (arch is None and pkg_type != "srpm"):
+        compose.log_info(
+            "[SKIP ] Creating repo (arch: %s, variant: %s)" % (arch, variant)
+        )
+        return
+
+    createrepo_c = compose.conf["createrepo_c"]
+    createrepo_checksum = compose.conf["createrepo_checksum"]
+    repo = CreaterepoWrapper(createrepo_c=createrepo_c)
+    repo_dir_arch = None
+    if pkgset:
+        repo_dir_arch = pkgset.paths["global" if pkg_type == "srpm" else arch]
+
+    try:
+        repo_dir = types[pkg_type][1]()
+    except KeyError:
+        raise ValueError("Unknown package type: %s" % pkg_type)
+
+    msg = "Creating repo (arch: %s, variant: %s): %s" % (arch, variant, repo_dir)
+
+    # HACK: using global lock
+    # This is important when addons put packages into parent variant directory.
+    # There can't be multiple createrepo processes operating on the same
+    # directory.
+    with createrepo_lock:
+        if repo_dir in createrepo_dirs:
+            compose.log_warning("[SKIP ] Already in progress: %s" % msg)
+            return
+        createrepo_dirs.add(repo_dir)
+
+    compose.log_info("[BEGIN] %s" % msg)
+
+    # We only want delta RPMs for binary repos.
+    with_deltas = pkg_type == "rpm" and _has_deltas(compose, variant, arch)
+
+    rpms = set()
+    rpm_nevras = set()
+
+    # read rpms from metadata rather than guessing it by scanning filesystem
+    manifest_file = compose.paths.compose.metadata("rpms.json")
+    manifest = productmd.rpms.Rpms()
+    manifest.load(manifest_file)
+
+    for rpms_arch, data in manifest.rpms.get(variant.uid, {}).items():
+        if arch is not None and arch != rpms_arch:
+            continue
+        for srpm_data in data.values():
+            for rpm_nevra, rpm_data in srpm_data.items():
+                if types[pkg_type][0] != rpm_data["category"]:
+                    continue
+                path = os.path.join(compose.topdir, "compose", rpm_data["path"])
+                rel_path = relative_path(path, repo_dir.rstrip("/") + "/")
+                rpms.add(rel_path)
+                rpm_nevras.add(str(rpm_nevra))
+
+    file_list = compose.paths.work.repo_package_list(arch, variant, pkg_type)
+    with open(file_list, "w") as f:
+        for rel_path in sorted(rpms):
+            f.write("%s\n" % rel_path)
+
+    # Only find last compose when we actually want delta RPMs.
+    old_package_dirs = _get_old_package_dirs(compose, repo_dir) if with_deltas else None
+    if old_package_dirs:
+        # If we are creating deltas, we can not reuse existing metadata, as
+        # that would stop deltas from being created.
+        # This seems to only affect createrepo_c though.
+        repo_dir_arch = None
+
+    comps_path = None
+    if compose.has_comps and pkg_type == "rpm":
+        comps_path = compose.paths.work.comps(arch=arch, variant=variant)
+
+    if compose.conf["createrepo_enable_cache"]:
+        cachedir = os.path.join(
+            CACHE_TOPDIR,
+            "%s-%s" % (compose.conf["release_short"], os.getuid()),
+        )
+        if not os.path.exists(cachedir):
+            try:
+                os.makedirs(cachedir)
+            except Exception as e:
+                compose.log_warning(
+                    "Cache disabled because cannot create cache dir %s %s"
+                    % (cachedir, str(e))
+                )
+                cachedir = None
+    else:
+        cachedir = None
+    cmd = repo.get_createrepo_cmd(
+        repo_dir,
+        update=True,
+        database=compose.should_create_yum_database,
+        skip_stat=True,
+        pkglist=file_list,
+        outputdir=repo_dir,
+        workers=compose.conf["createrepo_num_workers"],
+        groupfile=comps_path,
+        update_md_path=repo_dir_arch,
+        checksum=createrepo_checksum,
+        deltas=with_deltas,
+        oldpackagedirs=old_package_dirs,
+        use_xz=compose.conf["createrepo_use_xz"],
+        extra_args=compose.conf["createrepo_extra_args"],
+        cachedir=cachedir,
+    )
+    log_file = compose.paths.log.log_file(
+        arch, "createrepo-%s.%s" % (variant, pkg_type)
+    )
+    run(cmd, logfile=log_file, show_cmd=True)
+
+    # call modifyrepo to inject productid
+    product_id = compose.conf.get("product_id")
+    if product_id and pkg_type == "rpm":
+        # add product certificate to base (rpm) repo; skip source and debug
+        product_id_path = compose.paths.work.product_id(arch, variant)
+        if os.path.isfile(product_id_path):
+            cmd = repo.get_modifyrepo_cmd(
+                os.path.join(repo_dir, "repodata"), product_id_path, compress_type="gz"
+            )
+            log_file = compose.paths.log.log_file(arch, "modifyrepo-%s" % variant)
+            run(cmd, logfile=log_file, show_cmd=True)
+            # productinfo is not supported by modifyrepo in any way
+            # this is a HACK to make CDN happy (dmach: at least I think,
+            # need to confirm with dgregor)
+            shutil.copy2(
+                product_id_path, os.path.join(repo_dir, "repodata", "productid")
+            )
+
+    # call modifyrepo to inject modulemd if needed
+    if pkg_type == "rpm" and arch in variant.arch_mmds and Modulemd is not None:
+        mod_index = Modulemd.ModuleIndex()
+        metadata = []
+
+        for module_id, mmd in variant.arch_mmds.get(arch, {}).items():
+            if modules_metadata:
+                module_rpms = mmd.get_rpm_artifacts()
+                metadata.append((module_id, module_rpms))
+            mod_index.add_module_stream(mmd)
+
+        module_names = set(mod_index.get_module_names())
+        defaults_dir = compose.paths.work.module_defaults_dir()
+        overrides_dir = compose.conf.get("module_defaults_override_dir")
+        collect_module_defaults(
+            defaults_dir, module_names, mod_index, overrides_dir=overrides_dir
+        )
+
+        obsoletes_dir = compose.paths.work.module_obsoletes_dir()
+        mod_index = collect_module_obsoletes(obsoletes_dir, module_names, mod_index)
+
+        # Add extra modulemd files
+        if variant.uid in compose.conf.get("createrepo_extra_modulemd", {}):
+            compose.log_debug("Adding extra modulemd for %s.%s", variant.uid, arch)
+            dirname = compose.paths.work.tmp_dir(variant=variant, create_dir=False)
+            for filepath in glob.glob(os.path.join(dirname, arch) + "/*.yaml"):
+                module_stream = read_single_module_stream_from_file(filepath)
+                if not mod_index.add_module_stream(module_stream):
+                    raise RuntimeError(
+                        "Failed parsing modulemd data from %s" % filepath
+                    )
+                # Add the module to metadata with dummy tag. We can't leave the
+                # value empty, but we don't know what the correct tag is.
+                nsvc = module_stream.get_nsvc()
+                variant.module_uid_to_koji_tag[nsvc] = "DUMMY"
+                metadata.append((nsvc, []))
+
+        log_file = compose.paths.log.log_file(arch, "modifyrepo-modules-%s" % variant)
+        add_modular_metadata(repo, repo_dir, mod_index, log_file)
+
+        for module_id, module_rpms in metadata:
+            modulemd_path = os.path.join(
+                types[pkg_type][1](relative=True),
+                find_file_in_repodata(repo_dir, "modules"),
+            )
+            modules_metadata.prepare_module_metadata(
+                variant,
+                arch,
+                module_id,
+                modulemd_path,
+                types[pkg_type][0],
+                list(module_rpms),
+            )
+
+    compose.log_info("[DONE ] %s" % msg)
+
+
+def add_modular_metadata(repo, repo_path, mod_index, log_file):
+    """Add modular metadata into a repository."""
+    # Dumping empty index fails, we need to check for that.
+    if not mod_index.get_module_names():
+        return
+
+    with temp_dir() as tmp_dir:
+        modules_path = os.path.join(tmp_dir, "modules.yaml")
+        with open(modules_path, "w") as f:
+            f.write(mod_index.dump_to_string())
+
+        cmd = repo.get_modifyrepo_cmd(
+            os.path.join(repo_path, "repodata"),
+            modules_path,
+            mdtype="modules",
+            compress_type="gz",
+        )
+        run(cmd, logfile=log_file, show_cmd=True)
+
+
+def find_file_in_repodata(repo_path, type_):
+    dom = xml.dom.minidom.parse(os.path.join(repo_path, "repodata", "repomd.xml"))
+    for entry in dom.getElementsByTagName("data"):
+        if entry.getAttribute("type") == type_:
+            return entry.getElementsByTagName("location")[0].getAttribute("href")
+        entry.unlink()
+    raise RuntimeError("No such file in repodata: %s" % type_)
+
+
+class CreaterepoThread(WorkerThread):
+    def __init__(self, pool, reference_pkgset, modules_metadata):
+        super(CreaterepoThread, self).__init__(pool)
+        self.reference_pkgset = reference_pkgset
+        self.modules_metadata = modules_metadata
+
+    def process(self, item, num):
+        compose, arch, variant, pkg_type = item
+        create_variant_repo(
+            compose,
+            arch,
+            variant,
+            pkg_type=pkg_type,
+            pkgset=self.reference_pkgset,
+            modules_metadata=self.modules_metadata,
+        )
+
+
+def get_productids_from_scm(compose):
+    # product_id is a scm_dict: {scm, repo, branch, dir}
+    # expected file name format: $variant_uid-$arch-*.pem
+    product_id = compose.conf.get("product_id")
+    if not product_id:
+        compose.log_info("No product certificates specified")
+        return
+
+    product_id_allow_missing = compose.conf["product_id_allow_missing"]
+
+    msg = "Getting product certificates from SCM..."
+    compose.log_info("[BEGIN] %s" % msg)
+
+    tmp_dir = compose.mkdtemp(prefix="pungi_")
+    try:
+        get_dir_from_scm(product_id, tmp_dir, compose=compose)
+    except OSError as e:
+        if e.errno == errno.ENOENT and product_id_allow_missing:
+            compose.log_warning("No product IDs in %s" % product_id)
+            return
+        raise
+
+    if compose.conf["product_id_allow_name_prefix"]:
+        pattern = "%s/*%s-%s-*.pem"
+    else:
+        pattern = "%s/%s-%s-*.pem"
+
+    for arch in compose.get_arches():
+        for variant in compose.get_variants(arch=arch):
+            # some layered products may use base product name before variant
+            pem_files = glob.glob(pattern % (tmp_dir, variant.uid, arch))
+            if not pem_files:
+                warning = "No product certificate found (arch: %s, variant: %s)" % (
+                    arch,
+                    variant.uid,
+                )
+                if product_id_allow_missing:
+                    compose.log_warning(warning)
+                    continue
+                else:
+                    shutil.rmtree(tmp_dir)
+                    raise RuntimeError(warning)
+            if len(pem_files) > 1:
+                shutil.rmtree(tmp_dir)
+                raise RuntimeError(
+                    "Multiple product certificates found (arch: %s, variant: %s): %s"
+                    % (
+                        arch,
+                        variant.uid,
+                        ", ".join(sorted([os.path.basename(i) for i in pem_files])),
+                    )
+                )
+            product_id_path = compose.paths.work.product_id(arch, variant)
+            shutil.copy2(pem_files[0], product_id_path)
+
+    try:
+        shutil.rmtree(tmp_dir)
+    except Exception as e:
+        compose.log_warning("Failed to clean up tmp dir: %s %s" % (tmp_dir, str(e)))
+    compose.log_info("[DONE ] %s" % msg)
+
+
+def _get_old_package_dirs(compose, repo_dir):
+    """Given a compose and a path to a repo in it, try to find corresponding
+    repo in an older compose and return a list of paths to directories with
+    packages in it.
+    """
+    if not compose.conf["createrepo_deltas"]:
+        return None
+    old_package_dirs = compose.paths.old_compose_path(
+        repo_dir, allowed_statuses=["FINISHED", "FINISHED_INCOMPLETE"]
+    )
+    if not old_package_dirs:
+        compose.log_info("No suitable old compose found in: %s" % compose.old_composes)
+        return None
+    old_package_dirs = os.path.join(old_package_dirs, "Packages")
+    if compose.conf["hashed_directories"]:
+        old_package_dirs = _find_package_dirs(old_package_dirs)
+    return old_package_dirs
+
+
+def _find_package_dirs(base):
+    """Assuming the packages are in directories hashed by first letter, find
+    all the buckets in given base.
+    """
+    buckets = set()
+    try:
+        for subdir in os.listdir(base):
+            bucket = os.path.join(base, subdir)
+            if os.path.isdir(bucket):
+                buckets.add(bucket)
+    except OSError:
+        # The directory does not exist, so no drpms for you!
+        pass
+    return sorted(buckets)
+
+
+def _has_deltas(compose, variant, arch):
+    """Check if delta RPMs are enabled for given variant and architecture."""
+    key = "createrepo_deltas"
+    if isinstance(compose.conf.get(key), bool):
+        return compose.conf[key]
+    return any(get_arch_variant_data(compose.conf, key, arch, variant))
+
+
+class ModulesMetadata(object):
+    def __init__(self, compose):
+        # Prepare empty module metadata
+        self.compose = compose
+        self.modules_metadata_file = self.compose.paths.compose.metadata("modules.json")
+        self.productmd_modules_metadata = productmd.modules.Modules()
+        self.productmd_modules_metadata.compose.id = copy.copy(self.compose.compose_id)
+        self.productmd_modules_metadata.compose.type = copy.copy(
+            self.compose.compose_type
+        )
+        self.productmd_modules_metadata.compose.date = copy.copy(
+            self.compose.compose_date
+        )
+        self.productmd_modules_metadata.compose.respin = copy.copy(
+            self.compose.compose_respin
+        )
+
+    def write_modules_metadata(self):
+        """
+        flush modules metadata into file
+        """
+        self.compose.log_info(
+            "Writing modules metadata: %s" % self.modules_metadata_file
+        )
+        self.productmd_modules_metadata.dump(self.modules_metadata_file)
+
+    def prepare_module_metadata(
+        self, variant, arch, nsvc, modulemd_path, category, module_rpms
+    ):
+        """
+        Find koji tag which corresponds to the module and add record into
+        module metadata structure.
+        """
+        koji_tag = variant.module_uid_to_koji_tag[nsvc]
+        self.productmd_modules_metadata.add(
+            variant.uid, arch, nsvc, koji_tag, modulemd_path, category, module_rpms
+        )

pungi/phases/extra_files.py

@@ -0,0 +1,141 @@
+# -*- coding: utf-8 -*-
+
+
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; version 2 of the License.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Library General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, see <https://gnu.org/licenses/>.
+
+
+import os
+import copy
+import fnmatch
+
+from productmd.extra_files import ExtraFiles
+
+from pungi.util import get_arch_variant_data, pkg_is_rpm, copy_all
+from pungi.arch import split_name_arch
+from pungi.wrappers.scm import get_file_from_scm, get_dir_from_scm
+from pungi.phases.base import ConfigGuardedPhase
+from pungi import metadata
+
+
+class ExtraFilesPhase(ConfigGuardedPhase):
+    """EXTRA_FILES"""
+
+    name = "extra_files"
+
+    def __init__(self, compose, pkgset_phase):
+        super(ExtraFilesPhase, self).__init__(compose)
+        # pkgset_phase provides package_sets
+        self.pkgset_phase = pkgset_phase
+        # Prepare metadata
+        self.metadata = ExtraFiles()
+        self.metadata.compose.id = self.compose.compose_id
+        self.metadata.compose.type = self.compose.compose_type
+        self.metadata.compose.date = self.compose.compose_date
+        self.metadata.compose.respin = self.compose.compose_respin
+
+    def run(self):
+        for variant in self.compose.get_variants():
+            if variant.is_empty:
+                continue
+            for arch in variant.arches + ["src"]:
+                cfg = get_arch_variant_data(self.compose.conf, self.name, arch, variant)
+                if cfg:
+                    copy_extra_files(
+                        self.compose,
+                        cfg,
+                        arch,
+                        variant,
+                        self.pkgset_phase.package_sets,
+                        self.metadata,
+                    )
+                else:
+                    self.compose.log_info(
+                        "[SKIP ] No extra files (arch: %s, variant: %s)"
+                        % (arch, variant.uid)
+                    )
+
+        metadata_path = self.compose.paths.compose.metadata("extra_files.json")
+        self.compose.log_info("Writing global extra files metadata: %s" % metadata_path)
+        self.metadata.dump(metadata_path)
+
+
+def copy_extra_files(
+    compose, cfg, arch, variant, package_sets, extra_metadata, checksum_type=None
+):
+    checksum_type = checksum_type or compose.conf["media_checksums"]
+    var_dict = {
+        "arch": arch,
+        "variant_id": variant.id,
+        "variant_id_lower": variant.id.lower(),
+        "variant_uid": variant.uid,
+        "variant_uid_lower": variant.uid.lower(),
+    }
+
+    msg = "Getting extra files (arch: %s, variant: %s)" % (arch, variant)
+    compose.log_info("[BEGIN] %s" % msg)
+
+    os_tree = compose.paths.compose.os_tree(arch, variant)
+    extra_files_dir = compose.paths.work.extra_files_dir(arch, variant)
+
+    for scm_dict in cfg:
+        scm_dict = copy.deepcopy(scm_dict)
+        # if scm is "rpm" and repo contains only a package name, find the
+        # package(s) in package set
+        if scm_dict["scm"] == "rpm" and not _is_external(scm_dict["repo"]):
+            rpms = []
+            pattern = scm_dict["repo"] % var_dict
+            pkg_name, pkg_arch = split_name_arch(pattern)
+            for package_set in package_sets:
+                for pkgset_file in package_set[arch]:
+                    pkg_obj = package_set[arch][pkgset_file]
+                    if pkg_is_rpm(pkg_obj) and _pkg_matches(
+                        pkg_obj, pkg_name, pkg_arch
+                    ):
+                        rpms.append(pkg_obj.file_path)
+            if not rpms:
+                raise RuntimeError(
+                    "No package matching %s in the package set." % pattern
+                )
+            scm_dict["repo"] = rpms
+
+        getter = get_file_from_scm if "file" in scm_dict else get_dir_from_scm
+        target_path = os.path.join(
+            extra_files_dir, scm_dict.get("target", "").lstrip("/")
+        )
+        getter(scm_dict, target_path, compose=compose)
+
+    if os.listdir(extra_files_dir):
+        metadata.populate_extra_files_metadata(
+            extra_metadata,
+            variant,
+            arch,
+            os_tree,
+            copy_all(extra_files_dir, os_tree),
+            compose.conf["media_checksums"],
+            relative_root=compose.paths.compose.topdir(),
+        )
+
+    compose.log_info("[DONE ] %s" % msg)
+
+
+def _pkg_matches(pkg_obj, name_glob, arch):
+    """Check if `pkg_obj` matches name and arch."""
+    return fnmatch.fnmatch(pkg_obj.name, name_glob) and (
+        arch is None or arch == pkg_obj.arch
+    )
+
+
+def _is_external(rpm):
+    """Check if path to rpm points outside of the compose: i.e. it is an
+    absolute path or a URL."""
+    return rpm.startswith("/") or "://" in rpm

pungi/phases/extra_isos.py

@@ -0,0 +1,554 @@
+# -*- coding: utf-8 -*-
+
+
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; version 2 of the License.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Library General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, see <https://gnu.org/licenses/>.
+
+import os
+import hashlib
+import json
+
+from kobo.shortcuts import force_list
+from kobo.threads import ThreadPool, WorkerThread
+import productmd.treeinfo
+from productmd.extra_files import ExtraFiles
+
+from pungi import createiso
+from pungi import metadata
+from pungi.phases.base import ConfigGuardedPhase, PhaseBase, PhaseLoggerMixin
+from pungi.phases.createiso import (
+    add_iso_to_metadata,
+    copy_boot_images,
+    run_createiso_command,
+    load_and_tweak_treeinfo,
+    compare_packages,
+    OldFileLinker,
+    get_iso_level_config,
+)
+from pungi.util import (
+    failable,
+    get_format_substs,
+    get_variant_data,
+    get_volid,
+    read_json_file,
+)
+from pungi.wrappers import iso
+from pungi.wrappers.scm import get_dir_from_scm, get_file_from_scm
+
+
+class ExtraIsosPhase(PhaseLoggerMixin, ConfigGuardedPhase, PhaseBase):
+    name = "extra_isos"
+
+    def __init__(self, compose, buildinstall_phase):
+        super(ExtraIsosPhase, self).__init__(compose)
+        self.pool = ThreadPool(logger=self.logger)
+        self.bi = buildinstall_phase
+
+    def validate(self):
+        for variant in self.compose.get_variants(types=["variant"]):
+            for config in get_variant_data(self.compose.conf, self.name, variant):
+                extra_arches = set(config.get("arches", [])) - set(variant.arches)
+                if extra_arches:
+                    self.compose.log_warning(
+                        "Extra iso config for %s mentions non-existing arches: %s"
+                        % (variant, ", ".join(sorted(extra_arches)))
+                    )
+
+    def run(self):
+        commands = []
+
+        for variant in self.compose.get_variants(types=["variant"]):
+            for config in get_variant_data(self.compose.conf, self.name, variant):
+                arches = set(variant.arches)
+                if config.get("arches"):
+                    arches &= set(config["arches"])
+                if not config["skip_src"]:
+                    arches.add("src")
+                for arch in sorted(arches):
+                    commands.append((config, variant, arch))
+
+        for config, variant, arch in commands:
+            self.pool.add(ExtraIsosThread(self.pool, self.bi))
+            self.pool.queue_put((self.compose, config, variant, arch))
+
+        self.pool.start()
+
+
+class ExtraIsosThread(WorkerThread):
+    def __init__(self, pool, buildinstall_phase):
+        super(ExtraIsosThread, self).__init__(pool)
+        self.bi = buildinstall_phase
+
+    def process(self, item, num):
+        self.num = num
+        compose, config, variant, arch = item
+        can_fail = arch in config.get("failable_arches", [])
+        with failable(
+            compose, can_fail, variant, arch, "extra_iso", logger=self.pool._logger
+        ):
+            self.worker(compose, config, variant, arch)
+
+    def worker(self, compose, config, variant, arch):
+        filename = get_filename(compose, variant, arch, config.get("filename"))
+        volid = get_volume_id(compose, variant, arch, config.get("volid", []))
+        iso_dir = compose.paths.compose.iso_dir(arch, variant)
+        iso_path = os.path.join(iso_dir, filename)
+
+        prepare_media_metadata(compose, variant, arch)
+
+        msg = "Creating ISO (arch: %s, variant: %s): %s" % (arch, variant, filename)
+        self.pool.log_info("[BEGIN] %s" % msg)
+
+        get_extra_files(compose, variant, arch, config.get("extra_files", []))
+
+        bootable = arch != "src" and bool(compose.conf.get("buildinstall_method"))
+
+        graft_points = get_iso_contents(
+            compose,
+            variant,
+            arch,
+            config["include_variants"],
+            filename,
+            bootable=bootable,
+            inherit_extra_files=config.get("inherit_extra_files", False),
+        )
+
+        opts = createiso.CreateIsoOpts(
+            output_dir=iso_dir,
+            iso_name=filename,
+            volid=volid,
+            graft_points=graft_points,
+            arch=arch,
+            supported=compose.supported,
+            hfs_compat=compose.conf["iso_hfs_ppc64le_compatible"],
+            use_xorrisofs=compose.conf.get("createiso_use_xorrisofs"),
+            iso_level=get_iso_level_config(compose, variant, arch),
+        )
+        os_tree = compose.paths.compose.os_tree(arch, variant)
+        if compose.conf["create_jigdo"]:
+            jigdo_dir = compose.paths.compose.jigdo_dir(arch, variant)
+            opts = opts._replace(jigdo_dir=jigdo_dir, os_tree=os_tree)
+
+        if bootable:
+            opts = opts._replace(
+                buildinstall_method=compose.conf["buildinstall_method"],
+                boot_iso=os.path.join(os_tree, "images", "boot.iso"),
+            )
+
+        # Check if it can be reused.
+        hash = hashlib.sha256()
+        hash.update(json.dumps(config, sort_keys=True).encode("utf-8"))
+        config_hash = hash.hexdigest()
+
+        if not self.try_reuse(compose, variant, arch, config_hash, opts):
+            script_dir = compose.paths.work.tmp_dir(arch, variant)
+            opts = opts._replace(script_dir=script_dir)
+            script_file = os.path.join(script_dir, "extraiso-%s.sh" % filename)
+            with open(script_file, "w") as f:
+                createiso.write_script(opts, f)
+
+            run_createiso_command(
+                self.num,
+                compose,
+                bootable,
+                arch,
+                ["bash", script_file],
+                [compose.topdir],
+                log_file=compose.paths.log.log_file(
+                    arch, "extraiso-%s" % os.path.basename(iso_path)
+                ),
+                iso_path=iso_path,
+            )
+
+        img = add_iso_to_metadata(
+            compose,
+            variant,
+            arch,
+            iso_path,
+            bootable,
+            additional_variants=config["include_variants"],
+        )
+        img._max_size = config.get("max_size")
+
+        save_reuse_metadata(compose, variant, arch, config_hash, opts, iso_path)
+
+        self.pool.log_info("[DONE ] %s" % msg)
+
+    def try_reuse(self, compose, variant, arch, config_hash, opts):
+        # Check explicit config
+        if not compose.conf["extraiso_allow_reuse"]:
+            return
+
+        log_msg = "Cannot reuse ISO for %s.%s" % (variant, arch)
+
+        if opts.buildinstall_method and not self.bi.reused(variant, arch):
+            # If buildinstall phase was not reused for some reason, we can not
+            # reuse any bootable image. If a package change caused rebuild of
+            # boot.iso, we would catch it here too, but there could be a
+            # configuration change in lorax template which would remain
+            # undetected.
+            self.pool.log_info("%s - boot configuration changed", log_msg)
+            return False
+
+        # Check old compose configuration: extra_files and product_ids can be
+        # reflected on ISO.
+        old_config = compose.load_old_compose_config()
+        if not old_config:
+            self.pool.log_info("%s - no config for old compose", log_msg)
+            return False
+
+        # Disable reuse if unsigned packages are allowed. The older compose
+        # could have unsigned packages, and those may have been signed since
+        # then. We want to regenerate the ISO to have signatures.
+        if None in compose.conf["sigkeys"]:
+            self.pool.log_info("%s - unsigned packages are allowed", log_msg)
+            return False
+
+        # Convert current configuration to JSON and back to encode it similarly
+        # to the old one
+        config = json.loads(json.dumps(compose.conf))
+        for opt in compose.conf:
+            # Skip a selection of options: these affect what packages can be
+            # included, which we explicitly check later on.
+            config_whitelist = set(
+                [
+                    "gather_lookaside_repos",
+                    "pkgset_koji_builds",
+                    "pkgset_koji_scratch_tasks",
+                    "pkgset_koji_module_builds",
+                ]
+            )
+            # Skip irrelevant options
+            config_whitelist.update(["osbs", "osbuild"])
+            if opt in config_whitelist:
+                continue
+
+            if old_config.get(opt) != config.get(opt):
+                self.pool.log_info("%s - option %s differs", log_msg, opt)
+                return False
+
+        old_metadata = load_old_metadata(compose, variant, arch, config_hash)
+        if not old_metadata:
+            self.pool.log_info("%s - no old metadata found", log_msg)
+            return False
+
+        # Test if volume ID matches - volid can be generated dynamically based on
+        # other values, and could change even if nothing else is different.
+        if opts.volid != old_metadata["opts"]["volid"]:
+            self.pool.log_info("%s - volume ID differs", log_msg)
+            return False
+
+        # Compare packages on the ISO.
+        if compare_packages(
+            old_metadata["opts"]["graft_points"],
+            opts.graft_points,
+        ):
+            self.pool.log_info("%s - packages differ", log_msg)
+            return False
+
+        try:
+            self.perform_reuse(
+                compose,
+                variant,
+                arch,
+                opts,
+                old_metadata["opts"]["output_dir"],
+                old_metadata["opts"]["iso_name"],
+            )
+            return True
+        except Exception as exc:
+            self.pool.log_error(
+                "Error while reusing ISO for %s.%s: %s", variant, arch, exc
+            )
+            compose.traceback("extraiso-reuse-%s-%s-%s" % (variant, arch, config_hash))
+            return False
+
+    def perform_reuse(self, compose, variant, arch, opts, old_iso_dir, old_file_name):
+        """
+        Copy all related files from old compose to the new one. As a last step
+        add the new image to metadata.
+        """
+        linker = OldFileLinker(self.pool._logger)
+        old_iso_path = os.path.join(old_iso_dir, old_file_name)
+        iso_path = os.path.join(opts.output_dir, opts.iso_name)
+        try:
+            # Hardlink ISO and manifest
+            for suffix in ("", ".manifest"):
+                linker.link(old_iso_path + suffix, iso_path + suffix)
+            # Copy log files
+            # The log file name includes filename of the image, so we need to
+            # find old file with the old name, and rename it to the new name.
+            log_file = compose.paths.log.log_file(arch, "extraiso-%s" % opts.iso_name)
+            old_log_file = compose.paths.old_compose_path(
+                compose.paths.log.log_file(arch, "extraiso-%s" % old_file_name)
+            )
+            linker.link(old_log_file, log_file)
+            # Copy jigdo files
+            if opts.jigdo_dir:
+                old_jigdo_dir = compose.paths.old_compose_path(opts.jigdo_dir)
+                for suffix in (".template", ".jigdo"):
+                    linker.link(
+                        os.path.join(old_jigdo_dir, old_file_name) + suffix,
+                        os.path.join(opts.jigdo_dir, opts.iso_name) + suffix,
+                    )
+        except Exception:
+            # A problem happened while linking some file, let's clean up
+            # everything.
+            linker.abort()
+            raise
+
+
+def save_reuse_metadata(compose, variant, arch, config_hash, opts, iso_path):
+    """
+    Save metadata for possible reuse of this image. The file name is determined
+    from the hash of a configuration snippet for this image. Any change in that
+    configuration in next compose will change the hash and thus reuse will be
+    blocked.
+    """
+    metadata = {"opts": opts._asdict()}
+    metadata_path = compose.paths.log.log_file(
+        arch,
+        "extraiso-reuse-%s-%s-%s" % (variant.uid, arch, config_hash),
+        ext="json",
+    )
+    with open(metadata_path, "w") as f:
+        json.dump(metadata, f, indent=2)
+
+
+def load_old_metadata(compose, variant, arch, config_hash):
+    metadata_path = compose.paths.log.log_file(
+        arch,
+        "extraiso-reuse-%s-%s-%s" % (variant.uid, arch, config_hash),
+        ext="json",
+    )
+    old_path = compose.paths.old_compose_path(metadata_path)
+    try:
+        return read_json_file(old_path)
+    except Exception:
+        return None
+
+
+def get_extra_files(compose, variant, arch, extra_files):
+    """Clone the configured files into a directory from where they can be
+    included in the ISO.
+    """
+    extra_files_dir = compose.paths.work.extra_iso_extra_files_dir(arch, variant)
+    filelist = []
+    for scm_dict in extra_files:
+        getter = get_file_from_scm if "file" in scm_dict else get_dir_from_scm
+        target = scm_dict.get("target", "").lstrip("/")
+        target_path = os.path.join(extra_files_dir, target).rstrip("/")
+        filelist.extend(
+            os.path.join(target, f)
+            for f in getter(scm_dict, target_path, compose=compose)
+        )
+
+    if filelist:
+        metadata.populate_extra_files_metadata(
+            ExtraFiles(),
+            variant,
+            arch,
+            extra_files_dir,
+            filelist,
+            compose.conf["media_checksums"],
+        )
+
+
+def get_iso_contents(
+    compose, variant, arch, include_variants, filename, bootable, inherit_extra_files
+):
+    """Find all files that should be on the ISO. For bootable image we start
+    with the boot configuration. Then for each variant we add packages,
+    repodata and extra files. Finally we add top-level extra files.
+    """
+    iso_dir = compose.paths.work.iso_dir(arch, filename)
+
+    files = {}
+    if bootable:
+        buildinstall_dir = compose.paths.work.buildinstall_dir(arch, create_dir=False)
+        if compose.conf["buildinstall_method"] == "lorax":
+            buildinstall_dir = os.path.join(buildinstall_dir, variant.uid)
+
+        copy_boot_images(buildinstall_dir, iso_dir)
+        files = iso.get_graft_points(
+            compose.paths.compose.topdir(), [buildinstall_dir, iso_dir]
+        )
+
+        # We need to point efiboot.img to compose/ tree, because it was
+        # modified in buildinstall phase and the file in work/ has different
+        # checksum to what is in the .treeinfo.
+        if "images/efiboot.img" in files:
+            files["images/efiboot.img"] = os.path.join(
+                compose.paths.compose.os_tree(arch, variant), "images/efiboot.img"
+            )
+
+    variants = [variant.uid] + include_variants
+    for variant_uid in variants:
+        var = compose.all_variants[variant_uid]
+
+        # Get packages...
+        package_dir = compose.paths.compose.packages(arch, var)
+        for k, v in iso.get_graft_points(
+            compose.paths.compose.topdir(), [package_dir]
+        ).items():
+            files[os.path.join(var.uid, "Packages", k)] = v
+
+        # Get repodata...
+        tree_dir = compose.paths.compose.repository(arch, var)
+        repo_dir = os.path.join(tree_dir, "repodata")
+        for k, v in iso.get_graft_points(
+            compose.paths.compose.topdir(), [repo_dir]
+        ).items():
+            files[os.path.join(var.uid, "repodata", k)] = v
+
+        if inherit_extra_files:
+            # Get extra files...
+            extra_files_dir = compose.paths.work.extra_files_dir(arch, var)
+            for k, v in iso.get_graft_points(
+                compose.paths.compose.topdir(), [extra_files_dir]
+            ).items():
+                files[os.path.join(var.uid, k)] = v
+
+    extra_files_dir = compose.paths.work.extra_iso_extra_files_dir(arch, variant)
+
+    original_treeinfo = os.path.join(
+        compose.paths.compose.os_tree(arch=arch, variant=variant), ".treeinfo"
+    )
+    tweak_treeinfo(
+        compose,
+        include_variants,
+        original_treeinfo,
+        os.path.join(extra_files_dir, ".treeinfo"),
+    )
+    tweak_repo_treeinfo(
+        compose,
+        include_variants,
+        original_treeinfo,
+        original_treeinfo,
+    )
+
+    # Add extra files specific for the ISO
+    files.update(
+        iso.get_graft_points(compose.paths.compose.topdir(), [extra_files_dir])
+    )
+
+    gp = "%s-graft-points" % iso_dir
+    iso.write_graft_points(gp, files, exclude=["*/lost+found", "*/boot.iso"])
+    return gp
+
+
+def tweak_repo_treeinfo(compose, include_variants, source_file, dest_file):
+    """
+    The method includes the variants to file .treeinfo of a variant. It takes
+    the variants which are described
+    by options `extra_isos -> include_variants`.
+    """
+    ti = productmd.treeinfo.TreeInfo()
+    ti.load(source_file)
+    main_variant = next(iter(ti.variants))
+    for variant_uid in include_variants:
+        variant = compose.all_variants[variant_uid]
+        var = productmd.treeinfo.Variant(ti)
+        var.id = variant.id
+        var.uid = variant.uid
+        var.name = variant.name
+        var.type = variant.type
+        ti.variants.add(var)
+
+    for variant_id in ti.variants:
+        var = ti.variants[variant_id]
+        if variant_id == main_variant:
+            var.paths.packages = 'Packages'
+            var.paths.repository = '.'
+        else:
+            var.paths.packages = os.path.join(
+                '../../..',
+                var.uid,
+                var.arch,
+                'os/Packages',
+            )
+            var.paths.repository = os.path.join(
+                '../../..',
+                var.uid,
+                var.arch,
+                'os',
+            )
+    ti.dump(dest_file, main_variant=main_variant)
+
+
+def tweak_treeinfo(compose, include_variants, source_file, dest_file):
+    ti = load_and_tweak_treeinfo(source_file)
+    for variant_uid in include_variants:
+        variant = compose.all_variants[variant_uid]
+        var = productmd.treeinfo.Variant(ti)
+        var.id = variant.id
+        var.uid = variant.uid
+        var.name = variant.name
+        var.type = variant.type
+        ti.variants.add(var)
+
+    for variant_id in ti.variants:
+        var = ti.variants[variant_id]
+        var.paths.packages = os.path.join(var.uid, "Packages")
+        var.paths.repository = var.uid
+    ti.dump(dest_file)
+
+
+def get_filename(compose, variant, arch, format):
+    disc_type = compose.conf["disc_types"].get("dvd", "dvd")
+    base_filename = compose.get_image_name(
+        arch, variant, disc_type=disc_type, disc_num=1
+    )
+    if not format:
+        return base_filename
+    kwargs = {
+        "arch": arch,
+        "disc_type": disc_type,
+        "disc_num": 1,
+        "suffix": ".iso",
+        "filename": base_filename,
+        "variant": variant,
+    }
+    args = get_format_substs(compose, **kwargs)
+    try:
+        return (format % args).format(**args)
+    except KeyError as err:
+        raise RuntimeError(
+            "Failed to create image name: unknown format element: %s" % err
+        )
+
+
+def get_volume_id(compose, variant, arch, formats):
+    disc_type = compose.conf["disc_types"].get("dvd", "dvd")
+    # Get volume ID for regular ISO so that we can substitute it in.
+    volid = get_volid(compose, arch, variant, disc_type=disc_type)
+    return get_volid(
+        compose,
+        arch,
+        variant,
+        disc_type=disc_type,
+        formats=force_list(formats),
+        volid=volid,
+    )
+
+
+def prepare_media_metadata(compose, variant, arch):
+    """Write a .discinfo and media.repo files to a directory that will be
+    included on the ISO. It's possible to overwrite the files by using extra
+    files.
+    """
+    md_dir = compose.paths.work.extra_iso_extra_files_dir(arch, variant)
+    description = metadata.get_description(compose, variant, arch)
+    metadata.create_media_repo(
+        os.path.join(md_dir, "media.repo"), description, timestamp=None
+    )
+    metadata.create_discinfo(os.path.join(md_dir, ".discinfo"), description, arch)

pungi/phases/gather/link.py

@@ -0,0 +1,167 @@
+# -*- coding: utf-8 -*-
+
+
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; version 2 of the License.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Library General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, see <https://gnu.org/licenses/>.
+
+
+import os
+
+import kobo.rpmlib
+
+from pungi.linker import LinkerPool
+
+
+# TODO: global Linker instance - to keep hardlinks on dest?
+# DONE: show overall progress, not each file
+#   TODO: (these should be logged separately)
+
+
+def _get_src_nevra(compose, pkg_obj, srpm_map):
+    """Return source N-E:V-R.A.rpm; guess if necessary."""
+    result = srpm_map.get(pkg_obj.sourcerpm, None)
+    if not result:
+        nvra = kobo.rpmlib.parse_nvra(pkg_obj.sourcerpm)
+        nvra["epoch"] = pkg_obj.epoch
+        result = kobo.rpmlib.make_nvra(nvra, add_rpm=True, force_epoch=True)
+        compose.log_warning(
+            "Package %s has no SRPM available, guessing epoch: %s"
+            % (pkg_obj.nevra, result)
+        )
+    return result
+
+
+def get_package_path(filename, hashed_directory=False):
+    """Get path for filename. If ``hashed_directory`` is ``True``, the path
+    will include a prefix based on the initial letter.
+
+    >>> get_package_path('my-package.rpm')
+    'my-package.rpm'
+    >>> get_package_path('my-package.rpm', True)
+    'm/my-package.rpm'
+    >>> get_package_path('My-Package.rpm', True)
+    'm/My-Package.rpm'
+    """
+    if hashed_directory:
+        prefix = filename[0].lower()
+        return os.path.join(prefix, filename)
+    return filename
+
+
+def link_files(compose, arch, variant, pkg_map, pkg_sets, manifest, srpm_map={}):
+    # srpm_map instance is shared between link_files() runs
+
+    msg = "Linking packages (arch: %s, variant: %s)" % (arch, variant)
+    compose.log_info("[BEGIN] %s" % msg)
+    link_type = compose.conf["link_type"]
+
+    pool = LinkerPool.with_workers(10, link_type, logger=compose._logger)
+
+    hashed_directories = compose.conf["hashed_directories"]
+
+    # Create temporary dict mapping package path to package object from pkgset
+    # so we do not have to search all pkg_sets for every package in pkg_map.
+    pkg_by_path = {}
+    for pkg_set in pkg_sets:
+        for path in pkg_set[arch]:
+            pkg_by_path[path] = pkg_set[arch][path]
+
+    packages_dir = compose.paths.compose.packages("src", variant)
+    packages_dir_relpath = compose.paths.compose.packages("src", variant, relative=True)
+    for pkg in pkg_map["srpm"]:
+        if "lookaside" in pkg["flags"]:
+            continue
+        package_path = get_package_path(
+            os.path.basename(pkg["path"]), hashed_directories
+        )
+        dst = os.path.join(packages_dir, package_path)
+        dst_relpath = os.path.join(packages_dir_relpath, package_path)
+
+        # link file
+        pool.queue_put((pkg["path"], dst))
+
+        # update rpm manifest
+        pkg_obj = pkg_by_path[pkg["path"]]
+        nevra = pkg_obj.nevra
+        manifest.add(
+            variant.uid,
+            arch,
+            nevra,
+            path=dst_relpath,
+            sigkey=pkg_obj.signature,
+            category="source",
+        )
+
+        # update srpm_map
+        srpm_map.setdefault(pkg_obj.file_name, nevra)
+
+    packages_dir = compose.paths.compose.packages(arch, variant)
+    packages_dir_relpath = compose.paths.compose.packages(arch, variant, relative=True)
+    for pkg in pkg_map["rpm"]:
+        if "lookaside" in pkg["flags"]:
+            continue
+        package_path = get_package_path(
+            os.path.basename(pkg["path"]), hashed_directories
+        )
+        dst = os.path.join(packages_dir, package_path)
+        dst_relpath = os.path.join(packages_dir_relpath, package_path)
+
+        # link file
+        pool.queue_put((pkg["path"], dst))
+
+        # update rpm manifest
+        pkg_obj = pkg_by_path[pkg["path"]]
+        nevra = pkg_obj.nevra
+        src_nevra = _get_src_nevra(compose, pkg_obj, srpm_map)
+        manifest.add(
+            variant.uid,
+            arch,
+            nevra,
+            path=dst_relpath,
+            sigkey=pkg_obj.signature,
+            category="binary",
+            srpm_nevra=src_nevra,
+        )
+
+    packages_dir = compose.paths.compose.debug_packages(arch, variant)
+    packages_dir_relpath = compose.paths.compose.debug_packages(
+        arch, variant, relative=True
+    )
+    for pkg in pkg_map["debuginfo"]:
+        if "lookaside" in pkg["flags"]:
+            continue
+        package_path = get_package_path(
+            os.path.basename(pkg["path"]), hashed_directories
+        )
+        dst = os.path.join(packages_dir, package_path)
+        dst_relpath = os.path.join(packages_dir_relpath, package_path)
+
+        # link file
+        pool.queue_put((pkg["path"], dst))
+
+        # update rpm manifest
+        pkg_obj = pkg_by_path[pkg["path"]]
+        nevra = pkg_obj.nevra
+        src_nevra = _get_src_nevra(compose, pkg_obj, srpm_map)
+        manifest.add(
+            variant.uid,
+            arch,
+            nevra,
+            path=dst_relpath,
+            sigkey=pkg_obj.signature,
+            category="debug",
+            srpm_nevra=src_nevra,
+        )
+
+    pool.start()
+    pool.stop()
+    compose.log_info("[DONE ] %s" % msg)

pungi/phases/gather/method.py

@@ -0,0 +1,19 @@
+# -*- coding: utf-8 -*-
+
+
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; version 2 of the License.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Library General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, see <https://gnu.org/licenses/>.
+
+
+class GatherMethodBase(object):
+    def __init__(self, compose):
+        self.compose = compose

pungi/phases/gather/methods/__init__.py

@@ -0,0 +1,24 @@
+# -*- coding: utf-8 -*-
+
+
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; version 2 of the License.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Library General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, see <https://gnu.org/licenses/>.
+
+from .method_deps import GatherMethodDeps
+from .method_nodeps import GatherMethodNodeps
+from .method_hybrid import GatherMethodHybrid
+
+ALL_METHODS = {
+    "deps": GatherMethodDeps,
+    "nodeps": GatherMethodNodeps,
+    "hybrid": GatherMethodHybrid,
+}

pungi/phases/gather/methods/method_deps.py

@@ -0,0 +1,286 @@
+# -*- coding: utf-8 -*-
+
+
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; version 2 of the License.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Library General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, see <https://gnu.org/licenses/>.
+
+
+import os
+import shutil
+
+from kobo.shortcuts import run
+from kobo.pkgset import SimpleRpmWrapper, RpmWrapper
+from kobo.rpmlib import parse_nvra
+
+from pungi.util import get_arch_variant_data, temp_dir
+from pungi.wrappers.pungi import PungiWrapper
+
+from pungi.arch import tree_arch_to_yum_arch
+import pungi.phases.gather
+from pungi.phases.pkgset.pkgsets import ExtendedRpmWrapper
+
+import pungi.phases.gather.method
+
+
+class GatherMethodDeps(pungi.phases.gather.method.GatherMethodBase):
+    def __call__(
+        self,
+        arch,
+        variant,
+        packages,
+        groups,
+        filter_packages,
+        multilib_whitelist,
+        multilib_blacklist,
+        package_sets,
+        path_prefix=None,
+        fulltree_excludes=None,
+        prepopulate=None,
+    ):
+        # result = {
+        #     "rpm": [],
+        #     "srpm": [],
+        #     "debuginfo": [],
+        # }
+
+        write_pungi_config(
+            self.compose,
+            arch,
+            variant,
+            packages,
+            groups,
+            filter_packages,
+            multilib_whitelist,
+            multilib_blacklist,
+            fulltree_excludes=fulltree_excludes,
+            prepopulate=prepopulate,
+            source_name=self.source_name,
+            package_sets=package_sets,
+        )
+        result, missing_deps = resolve_deps(
+            self.compose, arch, variant, source_name=self.source_name
+        )
+        raise_on_invalid_sigkeys(arch, variant, package_sets, result)
+        check_deps(self.compose, arch, variant, missing_deps)
+        return result
+
+
+def raise_on_invalid_sigkeys(arch, variant, package_sets, result):
+    """
+    Raises RuntimeError if some package in compose is signed with an invalid
+    sigkey.
+    """
+    invalid_sigkey_rpms = {}
+    for package in result["rpm"]:
+        name = parse_nvra(package["path"])["name"]
+        for pkgset in package_sets:
+            for forbidden_package in pkgset["global"].invalid_sigkey_rpms:
+                if name == forbidden_package["name"]:
+                    invalid_sigkey_rpms.setdefault(
+                        pkgset["global"].sigkey_ordering, []
+                    ).append(forbidden_package)
+
+    if invalid_sigkey_rpms:
+        package_sets[0]["global"].raise_invalid_sigkeys_exception(invalid_sigkey_rpms)
+
+
+def _format_packages(pkgs):
+    """Sort packages and merge name with arch."""
+    result = set()
+    for pkg, pkg_arch in pkgs:
+        if type(pkg) in [SimpleRpmWrapper, RpmWrapper, ExtendedRpmWrapper]:
+            pkg_name = pkg.name
+        else:
+            pkg_name = pkg
+        if pkg_arch:
+            result.add("%s.%s" % (pkg_name, pkg_arch))
+        else:
+            result.add(pkg_name)
+    return sorted(result)
+
+
+def write_pungi_config(
+    compose,
+    arch,
+    variant,
+    packages,
+    groups,
+    filter_packages,
+    multilib_whitelist,
+    multilib_blacklist,
+    fulltree_excludes=None,
+    prepopulate=None,
+    source_name=None,
+    package_sets=None,
+):
+    """write pungi config (kickstart) for arch/variant"""
+    pungi_wrapper = PungiWrapper()
+    pungi_cfg = compose.paths.work.pungi_conf(
+        variant=variant, arch=arch, source_name=source_name
+    )
+
+    compose.log_info(
+        "Writing pungi config (arch: %s, variant: %s): %s", arch, variant, pungi_cfg
+    )
+
+    repos = {}
+    for i, pkgset in enumerate(package_sets or []):
+        if not variant.pkgsets or pkgset.name in variant.pkgsets:
+            repos["pungi-repo-%d" % i] = pkgset.paths[arch]
+    if compose.has_comps:
+        repos["comps-repo"] = compose.paths.work.comps_repo(arch=arch, variant=variant)
+    if variant.type == "optional":
+        for var in variant.parent.get_variants(
+            arch=arch, types=["self", "variant", "addon", "layered-product"]
+        ):
+            repos["%s-comps" % var.uid] = compose.paths.work.comps_repo(
+                arch=arch, variant=var
+            )
+    if variant.type in ["addon", "layered-product"]:
+        repos["parent-comps"] = compose.paths.work.comps_repo(
+            arch=arch, variant=variant.parent
+        )
+
+    lookaside_repos = {}
+    for i, repo_url in enumerate(
+        pungi.phases.gather.get_lookaside_repos(compose, arch, variant)
+    ):
+        lookaside_repos["lookaside-repo-%s" % i] = repo_url
+
+    packages_str = list(_format_packages(packages))
+    filter_packages_str = list(_format_packages(filter_packages))
+
+    if not groups and not packages_str and not prepopulate:
+        raise RuntimeError(
+            "No packages included in %s.%s "
+            "(no comps groups, no input packages, no prepopulate)" % (variant.uid, arch)
+        )
+
+    pungi_wrapper.write_kickstart(
+        ks_path=pungi_cfg,
+        repos=repos,
+        groups=groups,
+        packages=packages_str,
+        exclude_packages=filter_packages_str,
+        lookaside_repos=lookaside_repos,
+        fulltree_excludes=fulltree_excludes,
+        multilib_whitelist=multilib_whitelist,
+        multilib_blacklist=multilib_blacklist,
+        prepopulate=prepopulate,
+    )
+
+
+def resolve_deps(compose, arch, variant, source_name=None):
+    pungi_wrapper = PungiWrapper()
+    pungi_log = compose.paths.work.pungi_log(arch, variant, source_name=source_name)
+
+    msg = "Running pungi (arch: %s, variant: %s)" % (arch, variant)
+
+    compose.log_info("[BEGIN] %s" % msg)
+    pungi_conf = compose.paths.work.pungi_conf(arch, variant, source_name=source_name)
+
+    multilib_methods = get_arch_variant_data(compose.conf, "multilib", arch, variant)
+
+    greedy_method = compose.conf["greedy_method"]
+
+    # variant
+    fulltree = compose.conf["gather_fulltree"]
+    selfhosting = compose.conf["gather_selfhosting"]
+
+    # profiling
+    profiler = compose.conf["gather_profiler"]
+
+    # optional
+    if variant.type == "optional":
+        fulltree = True
+        selfhosting = True
+
+    # addon
+    if variant.type in ["addon", "layered-product"]:
+        # packages having SRPM in parent variant are excluded from
+        # fulltree (via %fulltree-excludes)
+        fulltree = True
+        selfhosting = False
+
+    lookaside_repos = {}
+    for i, repo_url in enumerate(
+        pungi.phases.gather.get_lookaside_repos(compose, arch, variant)
+    ):
+        lookaside_repos["lookaside-repo-%s" % i] = repo_url
+
+    yum_arch = tree_arch_to_yum_arch(arch)
+    tmp_dir = compose.paths.work.tmp_dir(arch, variant)
+    cache_dir = compose.paths.work.pungi_cache_dir(arch, variant)
+    # TODO: remove YUM code, fully migrate to DNF
+    backends = {
+        "yum": pungi_wrapper.get_pungi_cmd,
+        "dnf": pungi_wrapper.get_pungi_cmd_dnf,
+    }
+    get_cmd = backends[compose.conf["gather_backend"]]
+    cmd = get_cmd(
+        pungi_conf,
+        destdir=tmp_dir,
+        name=variant.uid,
+        selfhosting=selfhosting,
+        fulltree=fulltree,
+        arch=yum_arch,
+        full_archlist=True,
+        greedy=greedy_method,
+        cache_dir=cache_dir,
+        lookaside_repos=lookaside_repos,
+        multilib_methods=multilib_methods,
+        profiler=profiler,
+    )
+    # Use temp working directory directory as workaround for
+    # https://bugzilla.redhat.com/show_bug.cgi?id=795137
+    with temp_dir(prefix="pungi_") as work_dir:
+        run(cmd, logfile=pungi_log, show_cmd=True, workdir=work_dir, env=os.environ)
+
+    # Clean up tmp dir
+    # Workaround for rpm not honoring sgid bit which only appears when yum is used.
+    yumroot_dir = os.path.join(tmp_dir, "work", arch, "yumroot")
+    if os.path.isdir(yumroot_dir):
+        try:
+            shutil.rmtree(yumroot_dir)
+        except Exception as e:
+            compose.log_warning(
+                "Failed to clean up tmp dir: %s %s" % (yumroot_dir, str(e))
+            )
+
+    with open(pungi_log, "r") as f:
+        packages, broken_deps, missing_comps_pkgs = pungi_wrapper.parse_log(f)
+
+    if missing_comps_pkgs:
+        log_msg = "Packages mentioned in comps do not exist for %s.%s: %s" % (
+            variant.uid,
+            arch,
+            ", ".join(sorted(missing_comps_pkgs)),
+        )
+        compose.log_warning(log_msg)
+        if compose.conf["require_all_comps_packages"]:
+            raise RuntimeError(log_msg)
+
+    compose.log_info("[DONE ] %s" % msg)
+    return packages, broken_deps
+
+
+def check_deps(compose, arch, variant, missing_deps):
+    if not compose.conf["check_deps"]:
+        return
+
+    if missing_deps:
+        for pkg in sorted(missing_deps):
+            compose.log_error(
+                "Unresolved dependencies for %s.%s in package %s: %s"
+                % (variant, arch, pkg, sorted(missing_deps[pkg]))
+            )
+        raise RuntimeError("Unresolved dependencies detected")

pungi/phases/gather/methods/method_hybrid.py

@@ -0,0 +1,580 @@
+# -*- coding: utf-8 -*-
+
+
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; version 2 of the License.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Library General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, see <https://gnu.org/licenses/>.
+
+import gzip
+import os
+from collections import defaultdict
+from fnmatch import fnmatch
+
+import createrepo_c as cr
+import kobo.rpmlib
+from kobo.shortcuts import run
+
+import pungi.phases.gather.method
+from pungi import multilib_dnf
+from pungi.module_util import Modulemd
+from pungi.arch import get_valid_arches, tree_arch_to_yum_arch
+from pungi.phases.gather import _mk_pkg_map
+from pungi.util import get_arch_variant_data, pkg_is_debug, temp_dir, as_local_file
+from pungi.wrappers import fus
+from pungi.wrappers.comps import CompsWrapper
+
+from .method_nodeps import expand_groups
+
+
+class FakePackage(object):
+    """This imitates a DNF package object and can be passed to python-multilib
+    library.
+    """
+
+    def __init__(self, pkg):
+        self.pkg = pkg
+
+    def __getattr__(self, attr):
+        return getattr(self.pkg, attr)
+
+    @property
+    def files(self):
+        paths = []
+        # createrepo_c.Package.files is a tuple, but its length differs across
+        # versions. The constants define index at which the related value is
+        # located.
+        for entry in self.pkg.files:
+            paths.append(
+                os.path.join(entry[cr.FILE_ENTRY_PATH], entry[cr.FILE_ENTRY_NAME])
+            )
+        return paths
+
+    @property
+    def provides(self):
+        # This is supposed to match what yum package object returns. It's a
+        # nested tuple (name, flag, (epoch, version, release)). This code only
+        # fills in the name, because that's all that python-multilib is using..
+        return [(p[0].split()[0], None, (None, None, None)) for p in self.pkg.provides]
+
+
+class GatherMethodHybrid(pungi.phases.gather.method.GatherMethodBase):
+    def __init__(self, *args, **kwargs):
+        super(GatherMethodHybrid, self).__init__(*args, **kwargs)
+        self.package_maps = {}
+        self.packages = {}
+        # Mapping from package name to set of langpack packages (stored as
+        # names).
+        self.langpacks = {}
+        # Set of packages for which we already added langpacks.
+        self.added_langpacks = set()
+        # Set of NEVRAs of modular packages
+        self.modular_packages = set()
+        # Arch -> pkg name -> set of pkg object
+        self.debuginfo = defaultdict(lambda: defaultdict(set))
+
+        # caches for processed packages
+        self.processed_multilib = set()
+        self.processed_debuginfo = set()
+
+    def _get_pkg_map(self, arch):
+        """Create a mapping from NEVRA to actual package object. This will be
+        done once for each architecture, since the package set is the same for
+        all variants.
+
+        The keys are in NEVRA format and only include the epoch if it's not
+        zero. This makes it easier to query by results for the depsolver.
+        """
+        if arch not in self.package_maps:
+            pkg_map = {}
+            for pkgset in self.package_sets:
+                for pkg_arch in pkgset.package_sets[arch].rpms_by_arch:
+                    for pkg in pkgset.package_sets[arch].rpms_by_arch[pkg_arch]:
+                        pkg_map[_fmt_nevra(pkg, pkg_arch)] = pkg
+            self.package_maps[arch] = pkg_map
+
+        return self.package_maps[arch]
+
+    def _prepare_packages(self):
+        for repo_path in self.get_repos():
+            md = cr.Metadata()
+            md.locate_and_load_xml(repo_path)
+            for key in md.keys():
+                pkg = md.get(key)
+                if pkg.arch in self.valid_arches:
+                    self.packages[_fmt_nevra(pkg, arch=pkg.arch)] = FakePackage(pkg)
+
+    def _get_package(self, nevra):
+        if not self.packages:
+            self._prepare_packages()
+        return self.packages[nevra]
+
+    def _prepare_debuginfo(self):
+        """Prepare cache of debuginfo packages for easy access. The cache is
+        indexed by package architecture and then by package name. There can be
+        more than one debuginfo package with the same name.
+        """
+        for pkgset in self.package_sets:
+            for pkg_arch in pkgset.package_sets[self.arch].rpms_by_arch:
+                for pkg in pkgset.package_sets[self.arch].rpms_by_arch[pkg_arch]:
+                    self.debuginfo[pkg.arch][pkg.name].add(pkg)
+
+    def _get_debuginfo(self, name, arch):
+        if not self.debuginfo:
+            self._prepare_debuginfo()
+        return self.debuginfo.get(arch, {}).get(name, set())
+
+    def expand_list(self, patterns):
+        """Given a list of globs, create a list of package names matching any
+        of the pattern.
+        """
+        expanded = set()
+        for pkgset in self.package_sets:
+            for pkg_arch in pkgset.package_sets[self.arch].rpms_by_arch:
+                for pkg in pkgset.package_sets[self.arch].rpms_by_arch[pkg_arch]:
+                    for pattern in patterns:
+                        if fnmatch(pkg.name, pattern):
+                            expanded.add(pkg)
+                            break
+        return expanded
+
+    def prepare_modular_packages(self):
+        for var in self.compose.all_variants.values():
+            for mmd in var.arch_mmds.get(self.arch, {}).values():
+                self.modular_packages.update(mmd.get_rpm_artifacts())
+
+    def prepare_langpacks(self, arch, variant):
+        if not self.compose.has_comps:
+            return
+        comps_file = self.compose.paths.work.comps(arch, variant, create_dir=False)
+        comps = CompsWrapper(comps_file)
+
+        for name, install in comps.get_langpacks().items():
+            # Replace %s with * for fnmatch.
+            install_match = install % "*"
+            self.langpacks[name] = set()
+            for pkgset in self.package_sets:
+                for pkg_arch in pkgset.package_sets[arch].rpms_by_arch:
+                    for pkg in pkgset.package_sets[arch].rpms_by_arch[pkg_arch]:
+                        if not fnmatch(pkg.name, install_match):
+                            # Does not match the pattern, ignore...
+                            continue
+                        if pkg.name.endswith("-devel") or pkg.name.endswith("-static"):
+                            continue
+                        if pkg_is_debug(pkg):
+                            continue
+                        self.langpacks[name].add(pkg.name)
+
+    def __call__(
+        self,
+        arch,
+        variant,
+        package_sets,
+        packages=[],
+        groups=[],
+        multilib_whitelist=[],
+        multilib_blacklist=[],
+        filter_packages=[],
+        prepopulate=[],
+        **kwargs
+    ):
+        self.arch = arch
+        self.variant = variant
+        self.valid_arches = get_valid_arches(arch, multilib=True)
+        self.package_sets = package_sets
+
+        self.prepare_langpacks(arch, variant)
+        self.prepare_modular_packages()
+
+        self.multilib_methods = get_arch_variant_data(
+            self.compose.conf, "multilib", arch, variant
+        )
+        self.multilib = multilib_dnf.Multilib(
+            self.multilib_methods,
+            set(p.name for p in self.expand_list(multilib_blacklist)),
+            set(p.name for p in self.expand_list(multilib_whitelist)),
+        )
+
+        platform = get_platform(self.compose, variant, arch)
+
+        packages.update(
+            expand_groups(self.compose, arch, variant, groups, set_pkg_arch=False)
+        )
+
+        packages.update(tuple(pkg.rsplit(".", 1)) for pkg in prepopulate)
+
+        # Filters are received as tuples (name, arch), we should convert it to
+        # strings.
+        filters = [_fmt_pkg(*p) for p in filter_packages]
+
+        cache_prefix = "fus-cache-%s-%s-%s-" % (self.compose.compose_id, variant, arch)
+        with temp_dir(prefix=cache_prefix) as cache_dir:
+            nvrs, out_modules = self.run_solver(
+                variant, arch, packages, platform, filters, cache_dir=cache_dir
+            )
+        filter_modules(variant, arch, out_modules)
+        return expand_packages(
+            self._get_pkg_map(arch),
+            pungi.phases.gather.get_lookaside_repos(self.compose, arch, variant),
+            nvrs,
+            filter_packages=filter_packages,
+        )
+        # maybe check invalid sigkeys
+
+    def get_repos(self):
+        repos = []
+        for pkgset in self.package_sets:
+            if self.variant.pkgsets and pkgset.name not in self.variant.pkgsets:
+                continue
+            repos.append(pkgset.paths[self.arch])
+        return repos
+
+    def run_solver(self, variant, arch, packages, platform, filter_packages, cache_dir):
+        repos = self.get_repos()
+        results = set()
+        result_modules = set()
+
+        modules = []
+        for mmd in variant.arch_mmds.get(arch, {}).values():
+            modules.append("%s:%s" % (mmd.get_module_name(), mmd.get_stream_name()))
+
+        input_packages = []
+        for pkg_name, pkg_arch in packages:
+            input_packages.extend(self._expand_wildcard(pkg_name, pkg_arch))
+
+        step = 0
+
+        while True:
+            step += 1
+            conf_file = self.compose.paths.work.fus_conf(arch, variant, step)
+            fus.write_config(conf_file, sorted(modules), sorted(input_packages))
+            cmd = fus.get_cmd(
+                conf_file,
+                tree_arch_to_yum_arch(arch),
+                repos,
+                pungi.phases.gather.get_lookaside_repos(self.compose, arch, variant),
+                platform=platform,
+                filter_packages=filter_packages,
+            )
+            logfile = self.compose.paths.log.log_file(
+                arch, "hybrid-depsolver-%s-iter-%d" % (variant, step)
+            )
+            # Adding this environment variable will tell GLib not to prefix
+            # any log messages with the PID of the fus process (which is quite
+            # useless for us anyway).
+            env = os.environ.copy()
+            env["G_MESSAGES_PREFIXED"] = ""
+            env["XDG_CACHE_HOME"] = cache_dir
+            self.compose.log_debug(
+                "[BEGIN] Running fus (arch: %s, variant: %s)" % (arch, variant)
+            )
+            run(cmd, logfile=logfile, show_cmd=True, env=env)
+            output, out_modules = fus.parse_output(logfile)
+            self.compose.log_debug(
+                "[DONE ] Running fus (arch: %s, variant: %s)" % (arch, variant)
+            )
+            # No need to resolve modules again. They are not going to change.
+            modules = []
+            # Reset input packages as well to only solve newly added things.
+            input_packages = []
+            # Preserve the results from this iteration.
+            results.update(output)
+            result_modules.update(out_modules)
+
+            new_multilib = self.add_multilib(variant, arch, output)
+            input_packages.extend(
+                _fmt_pkg(pkg_name, pkg_arch)
+                for pkg_name, pkg_arch in sorted(new_multilib)
+            )
+
+            new_debuginfo = self.add_debuginfo(arch, output)
+            input_packages.extend(
+                _fmt_pkg(pkg_name, pkg_arch)
+                for pkg_name, pkg_arch in sorted(new_debuginfo)
+            )
+
+            new_langpacks = self.add_langpacks(output)
+            input_packages.extend(new_langpacks)
+
+            if not input_packages:
+                # Nothing new was added, we can stop now.
+                break
+
+        return results, result_modules
+
+    def add_multilib(self, variant, arch, nvrs):
+        added = set()
+        if not self.multilib_methods:
+            return []
+
+        for nvr, pkg_arch, flags in nvrs:
+            if (nvr, pkg_arch) in self.processed_multilib:
+                continue
+            self.processed_multilib.add((nvr, pkg_arch))
+
+            if "modular" in flags:
+                continue
+
+            if pkg_arch != arch:
+                # Not a native package, not checking to add multilib
+                continue
+
+            nevr = kobo.rpmlib.parse_nvr(nvr)
+
+            for add_arch in self.valid_arches:
+                if add_arch == arch:
+                    continue
+                try:
+                    multilib_candidate = self._get_package("%s.%s" % (nvr, add_arch))
+                except KeyError:
+                    continue
+                if self.multilib.is_multilib(multilib_candidate):
+                    added.add((nevr["name"], add_arch))
+
+        return added
+
+    def add_debuginfo(self, arch, nvrs):
+        added = set()
+
+        for nvr, pkg_arch, flags in nvrs:
+            if (nvr, pkg_arch) in self.processed_debuginfo:
+                continue
+            self.processed_debuginfo.add((nvr, pkg_arch))
+
+            if "modular" in flags:
+                continue
+
+            pkg = self._get_package("%s.%s" % (nvr, pkg_arch))
+
+            # There are two ways how the debuginfo package can be named. We
+            # want to get them all.
+            source_name = kobo.rpmlib.parse_nvra(pkg.rpm_sourcerpm)["name"]
+            for debuginfo_name in [
+                "%s-debuginfo" % pkg.name,
+                "%s-debugsource" % source_name,
+            ]:
+                debuginfo = self._get_debuginfo(debuginfo_name, pkg_arch)
+                for dbg in debuginfo:
+                    # For each debuginfo package that matches on name and
+                    # architecture, we also need to check if it comes from the
+                    # same build.
+                    if dbg.sourcerpm == pkg.rpm_sourcerpm:
+                        added.add((dbg.name, dbg.arch))
+
+        return added
+
+    def add_langpacks(self, nvrs):
+        if not self.langpacks:
+            return set()
+
+        added = set()
+        for nvr, pkg_arch, flags in nvrs:
+            if "modular" in flags:
+                continue
+            name = nvr.rsplit("-", 2)[0]
+            if name in self.added_langpacks:
+                # This package is already processed.
+                continue
+            added.update(self.langpacks.get(name, []))
+            self.added_langpacks.add(name)
+
+        return sorted(added)
+
+    def _expand_wildcard(self, pkg_name, pkg_arch):
+        if "*" not in pkg_name:
+            return [_fmt_pkg(pkg_name, pkg_arch)]
+
+        packages = []
+
+        for pkg in self.expand_list([pkg_name]):
+            if pkg_is_debug(pkg):
+                # No debuginfo
+                continue
+
+            if pkg_arch:
+                if pkg_arch != pkg.arch:
+                    # Arch is specified and does not match, skip the package.
+                    continue
+            else:
+                if pkg.arch not in ("noarch", self.arch):
+                    # No arch specified and package does not match
+                    continue
+
+            strict_nevra = "%s-%s:%s-%s.%s" % (
+                pkg.name,
+                pkg.epoch or "0",
+                pkg.version,
+                pkg.release,
+                pkg.arch,
+            )
+            if strict_nevra in self.modular_packages:
+                # Wildcards should not match modular packages.
+                continue
+
+            packages.append(_fmt_nevra(pkg, pkg.arch))
+
+        return packages
+
+
+def iter_platforms_in_repo(url):
+    """Find all platform streams that any module in give repo requires at runtime.
+    Yields lists of stream names (possible empty).
+    """
+    repomd = os.path.join(url, "repodata/repomd.xml")
+    with as_local_file(repomd) as url_:
+        repomd = cr.Repomd(url_)
+    for rec in repomd.records:
+        if rec.type != "modules":
+            continue
+        # No with statement on Python 2.6 for GzipFile...
+        record_url = os.path.join(url, rec.location_href)
+        with as_local_file(record_url) as url_:
+            gzipped_file = gzip.GzipFile(url_, "rb")
+        mod_index = Modulemd.ModuleIndex.new()
+        mod_index.update_from_string(gzipped_file.read().decode("utf-8"), False)
+        gzipped_file.close()
+        for module_name in mod_index.get_module_names():
+            module = mod_index.get_module(module_name)
+            for module_stream in module.get_all_streams():
+                module_stream = module_stream.upgrade(2)
+                for dep in module_stream.get_dependencies():
+                    yield dep.get_runtime_streams("platform")
+
+
+def get_platform_from_lookasides(compose, variant, arch):
+    """Find a set of all platform dependencies in all lookaside repos."""
+    platforms = set()
+    for repo in pungi.phases.gather.get_lookaside_repos(compose, arch, variant):
+        for ps in iter_platforms_in_repo(fus._prep_path(repo)):
+            platforms.update(ps)
+    return platforms
+
+
+def get_platform(compose, variant, arch):
+    """Find platform stream for modules. Raises RuntimeError if there are
+    conflicting requests.
+    """
+    platforms = get_platform_from_lookasides(compose, variant, arch)
+
+    for var in compose.all_variants.values():
+        for mmd in var.arch_mmds.get(arch, {}).values():
+            for dep in mmd.get_dependencies():
+                streams = dep.get_runtime_streams("platform")
+                if streams:
+                    platforms.update(streams)
+
+    if len(platforms) > 1:
+        raise RuntimeError("There are conflicting requests for platform.")
+
+    return list(platforms)[0] if platforms else None
+
+
+def _fmt_pkg(pkg_name, arch):
+    if arch:
+        pkg_name += ".%s" % arch
+    return pkg_name
+
+
+def _nevra(**kwargs):
+    if kwargs.get("epoch") not in (None, "", 0, "0"):
+        return "%(name)s-%(epoch)s:%(version)s-%(release)s.%(arch)s" % kwargs
+    return "%(name)s-%(version)s-%(release)s.%(arch)s" % kwargs
+
+
+def _fmt_nevra(pkg, arch):
+    return _nevra(
+        name=pkg.name,
+        epoch=pkg.epoch,
+        version=pkg.version,
+        release=pkg.release,
+        arch=arch,
+    )
+
+
+def _get_srpm_nevra(pkg):
+    nevra = kobo.rpmlib.parse_nvra(pkg.sourcerpm)
+    nevra["epoch"] = nevra["epoch"] or pkg.epoch
+    return _nevra(**nevra)
+
+
+def _make_result(paths):
+    return [{"path": path, "flags": []} for path in sorted(paths)]
+
+
+def get_repo_packages(path):
+    """Extract file names of all packages in the given repository."""
+
+    packages = set()
+
+    def callback(pkg):
+        packages.add(os.path.basename(pkg.location_href))
+
+    repomd = os.path.join(path, "repodata/repomd.xml")
+    with as_local_file(repomd) as url_:
+        repomd = cr.Repomd(url_)
+    for rec in repomd.records:
+        if rec.type != "primary":
+            continue
+        record_url = os.path.join(path, rec.location_href)
+        with as_local_file(record_url) as url_:
+            cr.xml_parse_primary(url_, pkgcb=callback, do_files=False)
+
+    return packages
+
+
+def expand_packages(nevra_to_pkg, lookasides, nvrs, filter_packages):
+    """For each package add source RPM."""
+    # This will serve as the final result. We collect sets of paths to the
+    # packages.
+    rpms = set()
+    srpms = set()
+    debuginfo = set()
+
+    filters = set(filter_packages)
+
+    lookaside_packages = set()
+    for repo in lookasides:
+        lookaside_packages.update(get_repo_packages(repo))
+
+    for nvr, pkg_arch, flags in nvrs:
+        pkg = nevra_to_pkg["%s.%s" % (nvr, pkg_arch)]
+        if os.path.basename(pkg.file_path) in lookaside_packages:
+            # Fus can return lookaside package in output if the package is
+            # explicitly listed as input. This can happen during comps
+            # expansion.
+            continue
+        if pkg_is_debug(pkg):
+            debuginfo.add(pkg.file_path)
+        else:
+            rpms.add(pkg.file_path)
+
+        try:
+            srpm_nevra = _get_srpm_nevra(pkg)
+            srpm = nevra_to_pkg[srpm_nevra]
+            if (srpm.name, "src") in filters:
+                # Filtered package, skipping
+                continue
+            if os.path.basename(srpm.file_path) not in lookaside_packages:
+                srpms.add(srpm.file_path)
+        except KeyError:
+            # Didn't find source RPM.. this should be logged
+            pass
+
+    return _mk_pkg_map(_make_result(rpms), _make_result(srpms), _make_result(debuginfo))
+
+
+def filter_modules(variant, arch, nsvcs_to_keep):
+    """Remove any arch-specific module metadata from the module if it's not
+    listed in the list to keep. This will ultimately cause the module to not be
+    included in the final repodata and module metadata.
+    """
+    for nsvc in list(variant.arch_mmds.get(arch, {}).keys()):
+        if nsvc not in nsvcs_to_keep:
+            del variant.arch_mmds[arch][nsvc]

pungi/phases/gather/methods/method_nodeps.py

@@ -0,0 +1,184 @@
+# -*- coding: utf-8 -*-
+
+
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; version 2 of the License.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Library General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, see <https://gnu.org/licenses/>.
+
+import os
+from pprint import pformat
+import re
+import six
+
+import pungi.arch
+from pungi.util import pkg_is_rpm, pkg_is_srpm, pkg_is_debug
+from pungi.wrappers.comps import CompsWrapper
+from pungi.phases.pkgset.pkgsets import ExtendedRpmWrapper
+
+import pungi.phases.gather.method
+from kobo.pkgset import SimpleRpmWrapper, RpmWrapper
+
+
+class GatherMethodNodeps(pungi.phases.gather.method.GatherMethodBase):
+    def __call__(self, arch, variant, *args, **kwargs):
+        fname = "gather-nodeps-%s" % variant.uid
+        if self.source_name:
+            fname += "-" + self.source_name
+        log_file = self.compose.paths.log.log_file(arch, fname)
+        with open(log_file, "w") as log:
+            return self.worker(log, arch, variant, *args, **kwargs)
+
+    def worker(
+        self,
+        log,
+        arch,
+        variant,
+        pkgs,
+        groups,
+        filter_packages,
+        multilib_whitelist,
+        multilib_blacklist,
+        package_sets,
+        path_prefix=None,
+        fulltree_excludes=None,
+        prepopulate=None,
+    ):
+        result = {
+            "rpm": [],
+            "srpm": [],
+            "debuginfo": [],
+        }
+
+        group_packages = expand_groups(self.compose, arch, variant, groups)
+        packages = pkgs | group_packages
+        log.write("Requested packages:\n%s\n" % pformat(packages))
+
+        seen_rpms = {}
+        seen_srpms = {}
+
+        valid_arches = pungi.arch.get_valid_arches(arch, multilib=True)
+        compatible_arches = {}
+        for i in valid_arches:
+            compatible_arches[i] = pungi.arch.get_compatible_arches(i)
+
+        log.write("\nGathering rpms\n")
+        for pkg in iterate_packages(package_sets, arch):
+            if not pkg_is_rpm(pkg):
+                continue
+            for gathered_pkg, pkg_arch in packages:
+                if isinstance(gathered_pkg, six.string_types) and not re.match(
+                    gathered_pkg.replace(".", "\\.")
+                    .replace("+", "\\+")
+                    .replace("*", ".*")
+                    + "$",
+                    pkg.name,
+                ):
+                    continue
+                elif (
+                    type(gathered_pkg)
+                    in [SimpleRpmWrapper, RpmWrapper, ExtendedRpmWrapper]
+                    and pkg.nevra != gathered_pkg.nevra
+                ):
+                    continue
+                if (
+                    pkg_arch is not None
+                    and pkg.arch != pkg_arch
+                    and pkg.arch != "noarch"
+                ):
+                    continue
+                result["rpm"].append({"path": pkg.file_path, "flags": ["input"]})
+                seen_rpms.setdefault(pkg.name, set()).add(pkg.arch)
+                seen_srpms.setdefault(pkg.sourcerpm, set()).add(pkg.arch)
+                log.write(
+                    "Added %s (matched %s.%s) (sourcerpm: %s)\n"
+                    % (pkg, gathered_pkg, pkg_arch, pkg.sourcerpm)
+                )
+
+        log.write("\nGathering source rpms\n")
+        for pkg in iterate_packages(package_sets, arch):
+            if not pkg_is_srpm(pkg):
+                continue
+            if pkg.file_name in seen_srpms:
+                result["srpm"].append({"path": pkg.file_path, "flags": ["input"]})
+                log.write("Adding %s\n" % pkg)
+
+        log.write("\nGathering debuginfo packages\n")
+        for pkg in iterate_packages(package_sets, arch):
+            if not pkg_is_debug(pkg):
+                continue
+            if pkg.sourcerpm not in seen_srpms:
+                log.write("Not considering %s: corresponding srpm not included\n" % pkg)
+                continue
+            pkg_arches = set(compatible_arches[pkg.arch]) - set(["noarch"])
+            seen_arches = set(seen_srpms[pkg.sourcerpm]) - set(["noarch"])
+            if not (pkg_arches & seen_arches):
+                # We only want to pull in a debuginfo if we have a binary
+                # package for a compatible arch. Noarch packages should not
+                # pull debuginfo (they would pull in all architectures).
+                log.write("Not including %s: no package for this arch\n" % pkg)
+                continue
+            result["debuginfo"].append({"path": pkg.file_path, "flags": ["input"]})
+            log.write("Adding %s\n" % pkg)
+
+        return result
+
+
+def expand_groups(compose, arch, variant, groups, set_pkg_arch=True):
+    """Read comps file filtered for given architecture and variant and return
+    all packages in given groups.
+
+    :returns: A set of tuples (pkg_name, arch)
+    """
+    if not groups:
+        # No groups, nothing to do (this also covers case when there is no
+        # comps file.
+        return set()
+    comps = []
+    comps_file = compose.paths.work.comps(arch, variant, create_dir=False)
+    comps.append(CompsWrapper(comps_file))
+
+    if variant and variant.parent:
+        parent_comps_file = compose.paths.work.comps(
+            arch, variant.parent, create_dir=False
+        )
+        comps.append(CompsWrapper(parent_comps_file))
+
+        if variant.type == "optional":
+            for v in variant.parent.variants.values():
+                if v.id == variant.id:
+                    continue
+                comps_file = compose.paths.work.comps(arch, v, create_dir=False)
+                if os.path.exists(comps_file):
+                    comps.append(CompsWrapper(comps_file))
+
+    packages = set()
+    pkg_arch = arch if set_pkg_arch else None
+    for group in groups:
+        found = False
+        ex = None
+        for c in comps:
+            try:
+                packages.update([(pkg, pkg_arch) for pkg in c.get_packages(group)])
+                found = True
+                break
+            except KeyError as e:
+                ex = e
+
+        if not found:
+            raise ex
+
+    return packages
+
+
+def iterate_packages(package_sets, arch):
+    for pkgset in package_sets:
+        for pkg in pkgset[arch]:
+            yield pkgset[arch][pkg]

pungi/phases/gather/source.py

@@ -0,0 +1,19 @@
+# -*- coding: utf-8 -*-
+
+
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; version 2 of the License.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Library General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, see <https://gnu.org/licenses/>.
+
+
+class GatherSourceBase(object):
+    def __init__(self, compose):
+        self.compose = compose

pungi/phases/gather/sources/__init__.py

@@ -0,0 +1,26 @@
+# -*- coding: utf-8 -*-
+
+
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; version 2 of the License.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Library General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, see <https://gnu.org/licenses/>.
+
+from .source_comps import GatherSourceComps
+from .source_json import GatherSourceJson
+from .source_module import GatherSourceModule
+from .source_none import GatherSourceNone
+
+ALL_SOURCES = {
+    "comps": GatherSourceComps,
+    "json": GatherSourceJson,
+    "module": GatherSourceModule,
+    "none": GatherSourceNone,
+}

pungi/phases/gather/sources/source_comps.py

@@ -0,0 +1,42 @@
+# -*- coding: utf-8 -*-
+
+
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; version 2 of the License.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Library General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, see <https://gnu.org/licenses/>.
+
+
+"""
+Get a package list based on comps.xml.
+
+Input format:
+see comps.dtd
+
+Output:
+set([(rpm_name, rpm_arch or None)])
+"""
+
+
+from pungi.wrappers.comps import CompsWrapper
+import pungi.phases.gather.source
+
+
+class GatherSourceComps(pungi.phases.gather.source.GatherSourceBase):
+    def __call__(self, arch, variant):
+        groups = set()
+        if not self.compose.conf.get("comps_file"):
+            return set(), set()
+
+        comps = CompsWrapper(self.compose.paths.work.comps(arch=arch, variant=variant))
+
+        for i in comps.get_comps_groups():
+            groups.add(i)
+        return set(), groups

pungi/phases/gather/sources/source_json.py

@@ -0,0 +1,62 @@
+# -*- coding: utf-8 -*-
+
+
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; version 2 of the License.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Library General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, see <https://gnu.org/licenses/>.
+
+
+"""
+Get a package list based on a JSON mapping.
+
+Input format:
+{
+    variant: {
+        tree_arch: {
+            rpm_name: [rpm_arch, rpm_arch, ... (or None for any/best arch)],
+        }
+    }
+}
+
+Output:
+set([(rpm_name, rpm_arch or None)])
+"""
+
+
+import json
+import os
+
+import pungi.phases.gather.source
+
+
+class GatherSourceJson(pungi.phases.gather.source.GatherSourceBase):
+    def __call__(self, arch, variant):
+        json_path = self.compose.conf.get("gather_source_mapping")
+        if not json_path:
+            return set(), set()
+        with open(os.path.join(self.compose.config_dir, json_path), "r") as f:
+            mapping = json.load(f)
+
+        packages = set()
+        if variant is None:
+            # get all packages for all variants
+            for variant_uid in mapping:
+                for pkg_name, pkg_arches in mapping[variant_uid].get(arch, {}).items():
+                    for pkg_arch in pkg_arches:
+                        packages.add((pkg_name, pkg_arch))
+        else:
+            # get packages for a particular variant
+            for pkg_name, pkg_arches in (
+                mapping.get(variant.uid, {}).get(arch, {}).items()
+            ):
+                for pkg_arch in pkg_arches:
+                    packages.add((pkg_name, pkg_arch))
+        return packages, set()

pungi/phases/gather/sources/source_module.py

@@ -0,0 +1,52 @@
+# -*- coding: utf-8 -*-
+
+
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; version 2 of the License.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Library General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, see <https://gnu.org/licenses/>.
+
+
+"""
+Get a package list based on modulemd metadata loaded in pkgset phase. Each
+modulemd file contains a list of exact RPM NEVRAs that should be include, so
+just go over all modules in a given variant and join all lists together.
+"""
+
+
+import pungi.arch
+import pungi.phases.gather.source
+
+
+class GatherSourceModule(pungi.phases.gather.source.GatherSourceBase):
+    def __call__(self, arch, variant):
+        groups = set()
+        packages = set()
+
+        # Check if there is a variant. The method only makes sense for variants.
+        if variant is None:
+            return packages, groups
+
+        compatible_arches = pungi.arch.get_compatible_arches(arch, multilib=True)
+
+        for nsvc, module_stream in variant.arch_mmds.get(arch, {}).items():
+            available_rpms = sum(
+                (
+                    variant.nsvc_to_pkgset[nsvc].rpms_by_arch.get(a, [])
+                    for a in compatible_arches
+                ),
+                [],
+            )
+            to_include = set(module_stream.get_rpm_artifacts())
+            for rpm_obj in available_rpms:
+                if rpm_obj.nevra in to_include:
+                    packages.add((rpm_obj, None))
+
+        return packages, groups

pungi/phases/gather/sources/source_none.py

@@ -0,0 +1,33 @@
+# -*- coding: utf-8 -*-
+
+
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; version 2 of the License.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Library General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, see <https://gnu.org/licenses/>.
+
+
+"""
+Get an empty package list.
+
+Input:
+none
+
+Output:
+set()
+"""
+
+
+import pungi.phases.gather.source
+
+
+class GatherSourceNone(pungi.phases.gather.source.GatherSourceBase):
+    def __call__(self, arch, variant):
+        return set(), set()

pungi/phases/image_build.py

@@ -0,0 +1,526 @@
+# -*- coding: utf-8 -*-
+
+import copy
+import hashlib
+import json
+import os
+import shutil
+import time
+from kobo import shortcuts
+
+from pungi.util import makedirs, get_mtime, get_file_size, failable, log_failed_task
+from pungi.util import as_local_file, translate_path, get_repo_urls, version_generator
+from pungi.phases import base
+from pungi.linker import Linker
+from pungi.wrappers.kojiwrapper import KojiWrapper
+from kobo.threads import ThreadPool, WorkerThread
+from kobo.shortcuts import force_list
+from productmd.images import Image
+from productmd.rpms import Rpms
+
+
+# This is a mapping from formats to file extensions. The format is what koji
+# image-build command expects as argument, and the extension is what the file
+# name will be ending with. The extensions are used to filter out which task
+# results will be pulled into the compose.
+EXTENSIONS = {
+    "docker": ["tar.gz", "tar.xz"],
+    "iso": ["iso"],
+    "liveimg-squashfs": ["liveimg.squashfs"],
+    "qcow": ["qcow"],
+    "qcow2": ["qcow2"],
+    "raw": ["raw"],
+    "raw-xz": ["raw.xz"],
+    "rhevm-ova": ["rhevm.ova"],
+    "tar-gz": ["tar.gz"],
+    "vagrant-hyperv": ["vagrant-hyperv.box"],
+    "vagrant-libvirt": ["vagrant-libvirt.box"],
+    "vagrant-virtualbox": ["vagrant-virtualbox.box"],
+    "vagrant-vmware-fusion": ["vagrant-vmware-fusion.box"],
+    "vdi": ["vdi"],
+    "vmdk": ["vmdk"],
+    "vpc": ["vhd"],
+    "vhd-compressed": ["vhd.gz", "vhd.xz"],
+    "vsphere-ova": ["vsphere.ova"],
+}
+
+
+class ImageBuildPhase(
+    base.PhaseLoggerMixin, base.ImageConfigMixin, base.ConfigGuardedPhase
+):
+    """class for wrapping up koji image-build"""
+
+    name = "image_build"
+
+    def __init__(self, compose, buildinstall_phase=None):
+        super(ImageBuildPhase, self).__init__(compose)
+        self.pool = ThreadPool(logger=self.logger)
+        self.buildinstall_phase = buildinstall_phase
+
+    def _get_install_tree(self, image_conf, variant):
+        """
+        Get a path to os tree for a variant specified in `install_tree_from` or
+        current variant. If the config is set, it will be removed from the
+        dict.
+        """
+        if variant.type != "variant":
+            # Buildinstall only runs for top-level variants. Nested variants
+            # need to re-use install tree from parent.
+            variant = variant.parent
+
+        install_tree_from = image_conf.pop("install_tree_from", variant.uid)
+        if "://" in install_tree_from:
+            # It's a URL, return it unchanged
+            return install_tree_from
+        if install_tree_from.startswith("/"):
+            # It's a path on local filesystem.
+            return translate_path(self.compose, install_tree_from)
+
+        install_tree_source = self.compose.all_variants.get(install_tree_from)
+        if not install_tree_source:
+            raise RuntimeError(
+                "There is no variant %s to get install tree from "
+                "when building image for %s." % (install_tree_from, variant.uid)
+            )
+        return translate_path(
+            self.compose,
+            self.compose.paths.compose.os_tree(
+                "$arch", install_tree_source, create_dir=False
+            ),
+        )
+
+    def _get_repo(self, image_conf, variant):
+        """
+        Get a comma separated list of repos. First included are those
+        explicitly listed in config, followed by by repo for current variant
+        if it's not included in the list already.
+        """
+        repos = shortcuts.force_list(image_conf.get("repo", []))
+
+        if not variant.is_empty and variant.uid not in repos:
+            repos.append(variant.uid)
+
+        return ",".join(get_repo_urls(self.compose, repos, arch="$arch"))
+
+    def _get_arches(self, image_conf, arches):
+        if "arches" in image_conf["image-build"]:
+            arches = set(image_conf["image-build"].get("arches", [])) & arches
+        return sorted(arches)
+
+    def _set_release(self, image_conf):
+        """If release is set explicitly to None, replace it with date and respin."""
+        if "release" in image_conf:
+            image_conf["release"] = (
+                version_generator(self.compose, image_conf["release"])
+                or self.compose.image_release
+            )
+
+    def run(self):
+        for variant in self.compose.get_variants():
+            arches = set([x for x in variant.arches if x != "src"])
+
+            for image_conf in self.get_config_block(variant):
+                # We will modify the data, so we need to make a copy to
+                # prevent problems in next iteration where the original
+                # value is needed.
+                image_conf = copy.deepcopy(image_conf)
+                original_image_conf = copy.deepcopy(image_conf)
+
+                # image_conf is passed to get_image_build_cmd as dict
+
+                image_conf["image-build"]["arches"] = self._get_arches(
+                    image_conf, arches
+                )
+                if not image_conf["image-build"]["arches"]:
+                    continue
+
+                # Replace possible ambiguous ref name with explicit hash.
+                ksurl = self.get_ksurl(image_conf["image-build"])
+                if ksurl:
+                    image_conf["image-build"]["ksurl"] = ksurl
+
+                image_conf["image-build"]["variant"] = variant
+
+                image_conf["image-build"]["install_tree"] = self._get_install_tree(
+                    image_conf["image-build"], variant
+                )
+
+                release = self.get_release(image_conf["image-build"])
+                if release:
+                    image_conf["image-build"]["release"] = release
+
+                image_conf["image-build"]["version"] = self.get_version(
+                    image_conf["image-build"]
+                )
+                image_conf["image-build"]["target"] = self.get_config(
+                    image_conf["image-build"], "target"
+                )
+
+                # Pungi config can either contain old [(format, suffix)], or
+                # just list of formats, or a single format.
+                formats = []
+                for format in force_list(image_conf["image-build"]["format"]):
+                    formats.append(
+                        format[0] if isinstance(format, (tuple, list)) else format
+                    )
+                image_conf["image-build"]["format"] = formats
+                image_conf["image-build"]["repo"] = self._get_repo(
+                    image_conf["image-build"], variant
+                )
+
+                can_fail = image_conf["image-build"].pop("failable", [])
+                if can_fail == ["*"]:
+                    can_fail = image_conf["image-build"]["arches"]
+                if can_fail:
+                    image_conf["image-build"]["can_fail"] = sorted(can_fail)
+
+                cmd = {
+                    "original_image_conf": original_image_conf,
+                    "image_conf": image_conf,
+                    "conf_file": self.compose.paths.work.image_build_conf(
+                        image_conf["image-build"]["variant"],
+                        image_name=image_conf["image-build"]["name"],
+                        image_type="-".join(formats),
+                        arches=image_conf["image-build"]["arches"],
+                    ),
+                    "image_dir": self.compose.paths.compose.image_dir(variant),
+                    "relative_image_dir": self.compose.paths.compose.image_dir(
+                        variant, relative=True
+                    ),
+                    "link_type": self.compose.conf["link_type"],
+                    "scratch": image_conf["image-build"].pop("scratch", False),
+                }
+                self.pool.add(CreateImageBuildThread(self.pool))
+                self.pool.queue_put((self.compose, cmd, self.buildinstall_phase))
+
+        self.pool.start()
+
+
+class CreateImageBuildThread(WorkerThread):
+    def fail(self, compose, cmd):
+        self.pool.log_error("CreateImageBuild failed.")
+
+    def process(self, item, num):
+        compose, cmd, buildinstall_phase = item
+        variant = cmd["image_conf"]["image-build"]["variant"]
+        subvariant = cmd["image_conf"]["image-build"].get("subvariant", variant.uid)
+        self.failable_arches = cmd["image_conf"]["image-build"].get("can_fail", "")
+        self.can_fail = (
+            self.failable_arches == cmd["image_conf"]["image-build"]["arches"]
+        )
+        with failable(
+            compose,
+            self.can_fail,
+            variant,
+            "*",
+            "image-build",
+            subvariant,
+            logger=self.pool._logger,
+        ):
+            self.worker(num, compose, variant, subvariant, cmd, buildinstall_phase)
+
+    def worker(self, num, compose, variant, subvariant, cmd, buildinstall_phase):
+        arches = cmd["image_conf"]["image-build"]["arches"]
+        formats = "-".join(cmd["image_conf"]["image-build"]["format"])
+        dash_arches = "-".join(arches)
+        log_file = compose.paths.log.log_file(
+            dash_arches, "imagebuild-%s-%s-%s" % (variant.uid, subvariant, formats)
+        )
+        metadata_file = log_file[:-4] + ".reuse.json"
+
+        external_repo_checksum = {}
+        try:
+            for repo in cmd["original_image_conf"]["image-build"]["repo"]:
+                if repo in compose.all_variants:
+                    continue
+                with as_local_file(
+                    os.path.join(repo, "repodata/repomd.xml")
+                ) as filename:
+                    with open(filename, "rb") as f:
+                        external_repo_checksum[repo] = hashlib.sha256(
+                            f.read()
+                        ).hexdigest()
+        except Exception as e:
+            external_repo_checksum = None
+            self.pool.log_info(
+                "Can't calculate checksum of repomd.xml of external repo - %s" % str(e)
+            )
+
+        if self._try_to_reuse(
+            compose,
+            variant,
+            subvariant,
+            metadata_file,
+            log_file,
+            cmd,
+            external_repo_checksum,
+            buildinstall_phase,
+        ):
+            return
+
+        msg = (
+            "Creating image (formats: %s, arches: %s, variant: %s, subvariant: %s)"
+            % (formats, dash_arches, variant, subvariant)
+        )
+        self.pool.log_info("[BEGIN] %s" % msg)
+
+        koji_wrapper = KojiWrapper(compose)
+
+        # writes conf file for koji image-build
+        self.pool.log_info(
+            "Writing image-build config for %s.%s into %s"
+            % (variant, dash_arches, cmd["conf_file"])
+        )
+
+        koji_cmd = koji_wrapper.get_image_build_cmd(
+            cmd["image_conf"], conf_file_dest=cmd["conf_file"], scratch=cmd["scratch"]
+        )
+
+        # avoid race conditions?
+        # Kerberos authentication failed:
+        #   Permission denied in replay cache code (-1765328215)
+        # [workaround] Increased time delay from 3 to 10 sec until the issue in
+        # koji gets fixed https://pagure.io/koji/issue/2138
+        time.sleep(num * 10)
+        output = koji_wrapper.run_blocking_cmd(koji_cmd, log_file=log_file)
+        self.pool.log_debug("build-image outputs: %s" % (output))
+        if output["retcode"] != 0:
+            self.fail(compose, cmd)
+            raise RuntimeError(
+                "ImageBuild task failed: %s. See %s for more details."
+                % (output["task_id"], log_file)
+            )
+
+        # copy image to images/
+        image_infos = []
+
+        paths = koji_wrapper.get_image_paths(
+            output["task_id"],
+            callback=lambda arch: log_failed_task(
+                compose, variant, arch, "image-build", subvariant
+            ),
+        )
+
+        for arch, paths in paths.items():
+            for path in paths:
+                for format in cmd["image_conf"]["image-build"]["format"]:
+                    for suffix in EXTENSIONS[format]:
+                        if path.endswith(suffix):
+                            image_infos.append(
+                                {
+                                    "path": path,
+                                    "suffix": suffix,
+                                    "type": format,
+                                    "arch": arch,
+                                }
+                            )
+                            break
+
+        self._link_images(compose, variant, subvariant, cmd, image_infos)
+        self._write_reuse_metadata(
+            compose, metadata_file, cmd, image_infos, external_repo_checksum
+        )
+
+        self.pool.log_info("[DONE ] %s (task id: %s)" % (msg, output["task_id"]))
+
+    def _link_images(self, compose, variant, subvariant, cmd, image_infos):
+        """Link images to compose and update image manifest.
+
+        :param Compose compose: Current compose.
+        :param Variant variant: Current variant.
+        :param str subvariant:
+        :param dict cmd: Dict of params for image-build.
+        :param dict image_infos: Dict contains image info.
+        """
+        # The usecase here is that you can run koji image-build with multiple --format
+        # It's ok to do it serialized since we're talking about max 2 images per single
+        # image_build record
+        linker = Linker(logger=self.pool._logger)
+        for image_info in image_infos:
+            image_dir = cmd["image_dir"] % {"arch": image_info["arch"]}
+            makedirs(image_dir)
+            relative_image_dir = cmd["relative_image_dir"] % {
+                "arch": image_info["arch"]
+            }
+
+            # let's not change filename of koji outputs
+            image_dest = os.path.join(image_dir, os.path.basename(image_info["path"]))
+
+            src_file = compose.koji_downloader.get_file(
+                os.path.realpath(image_info["path"])
+            )
+            linker.link(src_file, image_dest, link_type=cmd["link_type"])
+
+            # Update image manifest
+            img = Image(compose.im)
+            img.type = image_info["type"]
+            img.format = image_info["suffix"]
+            img.path = os.path.join(relative_image_dir, os.path.basename(image_dest))
+            img.mtime = get_mtime(image_dest)
+            img.size = get_file_size(image_dest)
+            img.arch = image_info["arch"]
+            img.disc_number = 1  # We don't expect multiple disks
+            img.disc_count = 1
+            img.bootable = False
+            img.subvariant = subvariant
+            setattr(img, "can_fail", self.can_fail)
+            setattr(img, "deliverable", "image-build")
+            compose.im.add(variant=variant.uid, arch=image_info["arch"], image=img)
+
+    def _try_to_reuse(
+        self,
+        compose,
+        variant,
+        subvariant,
+        metadata_file,
+        log_file,
+        cmd,
+        external_repo_checksum,
+        buildinstall_phase,
+    ):
+        """Try to reuse images from old compose.
+
+        :param Compose compose: Current compose.
+        :param Variant variant: Current variant.
+        :param str subvariant:
+        :param str metadata_file: Path to reuse metadata file.
+        :param str log_file: Path to log file.
+        :param dict cmd: Dict of params for image-build.
+        :param dict external_repo_checksum: Dict contains checksum of repomd.xml
+            or None if can't get checksum.
+        :param BuildinstallPhase buildinstall_phase: buildinstall phase of
+            current compose.
+        """
+        log_msg = "Cannot reuse old image_build phase results - %s"
+        if not compose.conf["image_build_allow_reuse"]:
+            self.pool.log_info(
+                log_msg % "reuse of old image_build results is disabled."
+            )
+            return False
+
+        if external_repo_checksum is None:
+            self.pool.log_info(
+                log_msg % "Can't ensure that external repo is not changed."
+            )
+            return False
+
+        old_metadata_file = compose.paths.old_compose_path(metadata_file)
+        if not old_metadata_file:
+            self.pool.log_info(log_msg % "Can't find old reuse metadata file")
+            return False
+
+        try:
+            old_metadata = self._load_reuse_metadata(old_metadata_file)
+        except Exception as e:
+            self.pool.log_info(
+                log_msg % "Can't load old reuse metadata file: %s" % str(e)
+            )
+            return False
+
+        if old_metadata["cmd"]["original_image_conf"] != cmd["original_image_conf"]:
+            self.pool.log_info(log_msg % "image_build config changed")
+            return False
+
+        # Make sure external repo does not change
+        if (
+            old_metadata["external_repo_checksum"] is None
+            or old_metadata["external_repo_checksum"] != external_repo_checksum
+        ):
+            self.pool.log_info(log_msg % "External repo may be changed")
+            return False
+
+        # Make sure buildinstall phase is reused
+        for arch in cmd["image_conf"]["image-build"]["arches"]:
+            if buildinstall_phase and not buildinstall_phase.reused(variant, arch):
+                self.pool.log_info(log_msg % "buildinstall phase changed")
+                return False
+
+        # Make sure packages in variant not change
+        rpm_manifest_file = compose.paths.compose.metadata("rpms.json")
+        rpm_manifest = Rpms()
+        rpm_manifest.load(rpm_manifest_file)
+
+        old_rpm_manifest_file = compose.paths.old_compose_path(rpm_manifest_file)
+        old_rpm_manifest = Rpms()
+        old_rpm_manifest.load(old_rpm_manifest_file)
+
+        for repo in cmd["original_image_conf"]["image-build"]["repo"]:
+            if repo not in compose.all_variants:
+                # External repos are checked using other logic.
+                continue
+            for arch in cmd["image_conf"]["image-build"]["arches"]:
+                if (
+                    rpm_manifest.rpms[variant.uid][arch]
+                    != old_rpm_manifest.rpms[variant.uid][arch]
+                ):
+                    self.pool.log_info(
+                        log_msg % "Packages in %s.%s changed." % (variant.uid, arch)
+                    )
+                    return False
+
+        self.pool.log_info(
+            "Reusing images from old compose for variant %s" % variant.uid
+        )
+        try:
+            self._link_images(
+                compose, variant, subvariant, cmd, old_metadata["image_infos"]
+            )
+        except Exception as e:
+            self.pool.log_info(log_msg % "Can't link images %s" % str(e))
+            return False
+
+        old_log_file = compose.paths.old_compose_path(log_file)
+        try:
+            shutil.copy2(old_log_file, log_file)
+        except Exception as e:
+            self.pool.log_info(
+                log_msg % "Can't copy old log_file: %s %s" % (old_log_file, str(e))
+            )
+            return False
+
+        self._write_reuse_metadata(
+            compose,
+            metadata_file,
+            cmd,
+            old_metadata["image_infos"],
+            external_repo_checksum,
+        )
+
+        return True
+
+    def _write_reuse_metadata(
+        self, compose, metadata_file, cmd, image_infos, external_repo_checksum
+    ):
+        """Write metadata file.
+
+        :param Compose compose: Current compose.
+        :param str metadata_file: Path to reuse metadata file.
+        :param dict cmd: Dict of params for image-build.
+        :param dict image_infos: Dict contains image info.
+        :param dict external_repo_checksum: Dict contains checksum of repomd.xml
+            or None if can't get checksum.
+        """
+        msg = "Writing reuse metadata file: %s" % metadata_file
+        self.pool.log_info(msg)
+
+        cmd_copy = copy.deepcopy(cmd)
+        del cmd_copy["image_conf"]["image-build"]["variant"]
+
+        data = {
+            "cmd": cmd_copy,
+            "image_infos": image_infos,
+            "external_repo_checksum": external_repo_checksum,
+        }
+        try:
+            with open(metadata_file, "w") as f:
+                json.dump(data, f, indent=4)
+        except Exception as e:
+            self.pool.log_info("%s Failed: %s" % (msg, str(e)))
+
+    def _load_reuse_metadata(self, metadata_file):
+        """Load metadata file.
+
+        :param str metadata_file: Path to reuse metadata file.
+        """
+        with open(metadata_file, "r") as f:
+            return json.load(f)

pungi/phases/image_checksum.py

@@ -0,0 +1,206 @@
+# -*- coding: utf-8 -*-
+
+import os
+from kobo import shortcuts
+from collections import defaultdict
+import threading
+
+from .base import PhaseBase
+from ..util import get_format_substs, get_file_size
+
+
+MULTIPLE_CHECKSUMS_ERROR = (
+    'Config option "media_checksum_one_file" requires only one checksum'
+    ' to be configured in "media_checksums".'
+)
+
+
+class ImageChecksumPhase(PhaseBase):
+    """Go through images specified in image manifest and generate their
+    checksums. The manifest will be updated with the checksums.
+    """
+
+    name = "image_checksum"
+
+    def __init__(self, compose):
+        super(ImageChecksumPhase, self).__init__(compose)
+        self.checksums = self.compose.conf["media_checksums"]
+        self.one_file = self.compose.conf["media_checksum_one_file"]
+
+    def skip(self):
+        # Skipping this phase does not make sense:
+        #  * if there are no images, it doesn't do anything and is quick
+        #  * if there are images, they must have checksums computed or else
+        #    writing metadata will fail
+        return False
+
+    def validate(self):
+        errors = []
+
+        if self.one_file and len(self.checksums) != 1:
+            errors.append(MULTIPLE_CHECKSUMS_ERROR)
+
+        if errors:
+            raise ValueError("\n".join(errors))
+
+    def _get_images(self):
+        """Returns a mapping from directories to sets of ``Image``s.
+
+        The paths to dirs are absolute.
+        """
+        top_dir = self.compose.paths.compose.topdir()
+        images = {}
+        for variant in self.compose.im.images:
+            for arch in self.compose.im.images[variant]:
+                for image in self.compose.im.images[variant][arch]:
+                    path = os.path.dirname(os.path.join(top_dir, image.path))
+                    images.setdefault((variant, arch, path), set()).add(image)
+        return images
+
+    def _get_base_filename(self, variant, arch, **kwargs):
+        base_checksum_name = self.compose.conf["media_checksum_base_filename"]
+        if base_checksum_name:
+            substs = get_format_substs(
+                self.compose, variant=variant, arch=arch, **kwargs
+            )
+            base_checksum_name = (base_checksum_name % substs).format(**substs)
+            base_checksum_name += "-"
+        return base_checksum_name
+
+    def run(self):
+        topdir = self.compose.paths.compose.topdir()
+
+        make_checksums(
+            topdir,
+            self.compose.im,
+            self.checksums,
+            self.one_file,
+            self._get_base_filename,
+        )
+
+
+def _compute_checksums(
+    results,
+    cache,
+    variant,
+    arch,
+    path,
+    images,
+    checksum_types,
+    base_checksum_name_gen,
+    one_file,
+    results_lock,
+    cache_lock,
+):
+    for image in images:
+        filename = os.path.basename(image.path)
+        full_path = os.path.join(path, filename)
+        if not os.path.exists(full_path):
+            continue
+
+        filesize = image.size or get_file_size(full_path)
+
+        cache_lock.acquire()
+        if full_path not in cache:
+            cache_lock.release()
+            # Source ISO is listed under each binary architecture. There's no
+            # point in checksumming it twice, so we can just remember the
+            # digest from first run..
+            checksum_value = shortcuts.compute_file_checksums(full_path, checksum_types)
+            with cache_lock:
+                cache[full_path] = checksum_value
+        else:
+            cache_lock.release()
+
+        with cache_lock:
+            digests = cache[full_path]
+
+        for checksum, digest in digests.items():
+            # Update metadata with the checksum
+            image.add_checksum(None, checksum, digest)
+            # If not turned of, create the file-specific checksum file
+            if not one_file:
+                checksum_filename = os.path.join(
+                    path, "%s.%sSUM" % (filename, checksum.upper())
+                )
+                with results_lock:
+                    results[checksum_filename].add(
+                        (filename, filesize, checksum, digest)
+                    )
+
+            if one_file:
+                dirname = os.path.basename(path)
+                base_checksum_name = base_checksum_name_gen(
+                    variant, arch, dirname=dirname
+                )
+                checksum_filename = base_checksum_name + "CHECKSUM"
+            else:
+                base_checksum_name = base_checksum_name_gen(variant, arch)
+                checksum_filename = "%s%sSUM" % (base_checksum_name, checksum.upper())
+            checksum_path = os.path.join(path, checksum_filename)
+
+            with results_lock:
+                results[checksum_path].add((filename, filesize, checksum, digest))
+
+
+def make_checksums(topdir, im, checksum_types, one_file, base_checksum_name_gen):
+    results = defaultdict(set)
+    cache = {}
+    threads = []
+    results_lock = threading.Lock()  # lock to synchronize access to the results dict.
+    cache_lock = threading.Lock()  # lock to synchronize access to the cache dict.
+
+    # create all worker threads
+    for (variant, arch, path), images in get_images(topdir, im).items():
+        threads.append(
+            threading.Thread(
+                target=_compute_checksums,
+                args=[
+                    results,
+                    cache,
+                    variant,
+                    arch,
+                    path,
+                    images,
+                    checksum_types,
+                    base_checksum_name_gen,
+                    one_file,
+                    results_lock,
+                    cache_lock,
+                ],
+            )
+        )
+        threads[-1].start()
+
+    # wait for all worker threads to finish
+    for thread in threads:
+        thread.join()
+
+    for file in results:
+        dump_checksums(file, results[file])
+
+
+def dump_checksums(checksum_file, data):
+    """Write checksums to file.
+
+    :param checksum_file: where to write the checksums
+    :param data: an iterable of tuples (filename, filesize, checksum_type, hash)
+    """
+    with open(checksum_file, "w") as f:
+        for filename, filesize, alg, checksum in sorted(data):
+            f.write("# %s: %s bytes\n" % (filename, filesize))
+            f.write("%s (%s) = %s\n" % (alg.upper(), filename, checksum))
+
+
+def get_images(top_dir, manifest):
+    """Returns a mapping from directories to sets of ``Image``s.
+
+    The paths to dirs are absolute.
+    """
+    images = {}
+    for variant in manifest.images:
+        for arch in manifest.images[variant]:
+            for image in manifest.images[variant][arch]:
+                path = os.path.dirname(os.path.join(top_dir, image.path))
+                images.setdefault((variant, arch, path), []).append(image)
+    return images