Move the package requirements for live-iso setup out of the template

In order to support iso creation on multiple arches with the templates
we need to be able to select different packages based on arch.
lorax-composer uses the arch-specific Lorax templates in order to
generate the output iso so this patch:

1. Creates a new template and type to parse it, live-install.tmpl
   which contains only installpkg commands and #if clauses for arch
2. Removes bootloader related packages from the live-iso.ks
3. Remove dracut-config-rescue exclusion because it can cause problems
   with some blueprints.
4. Switch logo requirement to system-logos which is satisfied by
   generic-logos or fedora-logos. This prevents conflicts when a blueprint
   installs fedora-release-workstation.

So in the future, if x86.tmpl, etc. need a new package to support
creating the iso it should be added to the correct section in
./share/live/live-install.tmpl

share/composer/live-iso.ks

@@ -351,22 +351,9 @@ EOF
 # Packages requires to support this output format go here
 isomd5sum
 kernel
-memtest86+
-syslinux
--dracut-config-rescue
 dracut-config-generic
 dracut-live
-generic-logos
+system-logos
 selinux-policy-targeted
 
-# This package is needed to boot the iso on UEFI
-shim
-shim-ia32
-grub2
-grub2-efi
-grub2-efi-*-cdboot
-grub2-efi-ia32
-efibootmgr
-
-
 # NOTE lorax-composer will add the blueprint packages below here, including the final %end%packages

share/templates.d/99-generic/live/live-install.tmpl

@@ -0,0 +1,32 @@
+## livemedia-creator: Install packages needed for iso creation using per-arch templates
+<%page args="basearch"/>
+
+## arch-specific bootloader packages
+%if basearch == "aarch64":
+    installpkg efibootmgr
+    installpkg grub2-efi-aa64-cdboot shim-aa64
+    installpkg uboot-tools
+%endif
+%if basearch in ("arm", "armhfp"):
+    installpkg efibootmgr
+    installpkg grub2-efi-arm-cdboot
+    installpkg uboot-tools
+%endif
+%if basearch == "x86_64":
+    installpkg grub2-tools-efi
+    installpkg efibootmgr
+    installpkg shim-x64 grub2-efi-x64-cdboot
+    installpkg shim-ia32 grub2-efi-ia32-cdboot
+%endif
+%if basearch in ("i386", "x86_64"):
+    installpkg biosdevname memtest86+ syslinux
+    installpkg grub2-tools grub2-tools-minimal grub2-tools-extra
+%endif
+%if basearch in ("ppc64le"):
+    installpkg powerpc-utils
+    installpkg grub2-tools grub2-tools-minimal grub2-tools-extra
+    installpkg grub2-${basearch}
+%endif
+%if basearch == "s390x":
+    installpkg s390utils-base
+%endif

src/pylorax/api/compose.py

@@ -44,11 +44,14 @@ from uuid import uuid4
 from pykickstart.parser import KickstartParser
 from pykickstart.version import makeVersion
 
+from pylorax import ArchData, find_templates, get_buildarch
 from pylorax.api.projects import projects_depsolve, projects_depsolve_with_size, dep_nevra
 from pylorax.api.projects import ProjectsError
 from pylorax.api.recipes import read_recipe_and_id
 from pylorax.api.timestamp import TS_CREATED, write_timestamp
+from pylorax.base import DataHolder
 from pylorax.imgutils import default_image_name
+from pylorax.ltmpl import LiveTemplateRunner
 from pylorax.sysutils import joinpaths, flatconfig
 
 
@@ -283,6 +286,40 @@ def add_customizations(f, recipe):
     if not wrote_rootpw:
         f.write('rootpw --lock\n')
 
+
+def get_extra_pkgs(dbo, share_dir, compose_type):
+    """Return extra packages needed for the output type
+
+    :param dbo: dnf base object
+    :type dbo: dnf.Base
+    :param share_dir: Path to the top level share directory
+    :type share_dir: str
+    :param compose_type: The type of output to create from the recipe
+    :type compose_type: str
+    :returns: List of package names (name only, not NEVRA)
+    :rtype: list
+
+    Currently this is only needed by live-iso, it reads ./live/live-install.tmpl and
+    processes only the installpkg lines. It lists the packages needed to complete creation of the
+    iso using the templates such as x86.tmpl
+
+    Keep in mind that the live-install.tmpl is shared between livemedia-creator and lorax-composer,
+    even though the results are applied differently.
+    """
+    if compose_type != "live-iso":
+        return []
+
+    # get the arch information to pass to the runner
+    arch = ArchData(get_buildarch(dbo))
+    defaults = DataHolder(basearch=arch.basearch)
+    templatedir = joinpaths(find_templates(share_dir), "live")
+    runner = LiveTemplateRunner(dbo, templatedir=templatedir, defaults=defaults)
+    runner.run("live-install.tmpl")
+    log.debug("extra pkgs = %s", runner.pkgs)
+
+    return runner.pkgnames
+
+
 def start_build(cfg, dnflock, gitlock, branch, recipe_name, compose_type, test_mode=0):
     """ Start the build
 
@@ -304,15 +341,22 @@ def start_build(cfg, dnflock, gitlock, branch, recipe_name, compose_type, test_m
     if compose_type not in compose_types(share_dir):
         raise RuntimeError("Invalid compose type (%s), must be one of %s" % (compose_type, compose_types(share_dir)))
 
+    # Some image types (live-iso) need extra packages for composer to execute the output template
+    with dnflock.lock:
+        extra_pkgs = get_extra_pkgs(dnflock.dbo, share_dir, compose_type)
+    log.debug("Extra packages needed for %s: %s", compose_type, extra_pkgs)
+
     with gitlock.lock:
         (commit_id, recipe) = read_recipe_and_id(gitlock.repo, branch, recipe_name)
 
     # Combine modules and packages and depsolve the list
-    # TODO include the version/glob in the depsolving
     module_nver = recipe.module_nver
     package_nver = recipe.package_nver
+    package_nver.extend([(name, '*') for name in extra_pkgs])
+
     projects = sorted(set(module_nver+package_nver), key=lambda p: p[0].lower())
     deps = []
+    log.info("depsolving %s", recipe["name"])
     try:
         # This can possibly update repodata and reset the YumBase object.
         with dnflock.lock_check:

src/pylorax/ltmpl.py

@@ -811,6 +811,7 @@ class LiveTemplateRunner(TemplateRunner):
     def __init__(self, dbo, fatalerrors=True, templatedir=None, defaults=None):
         self.dbo = dbo
         self.pkgs = []
+        self.pkgnames = []
 
         super(LiveTemplateRunner, self).__init__(fatalerrors, templatedir, defaults)
 
@@ -871,6 +872,7 @@ class LiveTemplateRunner(TemplateRunner):
                     logger.info("installpkg: %s expands to %s", p, ",".join(pkgnvrs))
 
                 self.pkgs.extend(pkgnvrs)
+                self.pkgnames.extend([pkg.name for pkg in pkgnames])
             except Exception as e: # pylint: disable=broad-except
                 logger.error("installpkg %s failed: %s", p, str(e))
                 errors = True

tests/pylorax/test_compose.py

@@ -15,9 +15,15 @@
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 #
 from io import StringIO
+import os
+import shutil
+import tempfile
 import unittest
 
-from pylorax.api.compose import add_customizations
+from pylorax import get_buildarch
+from pylorax.api.compose import add_customizations, get_extra_pkgs
+from pylorax.api.config import configure, make_dnf_dirs
+from pylorax.api.dnfbase import get_base_object
 from pylorax.api.recipes import recipe_from_toml
 
 BASE_RECIPE = """name = "test-cases"
@@ -223,3 +229,41 @@ class CustomizationsTestCase(unittest.TestCase):
         self.assertCustomization(ROOT_PLAIN_KEY, 'rootpw --plaintext "plainpassword"')
         self.assertCustomization(ROOT_PLAIN_KEY, 'sshkey --user root "A SSH KEY FOR THE USER"')
         self.assertNotCustomization(ROOT_PLAIN_KEY, "rootpw --lock")
+
+
+class ExtraPkgsTest(unittest.TestCase):
+    @classmethod
+    def setUpClass(self):
+        self.tmp_dir = tempfile.mkdtemp(prefix="lorax.test.repo.")
+        self.config = configure(root_dir=self.tmp_dir, test_config=True)
+        make_dnf_dirs(self.config, os.getuid(), os.getgid())
+        self.dbo = get_base_object(self.config)
+
+    @classmethod
+    def tearDownClass(self):
+        shutil.rmtree(self.tmp_dir)
+
+    def test_live_install(self):
+        """Check that live-install.tmpl is parsed correctly"""
+        # A package for each arch to test for
+        arch_pkg = {
+            "aarch64":  "shim-aa64",
+            "arm":      "grub2-efi-arm-cdboot",
+            "armhfp":   "grub2-efi-arm-cdboot",
+            "x86_64":   "shim-x64",
+            "i386":     "memtest86+",
+            "ppc64le":  "powerpc-utils",
+            "s390x":    "s390utils-base"
+        }
+
+        extra_pkgs = get_extra_pkgs(self.dbo, "./share/", "live-iso")
+        self.assertTrue(len(extra_pkgs) > 0)
+
+        # Results depend on arch
+        arch = get_buildarch(self.dbo)
+        self.assertTrue(arch_pkg[arch] in extra_pkgs)
+
+    def test_other_install(self):
+        """Test that non-live doesn't parse live-install.tmpl"""
+        extra_pkgs = get_extra_pkgs(self.dbo, "./share/", "qcow2")
+        self.assertEqual(extra_pkgs, [])