From 1fd73cde52af86086d9fa88b7c1ca5119801f707 Mon Sep 17 00:00:00 2001 From: "Brian C. Lane" Date: Mon, 18 Feb 2019 16:56:26 -0800 Subject: [PATCH] Move the package requirements for live-iso setup out of the template In order to support iso creation on multiple arches with the templates we need to be able to select different packages based on arch. lorax-composer uses the arch-specific Lorax templates in order to generate the output iso so this patch: 1. Creates a new template and type to parse it, live-install.tmpl which contains only installpkg commands and #if clauses for arch 2. Removes bootloader related packages from the live-iso.ks 3. Remove dracut-config-rescue exclusion because it can cause problems with some blueprints. 4. Switch logo requirement to system-logos which is satisfied by generic-logos or fedora-logos. This prevents conflicts when a blueprint installs fedora-release-workstation. So in the future, if x86.tmpl, etc. need a new package to support creating the iso it should be added to the correct section in ./share/live/live-install.tmpl (cherry picked from commit 3b8de2a2331360eb2f795e4333389d81f938a14c) --- share/composer/live-iso.ks | 15 +----- .../99-generic/live/live-install.tmpl | 32 +++++++++++++ src/pylorax/api/compose.py | 46 ++++++++++++++++++- src/pylorax/ltmpl.py | 2 + tests/pylorax/test_compose.py | 46 ++++++++++++++++++- 5 files changed, 125 insertions(+), 16 deletions(-) create mode 100644 share/templates.d/99-generic/live/live-install.tmpl diff --git a/share/composer/live-iso.ks b/share/composer/live-iso.ks index fada4f77..a88e8c43 100644 --- a/share/composer/live-iso.ks +++ b/share/composer/live-iso.ks @@ -351,22 +351,9 @@ EOF # Packages requires to support this output format go here isomd5sum kernel -memtest86+ -syslinux --dracut-config-rescue dracut-config-generic dracut-live -generic-logos +system-logos selinux-policy-targeted -# This package is needed to boot the iso on UEFI -shim -shim-ia32 -grub2 -grub2-efi -grub2-efi-*-cdboot -grub2-efi-ia32 -efibootmgr - - # NOTE lorax-composer will add the blueprint packages below here, including the final %end%packages diff --git a/share/templates.d/99-generic/live/live-install.tmpl b/share/templates.d/99-generic/live/live-install.tmpl new file mode 100644 index 00000000..88abff59 --- /dev/null +++ b/share/templates.d/99-generic/live/live-install.tmpl @@ -0,0 +1,32 @@ +## livemedia-creator: Install packages needed for iso creation using per-arch templates +<%page args="basearch"/> + +## arch-specific bootloader packages +%if basearch == "aarch64": + installpkg efibootmgr + installpkg grub2-efi-aa64-cdboot shim-aa64 + installpkg uboot-tools +%endif +%if basearch in ("arm", "armhfp"): + installpkg efibootmgr + installpkg grub2-efi-arm-cdboot + installpkg uboot-tools +%endif +%if basearch == "x86_64": + installpkg grub2-tools-efi + installpkg efibootmgr + installpkg shim-x64 grub2-efi-x64-cdboot + installpkg shim-ia32 grub2-efi-ia32-cdboot +%endif +%if basearch in ("i386", "x86_64"): + installpkg biosdevname memtest86+ syslinux + installpkg grub2-tools grub2-tools-minimal grub2-tools-extra +%endif +%if basearch in ("ppc64le"): + installpkg powerpc-utils + installpkg grub2-tools grub2-tools-minimal grub2-tools-extra + installpkg grub2-${basearch} +%endif +%if basearch == "s390x": + installpkg s390utils-base +%endif diff --git a/src/pylorax/api/compose.py b/src/pylorax/api/compose.py index 9b1e119c..29b9f43e 100644 --- a/src/pylorax/api/compose.py +++ b/src/pylorax/api/compose.py @@ -44,11 +44,14 @@ from uuid import uuid4 from pykickstart.parser import KickstartParser from pykickstart.version import makeVersion +from pylorax import ArchData, find_templates, get_buildarch from pylorax.api.projects import projects_depsolve, projects_depsolve_with_size, dep_nevra from pylorax.api.projects import ProjectsError from pylorax.api.recipes import read_recipe_and_id from pylorax.api.timestamp import TS_CREATED, write_timestamp +from pylorax.base import DataHolder from pylorax.imgutils import default_image_name +from pylorax.ltmpl import LiveTemplateRunner from pylorax.sysutils import joinpaths, flatconfig @@ -283,6 +286,40 @@ def add_customizations(f, recipe): if not wrote_rootpw: f.write('rootpw --lock\n') + +def get_extra_pkgs(dbo, share_dir, compose_type): + """Return extra packages needed for the output type + + :param dbo: dnf base object + :type dbo: dnf.Base + :param share_dir: Path to the top level share directory + :type share_dir: str + :param compose_type: The type of output to create from the recipe + :type compose_type: str + :returns: List of package names (name only, not NEVRA) + :rtype: list + + Currently this is only needed by live-iso, it reads ./live/live-install.tmpl and + processes only the installpkg lines. It lists the packages needed to complete creation of the + iso using the templates such as x86.tmpl + + Keep in mind that the live-install.tmpl is shared between livemedia-creator and lorax-composer, + even though the results are applied differently. + """ + if compose_type != "live-iso": + return [] + + # get the arch information to pass to the runner + arch = ArchData(get_buildarch(dbo)) + defaults = DataHolder(basearch=arch.basearch) + templatedir = joinpaths(find_templates(share_dir), "live") + runner = LiveTemplateRunner(dbo, templatedir=templatedir, defaults=defaults) + runner.run("live-install.tmpl") + log.debug("extra pkgs = %s", runner.pkgs) + + return runner.pkgnames + + def start_build(cfg, dnflock, gitlock, branch, recipe_name, compose_type, test_mode=0): """ Start the build @@ -304,15 +341,22 @@ def start_build(cfg, dnflock, gitlock, branch, recipe_name, compose_type, test_m if compose_type not in compose_types(share_dir): raise RuntimeError("Invalid compose type (%s), must be one of %s" % (compose_type, compose_types(share_dir))) + # Some image types (live-iso) need extra packages for composer to execute the output template + with dnflock.lock: + extra_pkgs = get_extra_pkgs(dnflock.dbo, share_dir, compose_type) + log.debug("Extra packages needed for %s: %s", compose_type, extra_pkgs) + with gitlock.lock: (commit_id, recipe) = read_recipe_and_id(gitlock.repo, branch, recipe_name) # Combine modules and packages and depsolve the list - # TODO include the version/glob in the depsolving module_nver = recipe.module_nver package_nver = recipe.package_nver + package_nver.extend([(name, '*') for name in extra_pkgs]) + projects = sorted(set(module_nver+package_nver), key=lambda p: p[0].lower()) deps = [] + log.info("depsolving %s", recipe["name"]) try: # This can possibly update repodata and reset the YumBase object. with dnflock.lock_check: diff --git a/src/pylorax/ltmpl.py b/src/pylorax/ltmpl.py index 5d239e04..ed149a59 100644 --- a/src/pylorax/ltmpl.py +++ b/src/pylorax/ltmpl.py @@ -811,6 +811,7 @@ class LiveTemplateRunner(TemplateRunner): def __init__(self, dbo, fatalerrors=True, templatedir=None, defaults=None): self.dbo = dbo self.pkgs = [] + self.pkgnames = [] super(LiveTemplateRunner, self).__init__(fatalerrors, templatedir, defaults) @@ -871,6 +872,7 @@ class LiveTemplateRunner(TemplateRunner): logger.info("installpkg: %s expands to %s", p, ",".join(pkgnvrs)) self.pkgs.extend(pkgnvrs) + self.pkgnames.extend([pkg.name for pkg in pkgnames]) except Exception as e: # pylint: disable=broad-except logger.error("installpkg %s failed: %s", p, str(e)) errors = True diff --git a/tests/pylorax/test_compose.py b/tests/pylorax/test_compose.py index af265a83..b4f65e52 100644 --- a/tests/pylorax/test_compose.py +++ b/tests/pylorax/test_compose.py @@ -15,9 +15,15 @@ # along with this program. If not, see . # from io import StringIO +import os +import shutil +import tempfile import unittest -from pylorax.api.compose import add_customizations +from pylorax import get_buildarch +from pylorax.api.compose import add_customizations, get_extra_pkgs +from pylorax.api.config import configure, make_dnf_dirs +from pylorax.api.dnfbase import get_base_object from pylorax.api.recipes import recipe_from_toml BASE_RECIPE = """name = "test-cases" @@ -223,3 +229,41 @@ class CustomizationsTestCase(unittest.TestCase): self.assertCustomization(ROOT_PLAIN_KEY, 'rootpw --plaintext "plainpassword"') self.assertCustomization(ROOT_PLAIN_KEY, 'sshkey --user root "A SSH KEY FOR THE USER"') self.assertNotCustomization(ROOT_PLAIN_KEY, "rootpw --lock") + + +class ExtraPkgsTest(unittest.TestCase): + @classmethod + def setUpClass(self): + self.tmp_dir = tempfile.mkdtemp(prefix="lorax.test.repo.") + self.config = configure(root_dir=self.tmp_dir, test_config=True) + make_dnf_dirs(self.config, os.getuid(), os.getgid()) + self.dbo = get_base_object(self.config) + + @classmethod + def tearDownClass(self): + shutil.rmtree(self.tmp_dir) + + def test_live_install(self): + """Check that live-install.tmpl is parsed correctly""" + # A package for each arch to test for + arch_pkg = { + "aarch64": "shim-aa64", + "arm": "grub2-efi-arm-cdboot", + "armhfp": "grub2-efi-arm-cdboot", + "x86_64": "shim-x64", + "i386": "memtest86+", + "ppc64le": "powerpc-utils", + "s390x": "s390utils-base" + } + + extra_pkgs = get_extra_pkgs(self.dbo, "./share/", "live-iso") + self.assertTrue(len(extra_pkgs) > 0) + + # Results depend on arch + arch = get_buildarch(self.dbo) + self.assertTrue(arch_pkg[arch] in extra_pkgs) + + def test_other_install(self): + """Test that non-live doesn't parse live-install.tmpl""" + extra_pkgs = get_extra_pkgs(self.dbo, "./share/", "qcow2") + self.assertEqual(extra_pkgs, [])