From e90e7653df2b480263088d8dcbe2a8f8cb136841 Mon Sep 17 00:00:00 2001 From: "Brian C. Lane" Date: Mon, 18 Feb 2019 16:56:26 -0800 Subject: [PATCH] Move the package requirements for live-iso setup out of the template In order to support iso creation on multiple arches with the templates we need to be able to select different packages based on arch. lorax-composer uses the arch-specific Lorax templates in order to generate the output iso so this patch: 1. Creates a new template and type to parse it, live-install.tmpl which contains only installpkg commands and #if clauses for arch 2. Removes bootloader related packages from the live-iso.ks 3. Remove dracut-config-rescue exclusion because it can cause problems with some blueprints. 4. Switch logo requirement to system-logos which is satisfied by generic-logos or fedora-logos. This prevents conflicts when a blueprint installs fedora-release-workstation. So in the future, if x86.tmpl, etc. need a new package to support creating the iso it should be added to the correct section in ./share/live/live-install.tmpl Resolves: rhbz#1673744 --- share/composer/live-iso.ks | 13 ------ .../99-generic/live/live-install.tmpl | 32 +++++++++++++ src/pylorax/api/compose.py | 46 ++++++++++++++++++- src/pylorax/ltmpl.py | 2 + tests/pylorax/test_compose.py | 45 +++++++++++++++++- 5 files changed, 123 insertions(+), 15 deletions(-) create mode 100644 share/templates.d/99-generic/live/live-install.tmpl diff --git a/share/composer/live-iso.ks b/share/composer/live-iso.ks index fec2a7ba..a88e8c43 100644 --- a/share/composer/live-iso.ks +++ b/share/composer/live-iso.ks @@ -351,22 +351,9 @@ EOF # Packages requires to support this output format go here isomd5sum kernel -memtest86+ -syslinux --dracut-config-rescue dracut-config-generic dracut-live system-logos selinux-policy-targeted -# This package is needed to boot the iso on UEFI -shim -shim-ia32 -grub2 -grub2-efi -grub2-efi-*-cdboot -grub2-efi-ia32 -efibootmgr - - # NOTE lorax-composer will add the blueprint packages below here, including the final %end%packages diff --git a/share/templates.d/99-generic/live/live-install.tmpl b/share/templates.d/99-generic/live/live-install.tmpl new file mode 100644 index 00000000..88abff59 --- /dev/null +++ b/share/templates.d/99-generic/live/live-install.tmpl @@ -0,0 +1,32 @@ +## livemedia-creator: Install packages needed for iso creation using per-arch templates +<%page args="basearch"/> + +## arch-specific bootloader packages +%if basearch == "aarch64": + installpkg efibootmgr + installpkg grub2-efi-aa64-cdboot shim-aa64 + installpkg uboot-tools +%endif +%if basearch in ("arm", "armhfp"): + installpkg efibootmgr + installpkg grub2-efi-arm-cdboot + installpkg uboot-tools +%endif +%if basearch == "x86_64": + installpkg grub2-tools-efi + installpkg efibootmgr + installpkg shim-x64 grub2-efi-x64-cdboot + installpkg shim-ia32 grub2-efi-ia32-cdboot +%endif +%if basearch in ("i386", "x86_64"): + installpkg biosdevname memtest86+ syslinux + installpkg grub2-tools grub2-tools-minimal grub2-tools-extra +%endif +%if basearch in ("ppc64le"): + installpkg powerpc-utils + installpkg grub2-tools grub2-tools-minimal grub2-tools-extra + installpkg grub2-${basearch} +%endif +%if basearch == "s390x": + installpkg s390utils-base +%endif diff --git a/src/pylorax/api/compose.py b/src/pylorax/api/compose.py index da2011f7..f6e1b1d9 100644 --- a/src/pylorax/api/compose.py +++ b/src/pylorax/api/compose.py @@ -44,12 +44,15 @@ from uuid import uuid4 from pykickstart.parser import KickstartParser from pykickstart.version import makeVersion +from pylorax import ArchData, find_templates, get_buildarch from pylorax.api.dnfbase import check_repos from pylorax.api.projects import projects_depsolve, projects_depsolve_with_size, dep_nevra from pylorax.api.projects import ProjectsError from pylorax.api.recipes import read_recipe_and_id from pylorax.api.timestamp import TS_CREATED, write_timestamp +from pylorax.base import DataHolder from pylorax.imgutils import default_image_name +from pylorax.ltmpl import LiveTemplateRunner from pylorax.sysutils import joinpaths, flatconfig @@ -277,6 +280,40 @@ def add_customizations(f, recipe): if not wrote_rootpw: f.write('rootpw --lock\n') + +def get_extra_pkgs(dbo, share_dir, compose_type): + """Return extra packages needed for the output type + + :param dbo: dnf base object + :type dbo: dnf.Base + :param share_dir: Path to the top level share directory + :type share_dir: str + :param compose_type: The type of output to create from the recipe + :type compose_type: str + :returns: List of package names (name only, not NEVRA) + :rtype: list + + Currently this is only needed by live-iso, it reads ./live/live-install.tmpl and + processes only the installpkg lines. It lists the packages needed to complete creation of the + iso using the templates such as x86.tmpl + + Keep in mind that the live-install.tmpl is shared between livemedia-creator and lorax-composer, + even though the results are applied differently. + """ + if compose_type != "live-iso": + return [] + + # get the arch information to pass to the runner + arch = ArchData(get_buildarch(dbo)) + defaults = DataHolder(basearch=arch.basearch) + templatedir = joinpaths(find_templates(share_dir), "live") + runner = LiveTemplateRunner(dbo, templatedir=templatedir, defaults=defaults) + runner.run("live-install.tmpl") + log.debug("extra pkgs = %s", runner.pkgs) + + return runner.pkgnames + + def start_build(cfg, dnflock, gitlock, branch, recipe_name, compose_type, test_mode=0): """ Start the build @@ -298,6 +335,11 @@ def start_build(cfg, dnflock, gitlock, branch, recipe_name, compose_type, test_m if compose_type not in compose_types(share_dir): raise RuntimeError("Invalid compose type (%s), must be one of %s" % (compose_type, compose_types(share_dir))) + # Some image types (live-iso) need extra packages for composer to execute the output template + with dnflock.lock: + extra_pkgs = get_extra_pkgs(dnflock.dbo, share_dir, compose_type) + log.debug("Extra packages needed for %s: %s", compose_type, extra_pkgs) + with gitlock.lock: (commit_id, recipe) = read_recipe_and_id(gitlock.repo, branch, recipe_name) @@ -307,11 +349,13 @@ def start_build(cfg, dnflock, gitlock, branch, recipe_name, compose_type, test_m raise RuntimeError("Compose requires non-CDN repos to be enabled") # Combine modules and packages and depsolve the list - # TODO include the version/glob in the depsolving module_nver = recipe.module_nver package_nver = recipe.package_nver + package_nver.extend([(name, '*') for name in extra_pkgs]) + projects = sorted(set(module_nver+package_nver), key=lambda p: p[0].lower()) deps = [] + log.info("depsolving %s", recipe["name"]) try: # This can possibly update repodata and reset the YumBase object. with dnflock.lock_check: diff --git a/src/pylorax/ltmpl.py b/src/pylorax/ltmpl.py index aa07a75b..7fbbc252 100644 --- a/src/pylorax/ltmpl.py +++ b/src/pylorax/ltmpl.py @@ -809,6 +809,7 @@ class LiveTemplateRunner(TemplateRunner): def __init__(self, dbo, fatalerrors=True, templatedir=None, defaults=None): self.dbo = dbo self.pkgs = [] + self.pkgnames = [] super(LiveTemplateRunner, self).__init__(fatalerrors, templatedir, defaults) @@ -869,6 +870,7 @@ class LiveTemplateRunner(TemplateRunner): logger.info("installpkg: %s expands to %s", p, ",".join(pkgnvrs)) self.pkgs.extend(pkgnvrs) + self.pkgnames.extend([pkg.name for pkg in pkgnames]) except Exception as e: # pylint: disable=broad-except logger.error("installpkg %s failed: %s", p, str(e)) errors = True diff --git a/tests/pylorax/test_compose.py b/tests/pylorax/test_compose.py index af265a83..e5aa67d1 100644 --- a/tests/pylorax/test_compose.py +++ b/tests/pylorax/test_compose.py @@ -15,9 +15,14 @@ # along with this program. If not, see . # from io import StringIO +import shutil +import tempfile import unittest -from pylorax.api.compose import add_customizations +from pylorax import get_buildarch +from pylorax.api.compose import add_customizations, get_extra_pkgs +from pylorax.api.config import configure, make_dnf_dirs +from pylorax.api.dnfbase import get_base_object from pylorax.api.recipes import recipe_from_toml BASE_RECIPE = """name = "test-cases" @@ -223,3 +228,41 @@ class CustomizationsTestCase(unittest.TestCase): self.assertCustomization(ROOT_PLAIN_KEY, 'rootpw --plaintext "plainpassword"') self.assertCustomization(ROOT_PLAIN_KEY, 'sshkey --user root "A SSH KEY FOR THE USER"') self.assertNotCustomization(ROOT_PLAIN_KEY, "rootpw --lock") + + +class ExtraPkgsTest(unittest.TestCase): + @classmethod + def setUpClass(self): + self.tmp_dir = tempfile.mkdtemp(prefix="lorax.test.repo.") + self.config = configure(root_dir=self.tmp_dir, test_config=True) + make_dnf_dirs(self.config) + self.dbo = get_base_object(self.config) + + @classmethod + def tearDownClass(self): + shutil.rmtree(self.tmp_dir) + + def test_live_install(self): + """Check that live-install.tmpl is parsed correctly""" + # A package for each arch to test for + arch_pkg = { + "aarch64": "shim-aa64", + "arm": "grub2-efi-arm-cdboot", + "armhfp": "grub2-efi-arm-cdboot", + "x86_64": "shim-x64", + "i386": "memtest86+", + "ppc64le": "powerpc-utils", + "s390x": "s390utils-base" + } + + extra_pkgs = get_extra_pkgs(self.dbo, "./share/", "live-iso") + self.assertTrue(len(extra_pkgs) > 0) + + # Results depend on arch + arch = get_buildarch(self.dbo) + self.assertTrue(arch_pkg[arch] in extra_pkgs) + + def test_other_install(self): + """Test that non-live doesn't parse live-install.tmpl""" + extra_pkgs = get_extra_pkgs(self.dbo, "./share/", "qcow2") + self.assertEqual(extra_pkgs, [])