From afa68df87339b49fb597b0e4d0d9a4d5114580de Mon Sep 17 00:00:00 2001 From: "Brian C. Lane" Date: Wed, 13 Mar 2019 10:56:20 -0700 Subject: [PATCH] lorax-composer: Add the ability to append to the kernel command-line Sometimes it is necessary to modify the kernel command-line of the image, this adds support for a [customizations.kernel] section to the blueprint: [customizations.kernel] append = "nosmt=force" This will be appended to the kickstart's bootloader --append argument. Includes tests for modifying the bootloader line, the kickstart template, and examining the final-kickstart.ks created for a compose. (cherry picked from commit 59464286f9579087d21c4a7ad2c8068c67762c38) (cherry picked from commit c5f4dfe113ced59cb601d8ace6e1ce152f721645) --- docs/lorax-composer.rst | 12 ++++ src/pylorax/api/compose.py | 63 +++++++++++++++++++- src/pylorax/api/recipes.py | 2 +- tests/pylorax/blueprints/example-append.toml | 18 ++++++ tests/pylorax/test_compose.py | 58 +++++++++++++++++- tests/pylorax/test_server.py | 51 +++++++++++++++- 6 files changed, 198 insertions(+), 6 deletions(-) create mode 100644 tests/pylorax/blueprints/example-append.toml diff --git a/docs/lorax-composer.rst b/docs/lorax-composer.rst index ecd4f4c8..d349f787 100644 --- a/docs/lorax-composer.rst +++ b/docs/lorax-composer.rst @@ -181,6 +181,18 @@ The ``[[customizations]]`` section can be used to configure the hostname of the hostname = "baseimage" +[customizations.kernel] +*********************** + +This allows you to append arguments to the bootloader's kernel commandline. This will not have any +effect on ``tar`` or ``ext4-filesystem`` images since they do not include a bootloader. + +For example:: + + [customizations.kernel] + append = "nosmt=force" + + [[customizations.sshkey]] ************************* diff --git a/src/pylorax/api/compose.py b/src/pylorax/api/compose.py index 29b9f43e..c37e982e 100644 --- a/src/pylorax/api/compose.py +++ b/src/pylorax/api/compose.py @@ -1,4 +1,4 @@ -# Copyright (C) 2018 Red Hat, Inc. +# Copyright (C) 2018-2019 Red Hat, Inc. # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -35,6 +35,7 @@ log = logging.getLogger("lorax-composer") import os from glob import glob +from io import StringIO from math import ceil import pytoml as toml import shutil @@ -123,6 +124,62 @@ def repo_to_ks(r, url="url"): return cmd +def bootloader_append(line, kernel_append): + """ Insert the kernel_append string into the --append argument + + :param line: The bootloader ... line + :type line: str + :param kernel_append: The arguments to append to the --append section + :type kernel_append: str + + Using pykickstart to process the line is the best way to make sure it + is parsed correctly, and re-assembled for inclusion into the final kickstart + """ + ks_version = makeVersion() + ks = KickstartParser(ks_version, errorsAreFatal=False, missingIncludeIsFatal=False) + ks.readKickstartFromString(line) + + if ks.handler.bootloader.appendLine: + ks.handler.bootloader.appendLine += " %s" % kernel_append + else: + ks.handler.bootloader.appendLine = kernel_append + + # Converting back to a string includes a comment, return just the bootloader line + return str(ks.handler.bootloader).splitlines()[-1] + + +def customize_ks_template(ks_template, recipe): + """ Customize the kickstart template and return it + + :param ks_template: The kickstart template + :type ks_template: str + :param recipe: + :type recipe: Recipe object + + Apply customizations.kernel.append to the bootloader argument in the template. + Add bootloader line if it is missing. + """ + if "customizations" not in recipe or \ + "kernel" not in recipe["customizations"] or \ + "append" not in recipe["customizations"]["kernel"]: + return ks_template + kernel_append = recipe["customizations"]["kernel"]["append"] + found_bootloader = False + output = StringIO() + for line in ks_template.splitlines(): + if not line.startswith("bootloader"): + print(line, file=output) + continue + found_bootloader = True + log.debug("Found bootloader line: %s", line) + print(bootloader_append(line, kernel_append), file=output) + + if found_bootloader: + return output.getvalue() + else: + return 'bootloader --append="%s" --location=none' % kernel_append + output.getvalue() + + def write_ks_root(f, user): """ Write kickstart root password and sshkey entry @@ -439,12 +496,14 @@ def start_build(cfg, dnflock, gitlock, branch, recipe_name, compose_type, test_m # Write the root partition and it's size in MB (rounded up) f.write('part / --size=%d\n' % ceil(installed_size / 1024**2)) - f.write(ks_template) + # Some customizations modify the template before writing it + f.write(customize_ks_template(ks_template, recipe)) for d in deps: f.write(dep_nevra(d)+"\n") f.write("%end\n") + # Other customizations can be appended to the kickstart add_customizations(f, recipe) # Setup the config to pass to novirt_install diff --git a/src/pylorax/api/recipes.py b/src/pylorax/api/recipes.py index 617044f5..87e8ee64 100644 --- a/src/pylorax/api/recipes.py +++ b/src/pylorax/api/recipes.py @@ -1,5 +1,5 @@ # -# Copyright (C) 2017 Red Hat, Inc. +# Copyright (C) 2017-2019 Red Hat, Inc. # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by diff --git a/tests/pylorax/blueprints/example-append.toml b/tests/pylorax/blueprints/example-append.toml new file mode 100644 index 00000000..e5d54535 --- /dev/null +++ b/tests/pylorax/blueprints/example-append.toml @@ -0,0 +1,18 @@ +name = "example-append" +description = "An example using kernel append customization" +version = "0.0.1" + +[[packages]] +name = "tmux" +version = "2.8" + +[[packages]] +name = "openssh-server" +version = "7.*" + +[[packages]] +name = "rsync" +version = "3.1.*" + +[customizations.kernel] +append = "nosmt=force" diff --git a/tests/pylorax/test_compose.py b/tests/pylorax/test_compose.py index b4f65e52..1d9de95a 100644 --- a/tests/pylorax/test_compose.py +++ b/tests/pylorax/test_compose.py @@ -21,10 +21,12 @@ import tempfile import unittest from pylorax import get_buildarch -from pylorax.api.compose import add_customizations, get_extra_pkgs +from pylorax.api.compose import add_customizations, get_extra_pkgs, compose_types +from pylorax.api.compose import bootloader_append, customize_ks_template from pylorax.api.config import configure, make_dnf_dirs from pylorax.api.dnfbase import get_base_object from pylorax.api.recipes import recipe_from_toml +from pylorax.sysutils import joinpaths BASE_RECIPE = """name = "test-cases" description = "Used for testing" @@ -230,6 +232,60 @@ class CustomizationsTestCase(unittest.TestCase): self.assertCustomization(ROOT_PLAIN_KEY, 'sshkey --user root "A SSH KEY FOR THE USER"') self.assertNotCustomization(ROOT_PLAIN_KEY, "rootpw --lock") + def test_bootloader_append(self): + """Test bootloader_append function""" + + self.assertEqual(bootloader_append("", "nosmt=force"), 'bootloader --append="nosmt=force" --location=none') + self.assertEqual(bootloader_append("", "nosmt=force console=ttyS0,115200n8"), + 'bootloader --append="nosmt=force console=ttyS0,115200n8" --location=none') + self.assertEqual(bootloader_append("bootloader --location=none", "nosmt=force"), + 'bootloader --append="nosmt=force" --location=none') + self.assertEqual(bootloader_append("bootloader --location=none", "console=ttyS0,115200n8 nosmt=force"), + 'bootloader --append="console=ttyS0,115200n8 nosmt=force" --location=none') + self.assertEqual(bootloader_append('bootloader --append="no_timer_check console=ttyS0,115200n8" --location=mbr', "nosmt=force"), + 'bootloader --append="no_timer_check console=ttyS0,115200n8 nosmt=force" --location=mbr') + self.assertEqual(bootloader_append('bootloader --append="console=tty1" --location=mbr --password="BADPASSWORD"', "nosmt=force"), + 'bootloader --append="console=tty1 nosmt=force" --location=mbr --password="BADPASSWORD"') + + def _checkBootloader(self, result, append_str): + """Find the bootloader line and make sure append_str is in it""" + + for line in result.splitlines(): + if line.startswith("bootloader") and append_str in line: + return True + return False + + def test_customize_ks_template(self): + """Test that [customizations.kernel] works correctly""" + blueprint_data = """name = "test-kernel" +description = "test recipe" +version = "0.0.1" + +[customizations.kernel] +append="nosmt=force" +""" + recipe = recipe_from_toml(blueprint_data) + + # Test against a kickstart without bootloader + result = customize_ks_template("firewall --enabled\n", recipe) + self.assertTrue(self._checkBootloader(result, "nosmt=force")) + + # Test against all of the available templates + share_dir = "./share/" + errors = [] + for compose_type in compose_types(share_dir): + # Read the kickstart template for this type + ks_template_path = joinpaths(share_dir, "composer", compose_type) + ".ks" + ks_template = open(ks_template_path, "r").read() + result = customize_ks_template(ks_template, recipe) + if not self._checkBootloader(result, "nosmt=force"): + errors.append(("compose_type %s failed" % compose_type, result)) + + # Print the bad results + for e, r in errors: + print("%s:\n%s\n\n" % (e, r)) + + self.assertEqual(errors, []) class ExtraPkgsTest(unittest.TestCase): @classmethod diff --git a/tests/pylorax/test_server.py b/tests/pylorax/test_server.py index cad572af..ee7c84ed 100644 --- a/tests/pylorax/test_server.py +++ b/tests/pylorax/test_server.py @@ -141,9 +141,9 @@ class ServerTestCase(unittest.TestCase): def test_02_blueprints_list(self): """Test the /api/v0/blueprints/list route""" - list_dict = {"blueprints":["example-atlas", "example-custom-base", "example-development", + list_dict = {"blueprints":["example-append", "example-atlas", "example-custom-base", "example-development", "example-glusterfs", "example-http-server", "example-jboss", - "example-kubernetes"], "limit":20, "offset":0, "total":7} + "example-kubernetes"], "limit":20, "offset":0, "total":8} resp = self.server.get("/api/v0/blueprints/list") data = json.loads(resp.data) self.assertEqual(data, list_dict) @@ -1163,6 +1163,53 @@ class ServerTestCase(unittest.TestCase): self.assertIn(build_id_fail, ids, "Failed build not listed by /compose/status status filter") self.assertNotIn(build_id_success, "Finished build listed by /compose/status status filter") + def test_compose_14_kernel_append(self): + """Test the /api/v0/compose with kernel append customization""" + test_compose = {"blueprint_name": "example-append", + "compose_type": "tar", + "branch": "master"} + + resp = self.server.post("/api/v0/compose?test=2", + data=json.dumps(test_compose), + content_type="application/json") + data = json.loads(resp.data) + self.assertNotEqual(data, None) + self.assertEqual(data["status"], True, "Failed to start test compose: %s" % data) + + build_id = data["build_id"] + + # Is it in the queue list (either new or run is fine, based on timing) + resp = self.server.get("/api/v0/compose/queue") + data = json.loads(resp.data) + self.assertNotEqual(data, None) + ids = [e["id"] for e in data["new"] + data["run"]] + self.assertEqual(build_id in ids, True, "Failed to add build to the queue") + + # Wait for it to start + self.assertEqual(self.wait_for_status(build_id, ["RUNNING"]), True, "Failed to start test compose") + + # Wait for it to finish + self.assertEqual(self.wait_for_status(build_id, ["FINISHED"]), True, "Failed to finish test compose") + + resp = self.server.get("/api/v0/compose/info/%s" % build_id) + data = json.loads(resp.data) + self.assertNotEqual(data, None) + self.assertEqual(data["queue_status"], "FINISHED", "Build not in FINISHED state") + + # Examine the final-kickstart.ks for the customizations + # A bit kludgy since it examines the filesystem directly, but that's better than unpacking the metadata + final_ks = open(joinpaths(self.repo_dir, "var/lib/lorax/composer/results/", build_id, "final-kickstart.ks")).read() + + # Check for the expected customizations in the kickstart + # nosmt=force should be in the bootloader line, find it and check it + bootloader_line = "" + for line in final_ks.splitlines(): + if line.startswith("bootloader"): + bootloader_line = line + break + self.assertNotEqual(bootloader_line, "", "No bootloader line found") + self.assertTrue("nosmt=force" in bootloader_line) + def assertInputError(self, resp): """Check all the conditions for a successful input check error result""" data = json.loads(resp.data)