lorax-composer: Add the ability to append to the kernel command-line

Sometimes it is necessary to modify the kernel command-line of the
image, this adds support for a [customizations.kernel] section to the
blueprint:

[customizations.kernel]
append = "nosmt=force"

This will be appended to the kickstart's bootloader --append argument.

Includes tests for modifying the bootloader line, the kickstart
template, and examining the final-kickstart.ks created for a compose.

(cherry picked from commit 59464286f9)
(cherry picked from commit c5f4dfe113)

docs/lorax-composer.rst

@@ -181,6 +181,18 @@ The ``[[customizations]]`` section can be used to configure the hostname of the
     hostname = "baseimage"
 
 
+[customizations.kernel]
+***********************
+
+This allows you to append arguments to the bootloader's kernel commandline. This will not have any
+effect on ``tar`` or ``ext4-filesystem`` images since they do not include a bootloader.
+
+For example::
+
+    [customizations.kernel]
+    append = "nosmt=force"
+
+
 [[customizations.sshkey]]
 *************************
 

src/pylorax/api/compose.py

@@ -1,4 +1,4 @@
-# Copyright (C) 2018 Red Hat, Inc.
+# Copyright (C) 2018-2019 Red Hat, Inc.
 #
 # This program is free software; you can redistribute it and/or modify
 # it under the terms of the GNU General Public License as published by
@@ -35,6 +35,7 @@ log = logging.getLogger("lorax-composer")
 
 import os
 from glob import glob
+from io import StringIO
 from math import ceil
 import pytoml as toml
 import shutil
@@ -123,6 +124,62 @@ def repo_to_ks(r, url="url"):
     return cmd
 
 
+def bootloader_append(line, kernel_append):
+    """ Insert the kernel_append string into the --append argument
+
+    :param line: The bootloader ... line
+    :type line: str
+    :param kernel_append: The arguments to append to the --append section
+    :type kernel_append: str
+
+    Using pykickstart to process the line is the best way to make sure it
+    is parsed correctly, and re-assembled for inclusion into the final kickstart
+    """
+    ks_version = makeVersion()
+    ks = KickstartParser(ks_version, errorsAreFatal=False, missingIncludeIsFatal=False)
+    ks.readKickstartFromString(line)
+
+    if ks.handler.bootloader.appendLine:
+        ks.handler.bootloader.appendLine += " %s" % kernel_append
+    else:
+        ks.handler.bootloader.appendLine = kernel_append
+
+    # Converting back to a string includes a comment, return just the bootloader line
+    return str(ks.handler.bootloader).splitlines()[-1]
+
+
+def customize_ks_template(ks_template, recipe):
+    """ Customize the kickstart template and return it
+
+    :param ks_template: The kickstart template
+    :type ks_template: str
+    :param recipe:
+    :type recipe: Recipe object
+
+    Apply customizations.kernel.append to the bootloader argument in the template.
+    Add bootloader line if it is missing.
+    """
+    if "customizations" not in recipe or \
+       "kernel" not in recipe["customizations"] or \
+       "append" not in recipe["customizations"]["kernel"]:
+        return ks_template
+    kernel_append = recipe["customizations"]["kernel"]["append"]
+    found_bootloader = False
+    output = StringIO()
+    for line in ks_template.splitlines():
+        if not line.startswith("bootloader"):
+            print(line, file=output)
+            continue
+        found_bootloader = True
+        log.debug("Found bootloader line: %s", line)
+        print(bootloader_append(line, kernel_append), file=output)
+
+    if found_bootloader:
+        return output.getvalue()
+    else:
+        return 'bootloader --append="%s" --location=none' % kernel_append + output.getvalue()
+
+
 def write_ks_root(f, user):
     """ Write kickstart root password and sshkey entry
 
@@ -439,12 +496,14 @@ def start_build(cfg, dnflock, gitlock, branch, recipe_name, compose_type, test_m
         # Write the root partition and it's size in MB (rounded up)
         f.write('part / --size=%d\n' % ceil(installed_size / 1024**2))
 
-        f.write(ks_template)
+        # Some customizations modify the template before writing it
+        f.write(customize_ks_template(ks_template, recipe))
 
         for d in deps:
             f.write(dep_nevra(d)+"\n")
         f.write("%end\n")
 
+        # Other customizations can be appended to the kickstart
         add_customizations(f, recipe)
 
     # Setup the config to pass to novirt_install

src/pylorax/api/recipes.py

@@ -1,5 +1,5 @@
 #
-# Copyright (C) 2017  Red Hat, Inc.
+# Copyright (C) 2017-2019  Red Hat, Inc.
 #
 # This program is free software; you can redistribute it and/or modify
 # it under the terms of the GNU General Public License as published by

tests/pylorax/blueprints/example-append.toml

@@ -0,0 +1,18 @@
+name = "example-append"
+description = "An example using kernel append customization"
+version = "0.0.1"
+
+[[packages]]
+name = "tmux"
+version = "2.8"
+
+[[packages]]
+name = "openssh-server"
+version = "7.*"
+
+[[packages]]
+name = "rsync"
+version = "3.1.*"
+
+[customizations.kernel]
+append = "nosmt=force"

tests/pylorax/test_compose.py

@@ -21,10 +21,12 @@ import tempfile
 import unittest
 
 from pylorax import get_buildarch
-from pylorax.api.compose import add_customizations, get_extra_pkgs
+from pylorax.api.compose import add_customizations, get_extra_pkgs, compose_types
+from pylorax.api.compose import bootloader_append, customize_ks_template
 from pylorax.api.config import configure, make_dnf_dirs
 from pylorax.api.dnfbase import get_base_object
 from pylorax.api.recipes import recipe_from_toml
+from pylorax.sysutils import joinpaths
 
 BASE_RECIPE = """name = "test-cases"
 description = "Used for testing"
@@ -230,6 +232,60 @@ class CustomizationsTestCase(unittest.TestCase):
         self.assertCustomization(ROOT_PLAIN_KEY, 'sshkey --user root "A SSH KEY FOR THE USER"')
         self.assertNotCustomization(ROOT_PLAIN_KEY, "rootpw --lock")
 
+    def test_bootloader_append(self):
+        """Test bootloader_append function"""
+
+        self.assertEqual(bootloader_append("", "nosmt=force"), 'bootloader --append="nosmt=force" --location=none')
+        self.assertEqual(bootloader_append("", "nosmt=force console=ttyS0,115200n8"),
+                         'bootloader --append="nosmt=force console=ttyS0,115200n8" --location=none')
+        self.assertEqual(bootloader_append("bootloader --location=none", "nosmt=force"),
+                         'bootloader --append="nosmt=force" --location=none')
+        self.assertEqual(bootloader_append("bootloader --location=none", "console=ttyS0,115200n8 nosmt=force"),
+                         'bootloader --append="console=ttyS0,115200n8 nosmt=force" --location=none')
+        self.assertEqual(bootloader_append('bootloader --append="no_timer_check console=ttyS0,115200n8" --location=mbr', "nosmt=force"),
+                         'bootloader --append="no_timer_check console=ttyS0,115200n8 nosmt=force" --location=mbr')
+        self.assertEqual(bootloader_append('bootloader --append="console=tty1" --location=mbr --password="BADPASSWORD"', "nosmt=force"),
+                         'bootloader --append="console=tty1 nosmt=force" --location=mbr --password="BADPASSWORD"')
+
+    def _checkBootloader(self, result, append_str):
+        """Find the bootloader line and make sure append_str is in it"""
+
+        for line in result.splitlines():
+            if line.startswith("bootloader") and append_str in line:
+                return True
+        return False
+
+    def test_customize_ks_template(self):
+        """Test that [customizations.kernel] works correctly"""
+        blueprint_data = """name = "test-kernel"
+description = "test recipe"
+version = "0.0.1"
+
+[customizations.kernel]
+append="nosmt=force"
+"""
+        recipe = recipe_from_toml(blueprint_data)
+
+        # Test against a kickstart without bootloader
+        result = customize_ks_template("firewall --enabled\n", recipe)
+        self.assertTrue(self._checkBootloader(result, "nosmt=force"))
+
+        # Test against all of the available templates
+        share_dir = "./share/"
+        errors = []
+        for compose_type in compose_types(share_dir):
+            # Read the kickstart template for this type
+            ks_template_path = joinpaths(share_dir, "composer", compose_type) + ".ks"
+            ks_template = open(ks_template_path, "r").read()
+            result = customize_ks_template(ks_template, recipe)
+            if not self._checkBootloader(result, "nosmt=force"):
+                errors.append(("compose_type %s failed" % compose_type, result))
+
+        # Print the bad results
+        for e, r in errors:
+            print("%s:\n%s\n\n" % (e, r))
+
+        self.assertEqual(errors, [])
 
 class ExtraPkgsTest(unittest.TestCase):
     @classmethod

tests/pylorax/test_server.py

@@ -141,9 +141,9 @@ class ServerTestCase(unittest.TestCase):
 
     def test_02_blueprints_list(self):
         """Test the /api/v0/blueprints/list route"""
-        list_dict = {"blueprints":["example-atlas", "example-custom-base", "example-development",
+        list_dict = {"blueprints":["example-append", "example-atlas", "example-custom-base", "example-development",
                                    "example-glusterfs", "example-http-server", "example-jboss",
-                                   "example-kubernetes"], "limit":20, "offset":0, "total":7}
+                                   "example-kubernetes"], "limit":20, "offset":0, "total":8}
         resp = self.server.get("/api/v0/blueprints/list")
         data = json.loads(resp.data)
         self.assertEqual(data, list_dict)
@@ -1163,6 +1163,53 @@ class ServerTestCase(unittest.TestCase):
         self.assertIn(build_id_fail, ids, "Failed build not listed by /compose/status status filter")
         self.assertNotIn(build_id_success, "Finished build listed by /compose/status status filter")
 
+    def test_compose_14_kernel_append(self):
+        """Test the /api/v0/compose with kernel append customization"""
+        test_compose = {"blueprint_name": "example-append",
+                        "compose_type": "tar",
+                        "branch": "master"}
+
+        resp = self.server.post("/api/v0/compose?test=2",
+                                data=json.dumps(test_compose),
+                                content_type="application/json")
+        data = json.loads(resp.data)
+        self.assertNotEqual(data, None)
+        self.assertEqual(data["status"], True, "Failed to start test compose: %s" % data)
+
+        build_id = data["build_id"]
+
+        # Is it in the queue list (either new or run is fine, based on timing)
+        resp = self.server.get("/api/v0/compose/queue")
+        data = json.loads(resp.data)
+        self.assertNotEqual(data, None)
+        ids = [e["id"] for e in data["new"] + data["run"]]
+        self.assertEqual(build_id in ids, True, "Failed to add build to the queue")
+
+        # Wait for it to start
+        self.assertEqual(self.wait_for_status(build_id, ["RUNNING"]), True, "Failed to start test compose")
+
+        # Wait for it to finish
+        self.assertEqual(self.wait_for_status(build_id, ["FINISHED"]), True, "Failed to finish test compose")
+
+        resp = self.server.get("/api/v0/compose/info/%s" % build_id)
+        data = json.loads(resp.data)
+        self.assertNotEqual(data, None)
+        self.assertEqual(data["queue_status"], "FINISHED", "Build not in FINISHED state")
+
+        # Examine the final-kickstart.ks for the customizations
+        # A bit kludgy since it examines the filesystem directly, but that's better than unpacking the metadata
+        final_ks = open(joinpaths(self.repo_dir, "var/lib/lorax/composer/results/", build_id, "final-kickstart.ks")).read()
+
+        # Check for the expected customizations in the kickstart
+        # nosmt=force should be in the bootloader line, find it and check it
+        bootloader_line = ""
+        for line in final_ks.splitlines():
+            if line.startswith("bootloader"):
+                bootloader_line = line
+                break
+        self.assertNotEqual(bootloader_line, "", "No bootloader line found")
+        self.assertTrue("nosmt=force" in bootloader_line)
+
     def assertInputError(self, resp):
         """Check all the conditions for a successful input check error result"""
         data = json.loads(resp.data)