lorax-composer: Add locale support to blueprints

You can now set the keyboard layout and language. Eg.

[customizations.locale]
languages = ["en_CA.utf8", "en_HK.utf8"]
keyboard = "de (dvorak)"

Existing entries in the kickstart templates are replaced with the new
ones. If there are no entries then it will default to 'keyboard us' and
'lang en_US.UTF-8'

Includes tests, and leaves the existing keyboard and lang entries in the
templates with a note that they can be replaced by the blueprint.

(cherry picked from commit e5a8700bdf)
(cherry picked from commit 49259c607b)

docs/lorax-composer.rst

@@ -265,24 +265,24 @@ cannot be overridden because they are required to boot in the selected environme
 timezone will be updated to the one selected in the blueprint.
 
 
-[[customizations.locale]]
+[customizations.locale]
-*************************
+***********************
 
 Customize the locale settings for the system::
 
-    [[customizations.locale]]
+    [customizations.locale]
-    language = "en_US.UTF-8"
+    languages = ["en_US.UTF-8"]
     keyboard = "us"
 
-The values supported by ``language`` can be listed by running ``localectl list-locales`` from
+The values supported by ``languages`` can be listed by running ``localectl list-locales`` from
 the command line.
 
 The values supported by ``keyboard`` can be listed by running ``localectl list-keymaps`` from
 the command line.
 
-Multiple locale and keyboard sections can be used. The first one becomes the
+Multiple languages can be added. The first one becomes the
-primary, and the others are added as secondary. One or the other of ``language``
+primary, and the others are added as secondary. One or the other of ``languages``
-or ``keyboard`` must be included (or both).
+or ``keyboard`` must be included (or both) in the section.
 
 
 [customizations.firewall]

share/composer/alibaba.ks

@@ -6,6 +6,7 @@ firewall --enabled
 # NOTE: The root account is locked by default
 # Network information
 network  --bootproto=dhcp --onboot=on --activate
+# NOTE: keyboard and lang can be replaced by blueprint customizations.locale settings
 # System keyboard
 keyboard --xlayouts=us --vckeymap=us
 # System language

share/composer/ami.ks

@@ -6,6 +6,7 @@ firewall --enabled
 # NOTE: The root account is locked by default
 # Network information
 network  --bootproto=dhcp --onboot=on --activate
+# NOTE: keyboard and lang can be replaced by blueprint customizations.locale settings
 # System keyboard
 keyboard --xlayouts=us --vckeymap=us
 # System language

share/composer/ext4-filesystem.ks

@@ -6,6 +6,7 @@ firewall --enabled
 # NOTE: The root account is locked by default
 # Network information
 network  --bootproto=dhcp --onboot=on --activate
+# NOTE: keyboard and lang can be replaced by blueprint customizations.locale settings
 # System keyboard
 keyboard --xlayouts=us --vckeymap=us
 # System language

share/composer/google.ks

@@ -6,6 +6,7 @@ firewall --disabled
 # NOTE: The root account is locked by default
 # Network information
 network  --bootproto=dhcp --onboot=on --mtu=1460 --noipv6 --activate
+# NOTE: keyboard and lang can be replaced by blueprint customizations.locale settings
 # System keyboard
 keyboard --xlayouts=us --vckeymap=us
 # System language

share/composer/hyper-v.ks

@@ -6,6 +6,7 @@ firewall --enabled
 # NOTE: The root account is locked by default
 # Network information
 network  --bootproto=dhcp --onboot=on --activate
+# NOTE: keyboard and lang can be replaced by blueprint customizations.locale settings
 # System keyboard
 keyboard --xlayouts=us --vckeymap=us
 # System language

share/composer/live-iso.ks

@@ -9,6 +9,7 @@ xconfig  --startxonboot
 rootpw --plaintext removethispw
 # Network information
 network  --bootproto=dhcp --device=link --activate
+# NOTE: keyboard and lang can be replaced by blueprint customizations.locale settings
 # System keyboard
 keyboard --xlayouts=us --vckeymap=us
 # System language

share/composer/openstack.ks

@@ -6,6 +6,7 @@ firewall --disabled
 # NOTE: The root account is locked by default
 # Network information
 network  --bootproto=dhcp --onboot=on --activate
+# NOTE: keyboard and lang can be replaced by blueprint customizations.locale settings
 # System keyboard
 keyboard --xlayouts=us --vckeymap=us
 # System language

share/composer/partitioned-disk.ks

@@ -6,6 +6,7 @@ firewall --enabled
 # NOTE: The root account is locked by default
 # Network information
 network  --bootproto=dhcp --onboot=on --activate
+# NOTE: keyboard and lang can be replaced by blueprint customizations.locale settings
 # System keyboard
 keyboard --xlayouts=us --vckeymap=us
 # System language

share/composer/qcow2.ks

@@ -6,6 +6,7 @@ firewall --enabled
 # NOTE: The root account is locked by default
 # Network information
 network  --bootproto=dhcp --onboot=on --activate
+# NOTE: keyboard and lang can be replaced by blueprint customizations.locale settings
 # System keyboard
 keyboard --xlayouts=us --vckeymap=us
 # System language

share/composer/tar.ks

@@ -6,6 +6,7 @@ firewall --enabled
 # NOTE: The root account is locked by default
 # Network information
 network  --bootproto=dhcp --onboot=on --activate
+# NOTE: keyboard and lang can be replaced by blueprint customizations.locale settings
 # System keyboard
 keyboard --xlayouts=us --vckeymap=us
 # System language

share/composer/vhd.ks

@@ -6,6 +6,7 @@ firewall --enabled
 # NOTE: The root account is locked by default
 # Network information
 network  --bootproto=dhcp --onboot=on --activate
+# NOTE: keyboard and lang can be replaced by blueprint customizations.locale settings
 # System keyboard
 keyboard --xlayouts=us --vckeymap=us
 # System language

share/composer/vmdk.ks

@@ -6,6 +6,7 @@ firewall --enabled
 # NOTE: The root account is locked by default
 # Network information
 network  --bootproto=dhcp --onboot=on --activate
+# NOTE: keyboard and lang can be replaced by blueprint customizations.locale settings
 # System keyboard
 keyboard --xlayouts=us --vckeymap=us
 # System language

src/pylorax/api/compose.py

@@ -202,6 +202,85 @@ def get_timezone_settings(recipe):
     return recipe["customizations"]["timezone"]
 
 
+def lang_cmd(line, languages):
+    """ Update the lang line with the languages
+
+    :param line: The lang ... line
+    :type line: str
+    :param settings: The list of languages
+    :type settings: list
+
+    Using pykickstart to process the line is the best way to make sure it
+    is parsed correctly, and re-assembled for inclusion into the final kickstart
+    """
+    ks_version = makeVersion()
+    ks = KickstartParser(ks_version, errorsAreFatal=False, missingIncludeIsFatal=False)
+    ks.readKickstartFromString(line)
+
+    if languages:
+        ks.handler.lang.lang = languages[0]
+
+        if len(languages) > 1:
+            ks.handler.lang.addsupport = languages[1:]
+
+    # Converting back to a string includes a comment, return just the lang line
+    return str(ks.handler.lang).splitlines()[-1]
+
+
+def get_languages(recipe):
+    """Return the customizations.locale.languages list
+
+    :param recipe: The recipe
+    :type recipe: Recipe object
+    :returns: list of language strings
+    :rtype: list
+    """
+    if "customizations" not in recipe or \
+       "locale" not in recipe["customizations"] or \
+       "languages" not in recipe["customizations"]["locale"]:
+        return []
+    return recipe["customizations"]["locale"]["languages"]
+
+
+def keyboard_cmd(line, layout):
+    """ Update the keyboard line with the layout
+
+    :param line: The keyboard ... line
+    :type line: str
+    :param settings: The keyboard layout
+    :type settings: str
+
+    Using pykickstart to process the line is the best way to make sure it
+    is parsed correctly, and re-assembled for inclusion into the final kickstart
+    """
+    ks_version = makeVersion()
+    ks = KickstartParser(ks_version, errorsAreFatal=False, missingIncludeIsFatal=False)
+    ks.readKickstartFromString(line)
+
+    if layout:
+        ks.handler.keyboard.keyboard = layout
+        ks.handler.keyboard.vc_keymap = ""
+        ks.handler.keyboard.x_layouts = []
+
+    # Converting back to a string includes a comment, return just the keyboard line
+    return str(ks.handler.keyboard).splitlines()[-1]
+
+
+def get_keyboard_layout(recipe):
+    """Return the customizations.locale.keyboard list
+
+    :param recipe: The recipe
+    :type recipe: Recipe object
+    :returns: The keyboard layout string
+    :rtype: str
+    """
+    if "customizations" not in recipe or \
+       "locale" not in recipe["customizations"] or \
+       "keyboard" not in recipe["customizations"]["locale"]:
+        return []
+    return recipe["customizations"]["locale"]["keyboard"]
+
+
 def customize_ks_template(ks_template, recipe):
     """ Customize the kickstart template and return it
 
@@ -229,6 +308,12 @@ def customize_ks_template(ks_template, recipe):
                 "timezone":   [timezone_cmd,
                                get_timezone_settings(recipe),
                                'timezone UTC', False],
+                "lang":       [lang_cmd,
+                               get_languages(recipe),
+                               'lang en_US.UTF-8', True],
+                "keyboard":   [keyboard_cmd,
+                               get_keyboard_layout(recipe),
+                               'keyboard --xlayouts us --vckeymap us', True],
                }
     found = {}
 

tests/pylorax/test_compose.py

@@ -23,6 +23,7 @@ import unittest
 from pylorax import get_buildarch
 from pylorax.api.compose import add_customizations, get_extra_pkgs, compose_types
 from pylorax.api.compose import timezone_cmd, get_timezone_settings
+from pylorax.api.compose import lang_cmd, get_languages, keyboard_cmd, get_keyboard_layout
 from pylorax.api.compose import get_kernel_append, bootloader_append, customize_ks_template
 from pylorax.api.config import configure, make_dnf_dirs
 from pylorax.api.dnfbase import get_base_object
@@ -316,6 +317,78 @@ ntpservers = ["0.north-america.pool.ntp.org", "1.north-america.pool.ntp.org"]
             {"timezone": "US/Samoa", "ntpservers": ["0.pool.ntp.org", "1.pool.ntp.org"]}),
                          'timezone US/Samoa --ntpservers=0.pool.ntp.org,1.pool.ntp.org')
 
+    def test_get_languages(self):
+        """Test get_languages function"""
+        blueprint_data = """name = "test-locale"
+description = "test recipe"
+version = "0.0.1"
+        """
+        blueprint2_data = blueprint_data + """
+[customizations.locale]
+languages = ["en_CA.utf8", "en_HK.utf8"]
+"""
+        blueprint3_data = blueprint_data + """
+[customizations.locale]
+keyboard = "de (dvorak)"
+languages = ["en_CA.utf8", "en_HK.utf8"]
+"""
+        recipe = recipe_from_toml(blueprint_data)
+        self.assertEqual(get_languages(recipe), [])
+
+        recipe = recipe_from_toml(blueprint2_data)
+        self.assertEqual(get_languages(recipe), ["en_CA.utf8", "en_HK.utf8"])
+
+        recipe = recipe_from_toml(blueprint3_data)
+        self.assertEqual(get_languages(recipe), ["en_CA.utf8", "en_HK.utf8"])
+
+    def test_lang_cmd(self):
+        """Test lang_cmd function"""
+
+        self.assertEqual(lang_cmd("lang en_CA.utf8", {}), 'lang en_CA.utf8')
+        self.assertEqual(lang_cmd("lang en_US.utf8", ["en_HK.utf8"]),
+                         'lang en_HK.utf8')
+        self.assertEqual(lang_cmd("lang en_US.utf8", ["en_CA.utf8", "en_HK.utf8"]),
+                         'lang en_CA.utf8 --addsupport=en_HK.utf8')
+
+        self.assertEqual(lang_cmd("lang --addsupport en_US.utf8 en_CA.utf8",
+                         ["en_CA.utf8", "en_HK.utf8", "en_GB.utf8"]),
+                         'lang en_CA.utf8 --addsupport=en_HK.utf8,en_GB.utf8')
+
+    def test_get_keyboard_layout(self):
+        """Test get_keyboard_layout function"""
+        blueprint_data = """name = "test-locale"
+description = "test recipe"
+version = "0.0.1"
+        """
+        blueprint2_data = blueprint_data + """
+[customizations.locale]
+keyboard = "de (dvorak)"
+"""
+        blueprint3_data = blueprint_data + """
+[customizations.locale]
+keyboard = "de (dvorak)"
+languages = ["en_CA.utf8", "en_HK.utf8"]
+"""
+        recipe = recipe_from_toml(blueprint_data)
+        self.assertEqual(get_keyboard_layout(recipe), [])
+
+        recipe = recipe_from_toml(blueprint2_data)
+        self.assertEqual(get_keyboard_layout(recipe), "de (dvorak)")
+
+        recipe = recipe_from_toml(blueprint3_data)
+        self.assertEqual(get_keyboard_layout(recipe), "de (dvorak)")
+
+    def test_keyboard_cmd(self):
+        """Test lang_cmd function"""
+
+        self.assertEqual(keyboard_cmd("keyboard us", {}), "keyboard 'us'")
+        self.assertEqual(keyboard_cmd("keyboard us", "de (dvorak)"),
+                         "keyboard 'de (dvorak)'")
+
+        self.assertEqual(keyboard_cmd("keyboard --vckeymap=us --xlayouts=us,gb",
+                         "de (dvorak)"),
+                         "keyboard 'de (dvorak)'")
+
     def _checkBootloader(self, result, append_str, line_limit=0):
         """Find the bootloader line and make sure append_str is in it"""
         # Optionally check to make sure the change is at the top of the template
@@ -347,6 +420,40 @@ ntpservers = ["0.north-america.pool.ntp.org", "1.north-america.pool.ntp.org"]
             line_num += 1
         return False
 
+    def _checkLang(self, result, locales, line_limit=0):
+        """Find the lang line and make sure it is as expected"""
+        # Optionally check to make sure the change is at the top of the template
+        line_num = 0
+        for line in result.splitlines():
+            if line.startswith("lang"):
+                if all([True for n in locales if n in line]):
+                    if line_limit == 0 or line_num < line_limit:
+                        return True
+                    else:
+                        print("FAILED: lang not in the first %d lines of the output" % line_limit)
+                        return False
+                else:
+                    print("FAILED: %s not matching %s" % (locales, line))
+            line_num += 1
+        return False
+
+    def _checkKeyboard(self, result, layout, line_limit=0):
+        """Find the keyboard line and make sure it is as expected"""
+        # Optionally check to make sure the change is at the top of the template
+        line_num = 0
+        for line in result.splitlines():
+            if line.startswith("keyboard"):
+                if layout in line:
+                    if line_limit == 0 or line_num < line_limit:
+                        return True
+                    else:
+                        print("FAILED: keyboard not in the first %d lines of the output" % line_limit)
+                        return False
+                else:
+                    print("FAILED: %s not matching %s" % (layout, line))
+            line_num += 1
+        return False
+
     def test_template_defaults(self):
         """Test that customize_ks_template includes defaults correctly"""
         blueprint_data = """name = "test-kernel"
@@ -395,6 +502,10 @@ append="nosmt=force"
 [customizations.timezone]
 timezone = "US/Samoa"
 ntpservers = ["0.north-america.pool.ntp.org", "1.north-america.pool.ntp.org"]
+
+[customizations.locale]
+keyboard = "de (dvorak)"
+languages = ["en_CA.utf8", "en_HK.utf8"]
 """
         tz_dict = {"timezone": "US/Samoa", "ntpservers": ["0.north-america.pool.ntp.org", "1.north-america.pool.ntp.org"]}
         recipe = recipe_from_toml(blueprint_data)
@@ -405,6 +516,10 @@ ntpservers = ["0.north-america.pool.ntp.org", "1.north-america.pool.ntp.org"]
         self.assertEqual(sum([1 for l in result.splitlines() if l.startswith("bootloader")]), 1)
         self.assertTrue(self._checkTimezone(result, tz_dict, line_limit=2))
         self.assertEqual(sum([1 for l in result.splitlines() if l.startswith("timezone")]), 1)
+        self.assertTrue(self._checkLang(result, ["en_CA.utf8", "en_HK.utf8"], line_limit=4))
+        self.assertEqual(sum([1 for l in result.splitlines() if l.startswith("lang")]), 1)
+        self.assertTrue(self._checkKeyboard(result, "de (dvorak)", line_limit=4))
+        self.assertEqual(sum([1 for l in result.splitlines() if l.startswith("keyboard")]), 1)
 
         # Test against a kickstart with a bootloader line
         result = customize_ks_template("firewall --enabled\nbootloader --location=mbr\n", recipe)
@@ -435,6 +550,16 @@ ntpservers = ["0.north-america.pool.ntp.org", "1.north-america.pool.ntp.org"]
             if sum([1 for l in result.splitlines() if l.startswith("timezone")]) != 1:
                 errors.append(("timezone for compose_type %s failed: More than 1 entry" % compose_type, result))
 
+            if not self._checkLang(result, ["en_CA.utf8", "en_HK.utf8"]):
+                errors.append(("lang for compose_type %s failed" % compose_type, result))
+            if sum([1 for l in result.splitlines() if l.startswith("lang")]) != 1:
+                errors.append(("lang for compose_type %s failed: More than 1 entry" % compose_type, result))
+
+            if not self._checkKeyboard(result, "de (dvorak)"):
+                errors.append(("keyboard for compose_type %s failed" % compose_type, result))
+            if sum([1 for l in result.splitlines() if l.startswith("keyboard")]) != 1:
+                errors.append(("keyboard for compose_type %s failed: More than 1 entry" % compose_type, result))
+
         # Print the bad results
         for e, r in errors:
             print("%s:\n%s\n\n" % (e, r))

tests/pylorax/test_recipes.py

@@ -469,6 +469,60 @@ ntpservers = ["1.north-america.pool.ntp.org"]
         self.assertEqual(ks.handler.timezone.timezone, "US/Samoa")
         self.assertEqual(ks.handler.timezone.ntpservers, ["1.north-america.pool.ntp.org"])
 
+    def test_locale_languages(self):
+        blueprint_data = """name = "test-locale"
+description = "test recipe"
+version = "0.0.1"
+"""
+        blueprint2_data = blueprint_data + """
+[customizations.locale]
+languages = ["en_CA.utf8"]
+"""
+        blueprint3_data = blueprint_data + """
+[customizations.locale]
+languages = ["en_CA.utf8", "en_HK.utf8"]
+"""
+        ks = self._blueprint_to_ks(blueprint2_data)
+        self.assertEqual(ks.handler.lang.lang, "en_CA.utf8")
+        self.assertEqual(ks.handler.lang.addsupport, [])
+
+        ks = self._blueprint_to_ks(blueprint3_data)
+        self.assertEqual(ks.handler.lang.lang, "en_CA.utf8")
+        self.assertEqual(ks.handler.lang.addsupport, ["en_HK.utf8"])
+
+    def test_locale_keyboard(self):
+        blueprint_data = """name = "test-locale"
+description = "test recipe"
+version = "0.0.1"
+"""
+        blueprint2_data = blueprint_data + """
+[customizations.locale]
+keyboard = "us"
+"""
+        blueprint3_data = blueprint_data + """
+[customizations.locale]
+keyboard = "de (dvorak)"
+"""
+        ks = self._blueprint_to_ks(blueprint2_data)
+        self.assertEqual(ks.handler.keyboard.keyboard, "us")
+
+        ks = self._blueprint_to_ks(blueprint3_data)
+        self.assertEqual(ks.handler.keyboard.keyboard, "de (dvorak)")
+
+    def test_locale(self):
+        blueprint_data = """name = "test-locale"
+description = "test recipe"
+version = "0.0.1"
+
+[customizations.locale]
+keyboard = "de (dvorak)"
+languages = ["en_CA.utf8", "en_HK.utf8"]
+"""
+        ks = self._blueprint_to_ks(blueprint_data)
+        self.assertEqual(ks.handler.keyboard.keyboard, "de (dvorak)")
+        self.assertEqual(ks.handler.lang.lang, "en_CA.utf8")
+        self.assertEqual(ks.handler.lang.addsupport, ["en_HK.utf8"])
+
     def test_user(self):
         blueprint_data = """name = "test-user"
 description = "test recipe"