Add tests for ltmpl.py

This covers things like installing globbed package names from multiple
repos, pinned package versions, and ltmpl functions

Related: rhbz#1548586
(cherry picked from commit a4783ba29f)

src/pylorax/ltmpl.py

@@ -1,7 +1,7 @@
 #
 # ltmpl.py
 #
-# Copyright (C) 2009-2015  Red Hat, Inc.
+# Copyright (C) 2009-2018  Red Hat, Inc.
 #
 # This program is free software; you can redistribute it and/or modify
 # it under the terms of the GNU General Public License as published by
@@ -209,7 +209,6 @@ class LoraxTemplateRunner(object):
             for pkg in debug_pkgs:
                 f.write("%s\n" % pkg)
 
-
     def run(self, templatefile, **variables):
         for k,v in list(self.defaults.items()) + list(self.builtins.items()):
             variables.setdefault(k,v)

tests/pylorax/templates/bad-template.tmpl

@@ -0,0 +1,3 @@
+<%page args="missing" />
+This is a bad template
+

tests/pylorax/templates/chmod-cmd.tmpl

@@ -0,0 +1,3 @@
+<%page />
+append /lorax-file "A text file"
+chmod /lorax-file 0641

tests/pylorax/templates/copy-cmd.tmpl

@@ -0,0 +1,3 @@
+<%page />
+append /lorax-file "A text file"
+copy /lorax-file /copied-file

tests/pylorax/templates/createaddrsize-cmd.tmpl

@@ -0,0 +1,6 @@
+<%page args="root" />
+<%
+INITRD_ADDRESS="0x02000000"
+%>
+append /initrd.img "I AM AN INITRD. TRUST ME!"
+createaddrsize ${INITRD_ADDRESS} ${root}/initrd.img ${root}/initrd.addrsize

tests/pylorax/templates/hardlink-cmd.tmpl

@@ -0,0 +1,5 @@
+<%page />
+append /lorax-file "A hardlinked file"
+mkdir /lorax-dir
+hardlink /lorax-file /linked-file
+hardlink /lorax-file /lorax-dir

tests/pylorax/templates/install-cmd.tmpl

@@ -0,0 +1,3 @@
+<%page />
+append /etc/lorax-test "TESTING LORAX TEMPLATES"
+install /etc/lorax-test /etc/lorax-test-dest

tests/pylorax/templates/install-remove-test.tmpl

@@ -0,0 +1,4 @@
+<%page />
+# Must be run after run_pkg_transaction is executed in another template
+removefrom lots-of-files --allbut /lorax-files/file-one.txt
+removepkg known-path

tests/pylorax/templates/install-test.tmpl

@@ -0,0 +1,8 @@
+<%page />
+installpkg anaconda-core
+installpkg --optional exact-1.3.17
+installpkg --except fake-homer fake-*
+installpkg --required lots-of-files
+installpkg known-path
+-installpkg missing-package
+run_pkg_transaction

tests/pylorax/templates/installimg-cmd.tmpl

@@ -0,0 +1,6 @@
+<%page />
+mkdir /product
+append /product/a-file "TEXT INSIDE A FILE"
+mkdir /images
+installimg /product images/product.img
+installimg --gzip -3 /product images/product.img.gz

tests/pylorax/templates/installinitrd-cmd.tmpl

@@ -0,0 +1,4 @@
+<%page />
+append /lorax-initrd "NOT REALLY AN INITRD"
+mkdir /kernels/
+installinitrd images /lorax-initrd /kernels/initrd.img

tests/pylorax/templates/installkernel-cmd.tmpl

@@ -0,0 +1,4 @@
+<%page />
+append /lorax-kernel "NOT REALLY A KERNEL"
+mkdir /kernels/
+installkernel images /lorax-kernel /kernels/vmlinuz

tests/pylorax/templates/installupgradeinitrd-cmd.tmpl

@@ -0,0 +1,4 @@
+<%page />
+append /lorax-upgrade "NOT REALLY AN INITRD"
+mkdir /kernels/
+installupgradeinitrd images /lorax-upgrade /kernels/upgrade.img

tests/pylorax/templates/mkdir-cmd.tmpl

@@ -0,0 +1,2 @@
+<%page />
+mkdir /etc/lorax-mkdir

tests/pylorax/templates/move-cmd.tmpl

@@ -0,0 +1,3 @@
+<%page />
+append /lorax-file "A text file"
+move /lorax-file /moved-file

tests/pylorax/templates/parse-test.tmpl

@@ -0,0 +1,10 @@
+<%page args="basearch"/>
+
+installpkg common-package
+installpkg foo-{one,two}
+
+%if basearch != "s390x":
+installpkg not-s390x-package
+%endif
+
+run_pkg_transaction

tests/pylorax/templates/remove-cmd.tmpl

@@ -0,0 +1,3 @@
+<%page />
+append /lorax-file "A text file"
+remove /lorax-*

tests/pylorax/templates/removekmod-cmd.tmpl

@@ -0,0 +1,10 @@
+<%page />
+mkdir /lib/modules/1.2.3/kernel/sound
+mkdir /lib/modules/1.2.3/kernel/drivers/video
+append /lib/modules/1.2.3/kernel/sound/foo1.ko "I AM A DRIVER"
+append /lib/modules/1.2.3/kernel/sound/foo2.ko "I AM A DRIVER"
+append /lib/modules/1.2.3/kernel/drivers/video/bar1.ko "I AM A DRIVER"
+append /lib/modules/1.2.3/kernel/drivers/video/bar2.ko "I AM A DRIVER"
+
+removekmod sound
+removekmod drivers/video --allbut bar1

tests/pylorax/templates/replace-cmd.tmpl

@@ -0,0 +1,3 @@
+<%page />
+append /etc/lorax-replace "Running @VERSION@ for lorax"
+replace @VERSION@ 1.2.3 /etc/lorax-replace

tests/pylorax/templates/runcmd-cmd.tmpl

@@ -0,0 +1,2 @@
+<%page args="root" />
+runcmd touch ${root}/lorax-runcmd

tests/pylorax/templates/symlink-cmd.tmpl

@@ -0,0 +1,5 @@
+<%page />
+append /lorax-file "A symlinked file"
+append /existing-file "An existing file"
+symlink /lorax-file /symlinked-file
+symlink /lorax-file /existing-file

tests/pylorax/templates/systemctl-cmd.tmpl

@@ -0,0 +1,4 @@
+<%page />
+mkdir /etc/systemd/system/
+append /etc/systemd/system/foo.service "[unit]\nDescription=foo\n[Install]\nWantedBy=multi-user.target"
+systemctl enable foo.service

tests/pylorax/templates/treeinfo-cmd.tmpl

@@ -0,0 +1,2 @@
+<%page />
+treeinfo images boot.iso images/boot.iso

tests/pylorax/templates/unknown-cmd.tmpl

@@ -0,0 +1,2 @@
+<%page />
+run_unknonwn_command

tests/pylorax/test_ltmpl.py

@@ -0,0 +1,278 @@
+#
+# Copyright (C) 2018  Red Hat, Inc.
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+#
+from contextlib import contextmanager
+import os
+from rpmfluff import SimpleRpmBuild, SourceFile, expectedArch
+import shutil
+import tempfile
+import unittest
+
+from pylorax.dnfbase import get_dnf_base_object
+from pylorax.ltmpl import LoraxTemplate, LoraxTemplateRunner
+from pylorax.ltmpl import brace_expand, split_and_expand, rglob, rexists
+from pylorax.sysutils import joinpaths
+
+class TemplateFunctionsTestCase(unittest.TestCase):
+    def test_brace_expand(self):
+        """Test expanding braces"""
+        self.assertEqual(list(brace_expand("foo")), ["foo"])
+        self.assertEqual(list(brace_expand("${foo}")), ["${foo}"])
+        self.assertEqual(list(brace_expand("${foo,bar}")), ["$foo", "$bar"])
+        self.assertEqual(list(brace_expand("foo {one,two,three,four}")), ["foo one", "foo two", "foo three", "foo four"])
+
+    def test_split_and_expand(self):
+        """Test splitting lines and expanding braces"""
+        self.assertEqual(list(split_and_expand("foo bar")), ["foo", "bar"])
+        self.assertEqual(list(split_and_expand("foo bar-{one,two}")), ["foo", "bar-one", "bar-two"])
+        self.assertEqual(list(split_and_expand("foo 'bar {one,two}'")), ["foo", "bar one", "bar two"])
+        self.assertEqual(list(split_and_expand('foo "bar {one,two}"')), ["foo", "bar one", "bar two"])
+
+    def test_rglob(self):
+        """Test rglob function"""
+        self.assertEqual(list(rglob("*http*toml", "./tests/pylorax/blueprints", fatal=False)), ["example-http-server.toml"])
+        self.assertEqual(list(rglob("einstein", "./tests/pylorax/blueprints", fatal=False)), [])
+        with self.assertRaises(IOError):
+            list(rglob("einstein", "./tests/pylorax/blueprints", fatal=True))
+
+    def test_rexists(self):
+        """Test rexists function"""
+        self.assertTrue(rexists("*http*toml", "./tests/pylorax/blueprints"))
+        self.assertFalse(rexists("einstein", "./tests/pylorax/blueprints"))
+
+class LoraxTemplateTestCase(unittest.TestCase):
+    @classmethod
+    def setUpClass(self):
+        self.templates = LoraxTemplate(["./tests/pylorax/templates/"])
+
+    def test_parse_template_x86_64(self):
+        """Test LoraxTemplate.parse() with basearch set to x86_64"""
+        commands = self.templates.parse("parse-test.tmpl", {"basearch": "x86_64"})
+        self.assertEqual(commands, [['installpkg', 'common-package'],
+                                    ['installpkg', 'foo-one', 'foo-two'],
+                                    ['installpkg', 'not-s390x-package'],
+                                    ['run_pkg_transaction']])
+
+    def test_parse_template_s390x(self):
+        """Test LoraxTemplate.parse() with basearch set to s390x"""
+        commands = self.templates.parse("parse-test.tmpl", {"basearch": "s390x"})
+        self.assertEqual(commands, [['installpkg', 'common-package'],
+                                    ['installpkg', 'foo-one', 'foo-two'],
+                                    ['run_pkg_transaction']])
+
+@contextmanager
+def in_tempdir(prefix='tmp'):
+    """Execute a block of code with chdir in a temporary location"""
+    oldcwd = os.getcwd()
+    tmpdir = tempfile.mkdtemp(prefix=prefix)
+    os.chdir(tmpdir)
+    yield
+    os.chdir(oldcwd)
+    shutil.rmtree(tmpdir)
+
+def makeFakeRPM(repo_dir, name, epoch, version, release, files=None):
+    """Make a fake rpm file in repo_dir"""
+    p = SimpleRpmBuild(name, version, release)
+    if epoch:
+        p.epoch = epoch
+    if not files:
+        p.add_simple_payload_file_random()
+    else:
+        # Make a number of fake file entries in the rpm
+        for f in files:
+            p.add_installed_file(
+                installPath = f,
+                sourceFile = SourceFile(os.path.basename(f), "THIS IS A FAKE FILE"))
+    with in_tempdir("lorax-test-rpms."):
+        p.make()
+        rpmfile = p.get_built_rpm(expectedArch)
+        shutil.move(rpmfile, repo_dir)
+
+class LoraxTemplateRunnerTestCase(unittest.TestCase):
+    @classmethod
+    def setUpClass(self):
+        # Create 2 repositories with rpmfluff
+        self.repo1_dir = tempfile.mkdtemp(prefix="lorax.test.repo.")
+        makeFakeRPM(self.repo1_dir, "anaconda-core", 0, "0.0.1", "1")
+        makeFakeRPM(self.repo1_dir, "exact", 0, "1.3.17", "1")
+        makeFakeRPM(self.repo1_dir, "fake-milhouse", 0, "1.0.0", "1")
+        makeFakeRPM(self.repo1_dir, "fake-bart", 2, "1.13.0", "6")
+        makeFakeRPM(self.repo1_dir, "fake-homer", 0, "0.4.0", "2")
+        makeFakeRPM(self.repo1_dir, "lots-of-files", 0, "0.1.1", 1,
+                    ["/lorax-files/file-one.txt",
+                     "/lorax-files/file-two.txt",
+                     "/lorax-files/file-three.txt"])
+        makeFakeRPM(self.repo1_dir, "known-path", 0, "0.1.8", 1, ["/known-path/file-one.txt"])
+        os.system("createrepo_c " + self.repo1_dir)
+
+        self.repo2_dir = tempfile.mkdtemp(prefix="lorax.test.repo.")
+        makeFakeRPM(self.repo2_dir, "fake-milhouse", 0, "1.3.0", "1")
+        makeFakeRPM(self.repo2_dir, "fake-lisa", 0, "1.2.0", "1")
+        os.system("createrepo_c " + self.repo2_dir)
+
+        self.repo3_dir = tempfile.mkdtemp(prefix="lorax.test.debug.repo.")
+        makeFakeRPM(self.repo3_dir, "fake-marge", 0, "2.3.0", "1", ["/fake-marge/file-one.txt"])
+        makeFakeRPM(self.repo3_dir, "fake-marge-debuginfo", 0, "2.3.0", "1", ["/fake-marge/file-one-debuginfo.txt"])
+        os.system("createrepo_c " + self.repo3_dir)
+
+        # Get a dbo with just these repos
+
+        # Setup a template runner
+        self.root_dir = tempfile.mkdtemp(prefix="lorax.test.repo.")
+        sources = ["file://"+self.repo1_dir, "file://"+self.repo2_dir, "file://"+self.repo3_dir]
+        self.dnfbase = get_dnf_base_object(self.root_dir, sources,
+                                           enablerepos=[], disablerepos=[])
+
+        self.runner = LoraxTemplateRunner(inroot=self.root_dir,
+                                          outroot=self.root_dir,
+                                          dbo=self.dnfbase,
+                                          templatedir="./tests/pylorax/templates")
+
+    @classmethod
+    def tearDownClass(self):
+        shutil.rmtree(self.repo1_dir)
+        shutil.rmtree(self.repo2_dir)
+        shutil.rmtree(self.root_dir)
+
+    def test_00_runner_multi_repo(self):
+        """Test installing packages with updates in a 2nd repo"""
+        # If this does not raise an error it means that:
+        #   Installing a named package works (anaconda-core)
+        #   Installing a pinned package works (exact-1.3.17)
+        #   Installing a globbed set of package names from multiple repos works
+        #   removepkg removes a package's files
+        #   removefrom removes some, but not all, of a package's files
+        #
+        # These all need to be done in one template because run_pkg_transaction can only run once
+        self.runner.run("install-test.tmpl")
+        self.runner.run("install-remove-test.tmpl")
+        self.assertFalse(os.path.exists(joinpaths(self.root_dir, "/known-path/file-one.txt")))
+        self.assertTrue(os.path.exists(joinpaths(self.root_dir, "/lorax-files/file-one.txt")))
+        self.assertFalse(os.path.exists(joinpaths(self.root_dir, "/lorax-files/file-two.txt")))
+
+        # Check the debug log
+        self.assertTrue(os.path.exists(joinpaths(self.root_dir, "/root/debug-pkgs.log")))
+
+    def test_install_file(self):
+        """Test append, and install template commands"""
+        self.runner.run("install-cmd.tmpl")
+        self.assertTrue(os.path.exists(joinpaths(self.root_dir, "/etc/lorax-test")))
+        self.assertEqual(open(joinpaths(self.root_dir, "/etc/lorax-test")).read(), "TESTING LORAX TEMPLATES\n")
+        self.assertTrue(os.path.exists(joinpaths(self.root_dir, "/etc/lorax-test-dest")))
+
+    def test_installimg(self):
+        """Test installimg template command"""
+        self.runner.run("installimg-cmd.tmpl")
+        self.assertTrue(os.path.exists(joinpaths(self.root_dir, "images/product.img")))
+
+    def test_mkdir(self):
+        """Test mkdir template command"""
+        self.runner.run("mkdir-cmd.tmpl")
+        self.assertTrue(os.path.isdir(joinpaths(self.root_dir, "/etc/lorax-mkdir")))
+
+    def test_replace(self):
+        """Test append, and replace template command"""
+        self.runner.run("replace-cmd.tmpl")
+        self.assertTrue(os.path.exists(joinpaths(self.root_dir, "/etc/lorax-replace")))
+        self.assertEqual(open(joinpaths(self.root_dir, "/etc/lorax-replace")).read(), "Running 1.2.3 for lorax\n")
+
+    def test_treeinfo(self):
+        """Test treeinfo template command"""
+        self.runner.run("treeinfo-cmd.tmpl")
+        self.assertEqual(self.runner.results.treeinfo["images"]["boot.iso"], "images/boot.iso")
+
+    def test_installkernel(self):
+        """Test installkernel template command"""
+        self.runner.run("installkernel-cmd.tmpl")
+        self.assertTrue(os.path.exists(joinpaths(self.root_dir, "/kernels/vmlinuz")))
+        self.assertEqual(self.runner.results.treeinfo["images"]["kernel"], "/kernels/vmlinuz")
+
+    def test_installinitrd(self):
+        """Test installinitrd template command"""
+        self.runner.run("installinitrd-cmd.tmpl")
+        self.assertTrue(os.path.exists(joinpaths(self.root_dir, "/kernels/initrd.img")))
+        self.assertEqual(self.runner.results.treeinfo["images"]["initrd"], "/kernels/initrd.img")
+
+    def test_installupgradeinitrd(self):
+        """Test installupgraedinitrd template command"""
+        self.runner.run("installupgradeinitrd-cmd.tmpl")
+        self.assertTrue(os.path.exists(joinpaths(self.root_dir, "/kernels/upgrade.img")))
+        self.assertEqual(self.runner.results.treeinfo["images"]["upgrade"], "/kernels/upgrade.img")
+
+    def test_hardlink(self):
+        """Test hardlink template command"""
+        self.runner.run("hardlink-cmd.tmpl")
+        self.assertTrue(os.path.exists(joinpaths(self.root_dir, "/linked-file")))
+        self.assertTrue(os.path.exists(joinpaths(self.root_dir, "/lorax-dir/lorax-file")))
+
+    def test_symlink(self):
+        """Test symlink template command"""
+        self.runner.run("symlink-cmd.tmpl")
+        self.assertTrue(os.path.islink(joinpaths(self.root_dir, "/symlinked-file")))
+
+    def test_copy(self):
+        """Test copy template command"""
+        self.runner.run("copy-cmd.tmpl")
+        self.assertTrue(os.path.exists(joinpaths(self.root_dir, "/copied-file")))
+
+    def test_move(self):
+        """Test move template command"""
+        self.runner.run("move-cmd.tmpl")
+        self.assertFalse(os.path.exists(joinpaths(self.root_dir, "/lorax-file")))
+        self.assertTrue(os.path.exists(joinpaths(self.root_dir, "/moved-file")))
+
+    def test_remove(self):
+        """Test remove template command"""
+        self.runner.run("remove-cmd.tmpl")
+        self.assertFalse(os.path.exists(joinpaths(self.root_dir, "/lorax-file")))
+
+    def test_chmod(self):
+        """Test chmod template command"""
+        self.runner.run("chmod-cmd.tmpl")
+        self.assertEqual(os.stat(joinpaths(self.root_dir, "/lorax-file")).st_mode, 0o100641)
+
+    def test_runcmd(self):
+        """Test runcmd template command"""
+        self.runner.run("runcmd-cmd.tmpl", root=self.root_dir)
+        self.assertTrue(os.path.exists(joinpaths(self.root_dir, "/lorax-runcmd")))
+
+    def test_removekmod(self):
+        """Test removekmod template command"""
+        self.runner.run("removekmod-cmd.tmpl")
+        self.assertTrue(os.path.exists(joinpaths(self.root_dir, "/lib/modules/1.2.3/kernel/drivers/video/bar1.ko")))
+        self.assertFalse(os.path.exists(joinpaths(self.root_dir, "/lib/modules/1.2.3/kernel/drivers/video/bar2.ko")))
+        self.assertFalse(os.path.exists(joinpaths(self.root_dir, "/lib/modules/1.2.3/kernel/sound/foo1.ko")))
+        self.assertFalse(os.path.exists(joinpaths(self.root_dir, "/lib/modules/1.2.3/kernel/sound/foo2.ko")))
+
+    def test_createaddrsize(self):
+        """Test createaddrsize template command"""
+        self.runner.run("createaddrsize-cmd.tmpl", root=self.root_dir)
+        self.assertTrue(os.path.exists(joinpaths(self.root_dir, "/initrd.addrsize")))
+
+    def test_systemctl(self):
+        """Test systemctl template command"""
+        self.runner.run("systemctl-cmd.tmpl")
+        self.assertTrue(os.path.islink(joinpaths(self.root_dir, "/etc/systemd/system/multi-user.target.wants/foo.service")))
+
+    def test_bad_template(self):
+        """Test parsing a bad template"""
+        with self.assertRaises(Exception):
+            self.runner.run("bad-template.tmpl")
+
+    def test_unknown_cmd(self):
+        """Test a template with an unknown command"""
+        with self.assertRaises(ValueError):
+            self.runner.run("unknown-cmd.tmpl")