From f81092404244a3721e560e93cbee8fb386044ba9 Mon Sep 17 00:00:00 2001
From: Lars Karlitski <lars@karlitski.net>
Date: Tue, 25 Jun 2019 10:22:32 +0200
Subject: [PATCH] test: Output results for cockpit's log.html

Cockpit's log.html has a peculiar format, but parsing logs in html is
much easier, because it shows expandable sections for each test.
---
 test/composertest.py | 74 +++++++++++++++++++++++++++++++++++++++++---
 1 file changed, 69 insertions(+), 5 deletions(-)

diff --git a/test/composertest.py b/test/composertest.py
index 4f68bc63..d33c5c85 100644
--- a/test/composertest.py
+++ b/test/composertest.py
@@ -4,6 +4,7 @@ import argparse
 import os
 import subprocess
 import sys
+import traceback
 import unittest
 
 # import Cockpit's machinery for test VMs and its browser test API
@@ -12,8 +13,6 @@ import testvm # pylint: disable=import-error
 
 
 def print_exception(etype, value, tb):
-    import traceback
-
     # only include relevant lines
     limit = 0
     while tb and '__unittest' in tb.tb_frame.f_globals:
@@ -31,7 +30,7 @@ class ComposerTestCase(unittest.TestCase):
         self.network = testvm.VirtNetwork(0)
         self.machine = testvm.VirtMachine(self.image, networking=self.network.host(), memory_mb=2048)
 
-        print(f"Starting virtual machine '{self.image}'")
+        print("Starting virtual machine '{}'".format(self.image))
         self.machine.start()
         self.machine.wait_boot()
 
@@ -89,13 +88,78 @@ class ComposerTestCase(unittest.TestCase):
         self.assertEqual(r.returncode, 0)
 
 
+class ComposerTestResult(unittest.TestResult):
+    def name(self, test):
+        name = test.id().replace("__main__.", "")
+        if test.shortDescription():
+            name += ": " + test.shortDescription()
+        return name
+
+    def startTest(self, test):
+        super().startTest(test)
+
+        print("# ----------------------------------------------------------------------")
+        print("# ", self.name(test))
+        print("", flush=True)
+
+    def stopTest(self, test):
+        print(flush=True)
+
+    def addSuccess(self, test):
+        super().addSuccess(test)
+        print("ok {} {}".format(self.testsRun, self.name(test)))
+
+    def addError(self, test, err):
+        super().addError(test, err)
+        traceback.print_exception(*err, file=sys.stdout)
+        print("not ok {} {}".format(self.testsRun, self.name(test)))
+
+    def addFailure(self, test, err):
+        super().addError(test, err)
+        traceback.print_exception(*err, file=sys.stdout)
+        print("not ok {} {}".format(self.testsRun, self.name(test)))
+
+    def addSkip(self, test, reason):
+        super().addSkip(test, reason)
+        print("ok {} {} # SKIP {}".format(self.testsRun, self.name(test), reason))
+
+    def addExpectedFailure(self, test, err):
+        super().addExpectedFailure(test, err)
+        print("ok {} {}".format(self.testsRun, self.name(test)))
+
+    def addUnexpectedSuccess(self, test):
+        super().addUnexpectedSuccess(test)
+        print("not ok {} {}".format(self.testsRun, self.name(test)))
+
+
+class ComposerTestRunner(object):
+    """A test runner that (in combination with ComposerTestResult) outputs
+    results in a way that cockpit's log.html can read and format them.
+    """
+
+    def __init__(self, failfast=False):
+        self.failfast = failfast
+
+    def run(self, testable):
+        result = ComposerTestResult()
+        result.failfast = self.failfast
+        result.startTestRun()
+        count = testable.countTestCases()
+        print("1.." + str(count))
+        try:
+            testable(result)
+        finally:
+            result.stopTestRun()
+        return result
+
+
 def print_tests(tests):
     for test in tests:
         if isinstance(test, unittest.TestSuite):
             print_tests(test)
         elif isinstance(test, unittest.loader._FailedTest):
             name = test.id().replace("unittest.loader._FailedTest.", "")
-            print(f"Error: '{name}' does not match a test", file=sys.stderr)
+            print("Error: '{}' does not match a test".format(name), file=sys.stderr)
         else:
             print(test.id().replace("__main__.", ""))
 
@@ -119,7 +183,7 @@ def main():
         print_tests(tests)
         return 0
 
-    runner = unittest.TextTestRunner(verbosity=2, failfast=args.sit)
+    runner = ComposerTestRunner(failfast=args.sit)
     result = runner.run(tests)
 
     sys.exit(not result.wasSuccessful())