diff --git a/beakerlib.spec b/beakerlib.spec index 78a443a..1a60f03 100644 --- a/beakerlib.spec +++ b/beakerlib.spec @@ -1,13 +1,13 @@ Name: beakerlib Summary: A shell-level integration testing library Version: 1.17 -Release: 13%{?dist} +Release: 15%{?dist} License: GPLv2 Group: Development/Libraries BuildArch: noarch URL: https://github.com/%{name} Requires: nfs-utils -Requires: python2 +Requires: python3 Requires: grep Requires: sed Requires: net-tools @@ -17,7 +17,8 @@ Requires: gzip Requires: util-linux Requires: which Requires: wget -Requires: python2-lxml +Recommends: python3-lxml +Recommends: xmllint Conflicts: beakerlib-redhat < 1-30 BuildRequires: /usr/bin/pod2man @@ -44,6 +45,8 @@ Patch13: final-summary-in-rlJournalEnd.patch Patch14: extended-coloring-capabilities.patch Patch15: unified-footer.patch Patch16: rlRun-output.patch +Patch17: python2.patch +Patch18: python3.patch %prep %autosetup -p1 @@ -92,6 +95,10 @@ Files for syntax highlighting BeakerLib tests in VIM editor %{_datadir}/vim/vimfiles/after/syntax/beakerlib.vim %changelog +* Mon Jun 25 2018 Dalibor Pospisil - 1.17-15 +- migrated to python3 +- weak dependency of python3-lxml - without this the journal.xml just will not be generated + * Sat Feb 24 2018 Dalibor Pospisil - 1.17-13 - rlRun -s now waits for output logs to be flushed, bz1361246 + bz1416796 diff --git a/python2.patch b/python2.patch new file mode 100644 index 0000000..4a70c15 --- /dev/null +++ b/python2.patch @@ -0,0 +1,45 @@ +diff -u a/src/python/journal-compare.py b/python/journal-compare.py +--- a/src/python/journal-compare.py 2018-05-15 16:16:15.198835559 +0200 ++++ b/src/python/journal-compare.py 2017-10-17 23:11:48.000000000 +0200 +@@ -1,4 +1,4 @@ +-#!/usr/bin/python ++#!/usr/bin/python2 + + # Copyright (c) 2006 Red Hat, Inc. All rights reserved. This copyrighted material + # is made available to anyone wishing to use, modify, copy, or +diff -u a/src/python/journalling.py b/src/python/journalling.py +--- a/src/python/journalling.py 2018-05-15 16:16:21.517818632 +0200 ++++ b/src/python/journalling.py 2017-10-17 23:11:48.000000000 +0200 +@@ -1,4 +1,4 @@ +-#!/usr/bin/python ++#!/usr/bin/python2 + + # Authors: Jakub Heger + # Dalibor Pospisil +diff -u a/src/python/rlMemAvg.py b/src/python/rlMemAvg.py +--- a/src/python/rlMemAvg.py 2018-05-15 16:16:24.976809367 +0200 ++++ b/src/python/rlMemAvg.py 2017-10-17 23:11:48.000000000 +0200 +@@ -1,4 +1,4 @@ +-#!/usr/bin/python ++#!/usr/bin/python2 + + # Authors: Petr Muller + # +diff -u a/src/python/rlMemPeak.py b/src/python/rlMemPeak.py +--- a/src/python/rlMemPeak.py 2018-05-15 16:16:29.153798179 +0200 ++++ b/src/python/rlMemPeak.py 2017-10-17 23:11:48.000000000 +0200 +@@ -1,4 +1,4 @@ +-#!/usr/bin/python ++#!/usr/bin/python2 + + # Authors: Petr Muller + # +diff -u a/src/python/testwatcher.py b/beakerlib-1.17/src/python/testwatcher.py +--- a/src/python/testwatcher.py 2018-05-15 16:16:35.369781528 +0200 ++++ b/src/python/testwatcher.py 2017-10-17 23:11:48.000000000 +0200 +@@ -1,4 +1,4 @@ +-#!/usr/bin/python -u ++#!/usr/bin/python2 -u + # + # Authors: Jiri Jaburek + # diff --git a/python3.patch b/python3.patch new file mode 100644 index 0000000..0c49d0e --- /dev/null +++ b/python3.patch @@ -0,0 +1,510 @@ +diff -u a/src/python/daemonize.py b/src/python/new/daemonize.py +--- a/src/python/daemonize.py 2017-10-17 23:11:48.000000000 +0200 ++++ b/src/python/new/daemonize.py 2018-06-25 21:06:09.000000000 +0200 +@@ -1,4 +1,4 @@ +-#!/usr/bin/env python ++#!/usr/bin/python3 + + # Authors: Jiri Jaburek + # +@@ -18,6 +18,7 @@ + # along with this program; if not, write to the Free Software + # Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + ++from __future__ import print_function + import os, sys + + from pwd import getpwnam +@@ -96,8 +97,8 @@ + # with original stderr (in case of errors), but with new uid/gid + if ioredir: + os.open(ioredir[0], os.O_RDWR) +- os.open(ioredir[1], os.O_RDWR | os.O_CREAT | os.O_TRUNC, 0666) +- os.open(ioredir[2], os.O_RDWR | os.O_CREAT | os.O_TRUNC, 0666) ++ os.open(ioredir[1], os.O_RDWR | os.O_CREAT | os.O_TRUNC, 0o666) ++ os.open(ioredir[2], os.O_RDWR | os.O_CREAT | os.O_TRUNC, 0o666) + + os.umask(0) + +@@ -116,7 +117,7 @@ + + # argument parsing + def error(msg): +- print >> sys.stderr, "error: " + str(msg) ++ print("error: " + str(msg), file=sys.stderr) + sys.exit(1) + + parser = OptionParser(usage='%prog [options] COMMAND') +diff -u a/src/python/journal-compare.py b/src/python/new/journal-compare.py +--- a/src/python/journal-compare.py 2018-06-25 21:01:54.490910141 +0200 ++++ b/src/python/new/journal-compare.py 2018-06-25 21:06:13.000000000 +0200 +@@ -1,6 +1,6 @@ +-#!/usr/bin/python2 ++#!/usr/bin/python3 + +-# Copyright (c) 2006 Red Hat, Inc. All rights reserved. This copyrighted material ++# Copyright (c) 2006 Red Hat, Inc. All rights reserved. This copyrighted material + # is made available to anyone wishing to use, modify, copy, or + # redistribute it subject to the terms and conditions of the GNU General + # Public License v.2. +@@ -15,6 +15,7 @@ + # + # Author: Petr Muller + ++from __future__ import print_function + import xml.dom.minidom + import sys + +@@ -125,9 +126,9 @@ + self.results = {} + + def addTestResult(self, name, result): +- if not self.results.has_key(name): +- self.results[name] = Test(name) +- self.results[name].addResult(result) ++ if name not in self.results: ++ self.results[name] = Test(name) ++ self.results[name].addResult(result) + + def compare(self, other): + result_list = [] +@@ -135,7 +136,7 @@ + try: + result_list.append(self.results[key].compare(other.results[key])) + except KeyError: +- print "[WARN] Could not find corresponding test for: %s" % key ++ print("[WARN] Could not find corresponding test for: %s" % key) + return result_list + + try: +@@ -161,7 +162,7 @@ + new_type, new_name = new_phases[i].getAttribute("type"), new_phases[i].getAttribute("name") + + if old_type == new_type and old_name == new_name: +- print "Types match, so we are comparing phase %s of type %s" % (old_type, new_type) ++ print( "Types match, so we are comparing phase %s of type %s" % (old_type, new_type)) + old_tests = TestSet() + new_tests = TestSet() + old_metrics = {} +@@ -179,20 +180,20 @@ + tolerance = float(metric.getAttribute("tolerance")) + metrics[key] = Metric(key, value, metric.getAttribute("type"), tolerance) + +- print "==== Actual compare ====" +- print " * Metrics * " ++ print("==== Actual compare ====") ++ print(" * Metrics * ") + metric_results = [] + for key in old_metrics.keys(): + metric_results.append(old_metrics[key].compare(new_metrics[key])) + for metric in metric_results: + for message in metric.messages: +- print "[%s] %s (%s)" % (metric.result, metric.name, message) +- print " * Tests * " ++ print("[%s] %s (%s)" % (metric.result, metric.name, message)) ++ print(" * Tests * ") + test_results = old_tests.compare(new_tests) + for test in test_results: +- print "[%s] %s" % (test.result, test.name) ++ print("[%s] %s" % (test.result, test.name)) + for message in test.messages: +- print "\t - %s" % message ++ print("\t - %s" % message) + + else: +- print "We are not doing any compare, types dont match" ++ print("We are not doing any compare, types dont match") +diff -u a/src/python/journalling.py b/src/python/new/journalling.py +--- a/src/python/journalling.py 2018-06-25 21:01:54.490910141 +0200 ++++ b/src/python/new/journalling.py 2018-06-25 21:06:19.000000000 +0200 +@@ -1,4 +1,4 @@ +-#!/usr/bin/python2 ++#!/usr/bin/python3 + + # Authors: Jakub Heger + # Dalibor Pospisil +@@ -20,16 +20,17 @@ + # along with this program; if not, write to the Free Software + # Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + ++# TODO fix xml pretty print ++ + +-import sys + import os +-import time + import re +-from optparse import OptionParser +-from lxml import etree ++import sys ++import six ++import time + import base64 +- +-# TODO fix xml pretty print ++from lxml import etree ++from optparse import OptionParser + + + xmlForbidden = [0, 1, 2, 3, 4, 5, 6, 7, 8, 11, 12, 14, 15, 16, 17, 18, 19, 20, +@@ -51,18 +52,19 @@ + return self.items[-1] + + ++# Saves the XML journal to a file. + def saveJournal(journal, journal_path): + try: + output = open(journal_path, 'wb') + output.write(etree.tostring(journal, xml_declaration=True, encoding='utf-8', pretty_print=True)) + output.close() + return 0 +- except IOError, e: ++ except IOError as e: + sys.stderr.write('Failed to save journal to %s: %s' % (journal_path, str(e))) + return 1 + + +-# Adds attributes starttime and endtime to a element ++# Adds attributes starttime and endtime to a element. + def addStartEndTime(element, starttime, endtime): + element.set("starttime", starttime) + element.set("endtime", endtime) +@@ -72,7 +74,7 @@ + return 0 + + +-# Find first and last timestamp to fill in starttime and endtime elements of given element ++# Find first and last timestamp to fill in starttime and endtime attributes of given element. + def getStartEndTime(element): + starttime = "" + endtime = "" +@@ -87,7 +89,7 @@ + + # Parses and decodes lines given to it + # Returns number of spaces before element, name of the element, +-# its attributes in a dictionary, and content of the element ++# its attributes in a dictionary, and content of the element. + def parseLine(line): + TIME_FORMAT = "%Y-%m-%d %H:%M:%S %Z" + CONTENT_FLAG = 0 +@@ -99,12 +101,12 @@ + # Count number of leading spaces + indent = len(line) - len(line.lstrip()) + +- # splitting the line into list ++ # Splitting the line into a list + splitted = line.split() + +- # if the line is not empty ++ # If the line is not empty + if splitted: +- # if first 2 characters are '-', it is not new element, but ending of pair element ++ # If first 2 characters are '-', it is not new element, but ending of pair element + if splitted[0][0] == '-' and splitted[0][1] == '-': + element = "" + else: +@@ -113,53 +115,82 @@ + else: + return 0, "", {}, "" + +- # parsing the rest of the line ++ # Parsing the rest of the line + for part in splitted: +- # if flag is set, string is an elements content ++ # If flag is set, string is an elements content + if CONTENT_FLAG == 1: +- # First and last characters(quotes) stripped and +- # string is decoded from base64 +- content = base64.b64decode(part[1:-1]) +- # end parsing after content is stored ++ # String is decoded from base64 ++ try: ++ content = base64.b64decode(part) ++ except TypeError as e: ++ sys.stderr.write('Failed to decode string \'%s\' from base64.\ ++ \nError: %s\nExiting unsuccessfully.\n' % (part[1:-1], e)) ++ exit(1) ++ # End parsing after content is stored + break +- # test if string is an elements content indicator ++ # Test if string is an elements content indicator + if part == '--': + CONTENT_FLAG = 1 + continue +- # test if string is an elements time attribute ++ ++ # Test if string is the elements time attribute + if re.match(r'^--timestamp=', part): + attribute_name = "timestamp" +- # Value is string after '=' sign and without first abd last char(quotes) +- attribute_value = part.split('=', 1)[1][1:-1] +- attributes[attribute_name] = time.strftime(TIME_FORMAT, time.localtime(int(attribute_value))) ++ # Value is string after '=' sign ++ attribute_value = part.split('=', 1)[1] ++ try: ++ attributes[attribute_name] = time.strftime(TIME_FORMAT, time.localtime(int(attribute_value))) ++ except ValueError as e: ++ sys.stderr.write('Failed to convert timestamp attribute to int.\ ++ \nError: %s\nExiting unsuccessfully.\n' % (e)) ++ exit(1) + continue +- # test if string is an elements regular attribute ++ ++ # Test if string is the elements regular attribute + if re.match(r'^--[a-zA-Z0-9]+=', part): + attribute_name = part.split('=', 1)[0][2:] +- # Value is string after '=' sign and without first abd last char(quotes) +- attribute_value = part.split('=', 1)[1][1:-1] +- attributes[attribute_name] = base64.b64decode(attribute_value) ++ # Value is string after '=' sign ++ attribute_value = part.split('=', 1)[1] ++ try: ++ attributes[attribute_name] = base64.b64decode(attribute_value) ++ except TypeError as e: ++ sys.stderr.write('Failed to decode string \'%s\' from base64.\ ++ \nError: %s\nExiting unsuccessfully.\n' % (attribute_value, e)) ++ exit(1) + continue + + return indent, element, attributes, content + + +-# Returns xml element created with ++# Returns XML element created with + # information given as parameters + def createElement(element, attributes, content): +- element = unicode(element, 'utf-8', errors='replace').translate(xmlTrans) ++ # In python 3 decoding from base64 causes retyping into bytes. ++ if isinstance(element, bytes): ++ # First bytes are decoded from utf8. ++ element = element.decode('utf8', 'replace') ++ # And then retyped to string, using 'six' module which adds python 2/3 compatible methods. ++ # XML not compatible characters are then also stripped from the string. ++ element = six.text_type(element).translate(xmlTrans) ++ + try: + new_el = etree.Element(element) +- except ValueError, e: ++ except ValueError as e: + sys.stderr.write('Failed to create element with name %s\nError: %s\nExiting unsuccessfully.\n' % (element, e)) + exit(1) + +- content = unicode(content, 'utf-8', errors='replace').translate(xmlTrans) +- new_el.text = content +- +- for key, value in attributes.iteritems(): +- key = unicode(key, 'utf-8', errors='replace').translate(xmlTrans) +- value = unicode(value, 'utf-8', errors='replace').translate(xmlTrans) ++ if isinstance(content, bytes): ++ content = content.decode('utf8', 'replace') ++ new_el.text = six.text_type(content).translate(xmlTrans) ++ ++ for key, value in attributes.items(): ++ if isinstance(key, bytes): ++ key = key.decode('utf8', 'replace') ++ key = six.text_type(key).translate(xmlTrans) ++ ++ if isinstance(value, bytes): ++ value = value.decode('utf8', 'replace') ++ value = six.text_type(value).translate(xmlTrans) + new_el.set(key, value) + return new_el + +@@ -172,7 +203,7 @@ + if options.metafile: + try: + fh = open(options.metafile, 'r+') +- except IOError, e: ++ except IOError as e: + sys.stderr.write('Failed to open queue file with' + str(e), 'FAIL') + return 1 + +@@ -205,8 +236,8 @@ + previous_el = new_el + + elif indent == old_indent: +- # Closing element with updates to it with no elements inside it + # TODO refactor ++ # Closing element with updates to it with no elements inside it + if element == "": + # Updating start and end time + starttime, endtime = getStartEndTime(previous_el) +@@ -214,9 +245,9 @@ + if "timestamp" in attributes: + endtime = attributes["timestamp"] + # Updating attributes found on closing line +- for key, value in attributes.iteritems(): ++ for key, value in attributes.items(): + previous_el.set(key, value) +- # add start/end time and remove timestamp attribute ++ # Add start/end time and remove timestamp attribute + addStartEndTime(previous_el, starttime, endtime) + # New element is on the same level as previous one + else: +@@ -231,7 +262,7 @@ + elif indent < old_indent: + # Difference between indent levels = how many paired elements will be closed + indent_diff = old_indent - indent +- for _ in xrange(indent_diff): ++ for _ in range(indent_diff): + el_stack.peek().append(previous_el) + previous_el = el_stack.pop() + +@@ -243,9 +274,9 @@ + if "timestamp" in attributes: + endtime = attributes["timestamp"] + # Updating attributes found on closing line +- for key, value in attributes.iteritems(): ++ for key, value in attributes.items(): + previous_el.set(key, value) +- # add start/end time and remove timestamp attribute ++ # Add start/end time and remove timestamp attribute + addStartEndTime(previous_el, starttime, endtime) + + # Ending paired element and creating new one on the same level as the paired one that just ended +@@ -285,9 +316,9 @@ + xslt = etree.parse(options.xslt) + transform = etree.XSLT(xslt) + journal = transform(journal) +- except etree.LxmlError: +- sys.stderr.write("\nTransformation template file " + options.xslt + +- " could not be parsed.\nAborting journal creation.") ++ except etree.LxmlError as e: ++ sys.stderr.write("\nTransformation template file \'" + options.xslt + ++ "\' could not be parsed.\nError: %s\nAborting journal creation.") % (e) + return 1 + + if options.journal: +diff -u a/src/python/rlMemAvg.py b/src/python/new/rlMemAvg.py +--- a/src/python/rlMemAvg.py 2018-06-25 21:01:54.490910141 +0200 ++++ b/src/python/new/rlMemAvg.py 2018-06-25 21:06:24.000000000 +0200 +@@ -1,4 +1,4 @@ +-#!/usr/bin/python2 ++#!/usr/bin/python3 + + # Authors: Petr Muller + # +@@ -18,6 +18,7 @@ + # along with this program; if not, write to the Free Software + # Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + ++from __future__ import print_function + import sys, time, re + + use_sub = False +@@ -31,7 +32,7 @@ + use_popen = True + + if len(sys.argv) < 2: +- print 'syntax: rlMemAvg ' ++ print('syntax: rlMemAvg ') + sys.exit(1) + + proglist = sys.argv[1:] +@@ -59,4 +60,4 @@ + if (use_sub and finish != None) or (use_popen and finish != -1): + break + +-print "%d" % (memsum/tick) ++print("%d" % (memsum/tick)) +diff -u a/src/python/rlMemPeak.py b/src/python/new/rlMemPeak.py +--- a/src/python/rlMemPeak.py 2018-06-25 21:01:54.491910137 +0200 ++++ b/src/python/new/rlMemPeak.py 2018-06-25 21:06:28.000000000 +0200 +@@ -1,6 +1,6 @@ +-#!/usr/bin/python2 ++#!/usr/bin/python3 + +-# Authors: Petr Muller ++# Authors: Petr Muller + # + # Description: Prints a memory consumption peak of an executed program + # +@@ -18,6 +18,7 @@ + # along with this program; if not, write to the Free Software + # Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + ++from __future__ import print_function + import sys, time, re + + use_sub = False +@@ -31,7 +32,7 @@ + use_popen = True + + if len(sys.argv) < 2: +- print 'syntax: rlMemPeak ' ++ print('syntax: rlMemPeak ') + sys.exit(1) + + proglist = sys.argv[1:] +@@ -57,4 +58,4 @@ + if (use_sub and finish != None) or (use_popen and finish != -1): + break + +-print "%d" % (maxmem) ++print("%d" % (maxmem)) +diff -u a/src/python/testwatcher.py b/src/python/new/testwatcher.py +--- a/src/python/testwatcher.py 2018-06-25 21:01:54.491910137 +0200 ++++ b/src/python/new/testwatcher.py 2018-06-25 21:06:32.000000000 +0200 +@@ -1,4 +1,4 @@ +-#!/usr/bin/python2 -u ++#!/usr/bin/python3 + # + # Authors: Jiri Jaburek + # +@@ -54,6 +54,7 @@ + # and the test sends the cleanup path to the watcher again + + ++from __future__ import print_function + import os + import sys + import signal +@@ -105,12 +106,12 @@ + ### HELPERS + # + def debug(msg): +- print 'TESTWATCHER: '+msg ++ print('TESTWATCHER: '+msg) + sys.stdout.flush() + + + def fatal(msg): +- print >> sys.stderr, 'TESTWATCHER fatal: '+msg ++ print('TESTWATCHER fatal: '+msg, file=sys.stderr) + sys.stderr.flush() + sys.exit(1) + +@@ -153,13 +154,13 @@ + debug('hooking beah LWD') + try: + os.makedirs(os.path.dirname(lwd_guard_file)) +- except OSError, e: ++ except OSError as e: + if e.errno == errno.EEXIST: + pass + f = open(lwd_guard_file, 'w') + f.write(watchdog_guard_cont) + f.close() +- os.chmod(lwd_guard_file, 0755) ++ os.chmod(lwd_guard_file, 0o755) + + + # called when EWD (external watchdog) is about to expire +@@ -234,7 +235,7 @@ + try: + os.waitpid(cleanuppid, 0) + cleanuppid = 0 +- except OSError, e: ++ except OSError as e: + if e.errno == errno.EINTR: + pass + if e.errno == errno.ECHILD: +@@ -291,7 +292,7 @@ + # wait for entire process group + os.waitpid(testpid, 0) + testpid = 0 +- except OSError, e: ++ except OSError as e: + # no traceback if interrupted by a signal + if e.errno == errno.EINTR: + pass