diff --git a/rteval-Allow-user-to-enter-compressed-cpu-lists-fix.patch b/rteval-Allow-user-to-enter-compressed-cpu-lists-fix.patch new file mode 100644 index 0000000..7567f28 --- /dev/null +++ b/rteval-Allow-user-to-enter-compressed-cpu-lists-fix.patch @@ -0,0 +1,133 @@ +From 3bfd38808c1caeec5844984d4c85bf430f85f470 Mon Sep 17 00:00:00 2001 +From: John Kacur +Date: Tue, 9 Aug 2022 17:32:44 -0400 +Subject: [PATCH 14/19] rteval: Allow user to enter compressed cpu-lists, fix + reporting + +Allow user to enter compressed cpu-lists (short form), +that is 0-4 instead of 0,1,2,3,4 + +Fix reporting, make sure that early reports consider offline cpus +For example +If the user specifies + +su -c './rteval-cmd -d5s --loads-cpulist=2-4 --measurement-cpulist=0,9-11' + +but cpu3 has been turned off, then we should see the following + +started 3 loads on cores 2,4 +started measurement threads on cores 0,9-11 + +and not +started 3 loads on cores 2-4 + +So, to summarize the changes here +1. Allow user to input compressed cpulists +2. Fix reporting using the shortened form of the cpulists as well +3. Adds the method online_cpulist(self, cpulist) to SysTopology that +returns the same list after removing offline cpus +4. converts one print to an f-string for --only-loads +5. Adds some more DEBUG messages + +Signed-off-by: John Kacur +--- + rteval-cmd | 37 +++++++++++++++++++++++++++---------- + rteval/systopology.py | 6 +++++- + 2 files changed, 32 insertions(+), 11 deletions(-) + +diff --git a/rteval-cmd b/rteval-cmd +index 13fd5c6950b9..6a928362828f 100755 +--- a/rteval-cmd ++++ b/rteval-cmd +@@ -49,10 +49,11 @@ from rteval import RtEval, rtevalConfig + from rteval.modules.loads import LoadModules + from rteval.modules.measurement import MeasurementModules + from rteval.version import RTEVAL_VERSION +-from rteval.systopology import CpuList, SysTopology ++from rteval.systopology import CpuList, SysTopology, collapse_cpulist + from rteval.modules.loads.kcompile import ModuleParameters + + compress_cpulist = CpuList.compress_cpulist ++expand_cpulist = CpuList.expand_cpulist + + def summarize(repfile, xslt): + """ Summarize an already existing XML report """ +@@ -199,6 +200,11 @@ def parse_options(cfg, parser, cmdargs): + + return cmd_args + ++def remove_offline(cpulist): ++ """ return cpulist in collapsed compressed form with only online cpus """ ++ tmplist = expand_cpulist(cpulist) ++ tmplist = SysTopology().online_cpulist(tmplist) ++ return collapse_cpulist(tmplist) + + + if __name__ == '__main__': +@@ -322,17 +328,29 @@ if __name__ == '__main__': + sys.exit(0) + + +- # if we only specified one set of cpus (loads or measurement) +- # default the other to the inverse of the specified list + ldcfg = config.GetSection('loads') + msrcfg = config.GetSection('measurement') ++ if ldcfg.cpulist and msrcfg.cpulist: ++ ldcfg.cpulist = remove_offline(ldcfg.cpulist) ++ msrcfg.cpulist = remove_offline(msrcfg.cpulist) ++ # if we only specified one set of cpus (loads or measurement) ++ # default the other to the inverse of the specified list + if not ldcfg.cpulist and msrcfg.cpulist: +- invlist = SysTopology().invert_cpulist(msrcfg.cpulist) +- ldcfg.cpulist = compress_cpulist(invlist) ++ tmplist = expand_cpulist(msrcfg.cpulist) ++ tmplist = SysTopology().invert_cpulist(tmplist) ++ ldcfg.cpulist = compress_cpulist(tmplist) ++ msrcfg.cpulist = remove_offline(msrcfg.cpulist) + if not msrcfg.cpulist and ldcfg.cpulist: +- invlist = SysTopology().invert_cpulist(ldcfg.cpulist) +- msrcfg.cpulist = compress_cpulist(invlist) +- ++ tmplist = expand_cpulist(ldcfg.cpulist) ++ tmplist = SysTopology().invert_cpulist(tmplist) ++ msrcfg.cpulist = compress_cpulist(tmplist) ++ ldcfg.cpulist = remove_offline(ldcfg.cpulist) ++ ++ if ldcfg.cpulist: ++ logger.log(Log.DEBUG, f"loads cpulist: {ldcfg.cpulist}") ++ # if --onlyload is specified msrcfg.cpulist is unused ++ if msrcfg.cpulist and not rtevcfg.onlyload: ++ logger.log(Log.DEBUG, f"measurement cpulist: {msrcfg.cpulist}") + logger.log(Log.DEBUG, f"workdir: {rtevcfg.workdir}") + + # if --summarize was specified then just parse the XML, print it and exit +@@ -374,8 +392,7 @@ if __name__ == '__main__': + # No reports will be created. + loadmods.Start() + nthreads = loadmods.Unleash() +- logger.log(Log.INFO, "Started %i load threads - will run for %f seconds" % ( +- nthreads, rtevcfg.duration)) ++ logger.log(Log.INFO, f"Started {nthreads} load threads - will run for {rtevcfg.duration} seconds") + logger.log(Log.INFO, "No measurements will be performed, due to the --onlyload option") + time.sleep(rtevcfg.duration) + loadmods.Stop() +diff --git a/rteval/systopology.py b/rteval/systopology.py +index ce8d02cf7318..26332c30bb0e 100644 +--- a/rteval/systopology.py ++++ b/rteval/systopology.py +@@ -329,7 +329,11 @@ class SysTopology: + + def invert_cpulist(self, cpulist): + """ return a list of online cpus not in cpulist """ +- return [c for c in self.online_cpus_str() if c not in cpulist] ++ return [c for c in self.online_cpus() if c not in cpulist] ++ ++ def online_cpulist(self, cpulist): ++ """ return a list of online cpus in cpulist """ ++ return [c for c in self.online_cpus() if c in cpulist] + + if __name__ == "__main__": + +-- +2.37.3 + diff --git a/rteval-Create-common-functions-in-CpuList-and-SysTop.patch b/rteval-Create-common-functions-in-CpuList-and-SysTop.patch new file mode 100644 index 0000000..b73ada1 --- /dev/null +++ b/rteval-Create-common-functions-in-CpuList-and-SysTop.patch @@ -0,0 +1,213 @@ +From f1c540ea023bbbcd901e6b6d27d993851eae0e9b Mon Sep 17 00:00:00 2001 +From: John Kacur +Date: Mon, 25 Jul 2022 11:14:41 -0400 +Subject: [PATCH 03/19] rteval: Create common functions in CpuList and + SysTopology + +The purpose is to remove functions out of misc and use the ones in the +file systopolgy.py + +- Add function collapse_cpulist(cpulist) outside of the CpuList + class +- Make methods longest_sequence and expand_cpulist accesible outside of + their class +- Add function online_cpus(self) to class SysTopology +- Add function online_cpus_str(self) to class SysTopology +- Add function invert_cpulist(self, cpulist) to class SysTopology +- Convert strings to f-strings for better readability +- Add a few missing docstrings to methods / functions, module etc +- Add a few more tests to the unit test + +TODO: CpuList is suited for use by SysTopology, but is not ideal for a +generic CpuList. A more generally usable CpuList should be created + +Signed-off-by: John Kacur +--- + rteval/systopology.py | 90 +++++++++++++++++++++++++++++++++++-------- + 1 file changed, 74 insertions(+), 16 deletions(-) + +diff --git a/rteval/systopology.py b/rteval/systopology.py +index e852f86e450f..ce8d02cf7318 100644 +--- a/rteval/systopology.py ++++ b/rteval/systopology.py +@@ -23,11 +23,32 @@ + # including keys needed to generate an equivalently functional executable + # are deemed to be part of the source code. + # ++""" Module for querying cpu cores and nodes """ + + import os + import os.path + import glob + ++# Utility version of collapse_cpulist that doesn't require a CpuList object ++def collapse_cpulist(cpulist): ++ """ Collapse a list of cpu numbers into a string range ++ of cpus (e.g. 0-5, 7, 9) """ ++ if len(cpulist) == 0: ++ return "" ++ idx = CpuList.longest_sequence(cpulist) ++ if idx == 0: ++ seq = str(cpulist[0]) ++ else: ++ if idx == 1: ++ seq = f"{cpulist[0]},{cpulist[idx]}" ++ else: ++ seq = f"{cpulist[0]}-{cpulist[idx]}" ++ ++ rest = collapse_cpulist(cpulist[idx+1:]) ++ if rest == "": ++ return seq ++ return ",".join((seq, rest)) ++ + def sysread(path, obj): + """ Helper function for reading system files """ + with open(os.path.join(path, obj), "r") as fp: +@@ -46,7 +67,7 @@ class CpuList: + if isinstance(cpulist, list): + self.cpulist = cpulist + elif isinstance(cpulist, str): +- self.cpulist = self.__expand_cpulist(cpulist) ++ self.cpulist = self.expand_cpulist(cpulist) + self.cpulist = self.online_cpulist(self.cpulist) + self.cpulist.sort() + +@@ -67,7 +88,7 @@ class CpuList: + return False + + @staticmethod +- def __longest_sequence(cpulist): ++ def longest_sequence(cpulist): + """ return index of last element of a sequence that steps by one """ + lim = len(cpulist) + for idx, _ in enumerate(cpulist): +@@ -83,14 +104,14 @@ class CpuList: + """ + if len(cpulist) == 0: + return "" +- idx = self.__longest_sequence(cpulist) ++ idx = self.longest_sequence(cpulist) + if idx == 0: + seq = str(cpulist[0]) + else: + if idx == 1: +- seq = "%d,%d" % (cpulist[0], cpulist[idx]) ++ seq = f"{cpulist[0]},{cpulist[idx]}" + else: +- seq = "%d-%d" % (cpulist[0], cpulist[idx]) ++ seq = f"{cpulist[0]}-{cpulist[idx]}" + + rest = self.__collapse_cpulist(cpulist[idx+1:]) + if rest == "": +@@ -98,7 +119,14 @@ class CpuList: + return ",".join((seq, rest)) + + @staticmethod +- def __expand_cpulist(cpulist): ++ def compress_cpulist(cpulist): ++ """ return a string representation of cpulist """ ++ if isinstance(cpulist[0], int): ++ return ",".join(str(e) for e in cpulist) ++ return ",".join(cpulist) ++ ++ @staticmethod ++ def expand_cpulist(cpulist): + """ expand a range string into an array of cpu numbers + don't error check against online cpus + """ +@@ -124,8 +152,8 @@ class CpuList: + def is_online(self, n): + """ check whether cpu n is online """ + if n not in self.cpulist: +- raise RuntimeError("invalid cpu number %d" % n) +- path = os.path.join(CpuList.cpupath, 'cpu%d' % n) ++ raise RuntimeError(f"invalid cpu number {n}") ++ path = os.path.join(CpuList.cpupath, f'cpu{n}') + + # Some hardware doesn't allow cpu0 to be turned off + if not os.path.exists(path + '/online') and n == 0: +@@ -240,8 +268,8 @@ class SysTopology: + return len(list(self.nodes.keys())) + + def __str__(self): +- s = "%d node system" % len(list(self.nodes.keys())) +- s += " (%d cores per node)" % (len(self.nodes[list(self.nodes.keys())[0]])) ++ s = f"{len(list(self.nodes.keys()))} node system " ++ s += f"({(len(self.nodes[list(self.nodes.keys())[0]]))} cores per node)" + return s + + def __contains__(self, node): +@@ -268,6 +296,7 @@ class SysTopology: + return n + + def getinfo(self): ++ """ Initialize class Systopology """ + nodes = glob.glob(os.path.join(SysTopology.nodepath, 'node[0-9]*')) + if nodes: + nodes.sort() +@@ -278,27 +307,56 @@ class SysTopology: + self.nodes[0] = SimNumaNode() + + def getnodes(self): ++ """ return a list of nodes """ + return list(self.nodes.keys()) + + def getcpus(self, node): ++ """ return a dictionary of cpus keyed with the node """ + return self.nodes[node].getcpulist() + ++ def online_cpus(self): ++ """ return a list of integers of all online cpus """ ++ cpulist = [] ++ for n in self.nodes: ++ cpulist += self.getcpus(n) ++ cpulist.sort() ++ return cpulist ++ ++ def online_cpus_str(self): ++ """ return a list of strings of numbers of all online cpus """ ++ cpulist = [str(cpu) for cpu in self.online_cpus()] ++ return cpulist ++ ++ def invert_cpulist(self, cpulist): ++ """ return a list of online cpus not in cpulist """ ++ return [c for c in self.online_cpus_str() if c not in cpulist] + + if __name__ == "__main__": + + def unit_test(): ++ """ unit test, run python rteval/systopology.py """ + s = SysTopology() + print(s) +- print("number of nodes: %d" % len(s)) ++ print(f"number of nodes: {len(s)}") + for n in s: +- print("node[%d]: %s" % (n.nodeid, n)) +- print("system has numa node 0: %s" % (0 in s)) +- print("system has numa node 2: %s" % (2 in s)) +- print("system has numa node 24: %s" % (24 in s)) ++ print(f"node[{n.nodeid}]: {n}") ++ print(f"system has numa node 0: {0 in s}") ++ print(f"system has numa node 2: {2 in s}") ++ print(f"system has numa node 24: {24 in s}") + + cpus = {} ++ print(f"nodes = {s.getnodes()}") + for node in s.getnodes(): + cpus[node] = s.getcpus(int(node)) +- print(f'cpus = {cpus}') ++ print(f'cpus = {cpus}') ++ ++ onlcpus = s.online_cpus() ++ print(f'onlcpus = {onlcpus}') ++ onlcpus = collapse_cpulist(onlcpus) ++ print(f'onlcpus = {onlcpus}') ++ ++ onlcpus_str = s.online_cpus_str() ++ print(f'onlcpus_str = {onlcpus_str}') + ++ print(f"invert of [ 2, 4, 5 ] = {s.invert_cpulist([2, 3, 4])}") + unit_test() +-- +2.37.3 + diff --git a/rteval-Fix-loads-cpulist-restriction.patch b/rteval-Fix-loads-cpulist-restriction.patch new file mode 100644 index 0000000..1bb4b62 --- /dev/null +++ b/rteval-Fix-loads-cpulist-restriction.patch @@ -0,0 +1,71 @@ +From 823870b5a28ead45881b3f400028b61217854dae Mon Sep 17 00:00:00 2001 +From: Valentin Schneider +Date: Fri, 5 Aug 2022 14:42:39 +0100 +Subject: [PATCH 13/19] rteval: Fix loads cpulist restriction + +A recent batch of commits, one of them being: + + 39115f0a826d ("rteval: Make use of systopology instead of misc in hackbench") + +has made the loads modules use CpuList.expand_cpulist() (which produces a +list(int)) instead of misc.expand_cpulist() (which produces a list(str)). +However, the bits handling restricting CPU affinity based on a user +argument still expects to handle a list(str), which results in: + + [DEBUG] [kcompile] node 0 has no available cpus, removing + [...] + [DEBUG] [hackbench] node 0 has no available cpus, removing + +Remove the leftover string casts. +Cyclictest is unaffected. + +Signed-off-by: Valentin Schneider +Signed-off-by: John Kacur +--- + rteval/modules/loads/hackbench.py | 2 +- + rteval/modules/loads/kcompile.py | 2 +- + rteval/modules/loads/stressng.py | 2 +- + 3 files changed, 3 insertions(+), 3 deletions(-) + +diff --git a/rteval/modules/loads/hackbench.py b/rteval/modules/loads/hackbench.py +index 538f60f00282..14e60d1ce2f6 100644 +--- a/rteval/modules/loads/hackbench.py ++++ b/rteval/modules/loads/hackbench.py +@@ -76,7 +76,7 @@ class Hackbench(CommandLineLoad): + self.cpus[n] = sysTop.getcpus(int(n)) + # if a cpulist was specified, only allow cpus in that list on the node + if self.cpulist: +- self.cpus[n] = [c for c in self.cpus[n] if str(c) in expand_cpulist(self.cpulist)] ++ self.cpus[n] = [c for c in self.cpus[n] if c in expand_cpulist(self.cpulist)] + + # track largest number of cpus used on a node + node_biggest = len(sysTop.getcpus(int(n))) +diff --git a/rteval/modules/loads/kcompile.py b/rteval/modules/loads/kcompile.py +index 7d78d0ff9cae..6faa686f81d0 100644 +--- a/rteval/modules/loads/kcompile.py ++++ b/rteval/modules/loads/kcompile.py +@@ -235,7 +235,7 @@ class Kcompile(CommandLineLoad): + + # if a cpulist was specified, only allow cpus in that list on the node + if self.cpulist: +- self.cpus[n] = [c for c in self.cpus[n] if str(c) in expand_cpulist(self.cpulist)] ++ self.cpus[n] = [c for c in self.cpus[n] if c in expand_cpulist(self.cpulist)] + + # remove nodes with no cpus available for running + for node, cpus in self.cpus.items(): +diff --git a/rteval/modules/loads/stressng.py b/rteval/modules/loads/stressng.py +index 287f4e232d17..85cb47392bcb 100644 +--- a/rteval/modules/loads/stressng.py ++++ b/rteval/modules/loads/stressng.py +@@ -68,7 +68,7 @@ class Stressng(CommandLineLoad): + cpus[n] = systop.getcpus(int(n)) + # if a cpulist was specified, only allow cpus in that list on the node + if self.cpulist: +- cpus[n] = [c for c in cpus[n] if str(c) in expand_cpulist(self.cpulist)] ++ cpus[n] = [c for c in cpus[n] if c in expand_cpulist(self.cpulist)] + + # remove nodes with no cpus available for running + for node, cpu in cpus.items(): +-- +2.37.3 + diff --git a/rteval-Make-use-of-systopology-instead-of-misc-in-cy.patch b/rteval-Make-use-of-systopology-instead-of-misc-in-cy.patch new file mode 100644 index 0000000..efccfbe --- /dev/null +++ b/rteval-Make-use-of-systopology-instead-of-misc-in-cy.patch @@ -0,0 +1,190 @@ +From d7e4b9e9c350b3572ebcf2b3dff20f21be0c21ca Mon Sep 17 00:00:00 2001 +From: John Kacur +Date: Tue, 26 Jul 2022 08:01:46 -0400 +Subject: [PATCH 08/19] rteval: Make use of systopology instead of misc in + cyclictest + +- Make use of systopology instead of misc in cyclictest +- Use f-strings instead of regular strings +- Use "with" for an open + +Signed-off-by: John Kacur +--- + rteval/modules/measurement/cyclictest.py | 67 +++++++++++++----------- + 1 file changed, 36 insertions(+), 31 deletions(-) + +diff --git a/rteval/modules/measurement/cyclictest.py b/rteval/modules/measurement/cyclictest.py +index 4c8d510c4a34..9e7f4ba25eab 100644 +--- a/rteval/modules/measurement/cyclictest.py ++++ b/rteval/modules/measurement/cyclictest.py +@@ -35,7 +35,10 @@ import math + import libxml2 + from rteval.Log import Log + from rteval.modules import rtevalModulePrototype +-from rteval.misc import expand_cpulist, online_cpus, cpuinfo ++from rteval.misc import cpuinfo ++from rteval.systopology import CpuList, SysTopology ++ ++expand_cpulist = CpuList.expand_cpulist + + class RunData: + '''class to keep instance data from a cyclictest run''' +@@ -58,14 +61,14 @@ class RunData: + self._log = logfnc + + def __str__(self): +- retval = "id: %s\n" % self.__id +- retval += "type: %s\n" % self.__type +- retval += "numsamples: %d\n" % self.__numsamples +- retval += "min: %d\n" % self.__min +- retval += "max: %d\n" % self.__max +- retval += "stddev: %f\n" % self.__stddev +- retval += "mad: %f\n" % self.__mad +- retval += "mean: %f\n" % self.__mean ++ retval = f"id: {self.__id}\n" ++ retval += f"type: {self.__type}\n" ++ retval += f"numsamples: {self.__numsamples}\n" ++ retval += f"min: {self.__min}\n" ++ retval += f"max: {self.__max}\n" ++ retval += f"stddev: {self.__stddev}\n" ++ retval += f"mad: {self.__mad}\n" ++ retval += f"mean: {self.__mean}\n" + return retval + + def get_max(self): +@@ -98,12 +101,12 @@ class RunData: + # only have 1 (or none) set the calculated values + # to zero and return + if self.__numsamples <= 1: +- self._log(Log.DEBUG, "skipping %s (%d samples)" % (self.__id, self.__numsamples)) ++ self._log(Log.DEBUG, f"skipping {self.__id} ({self.__numsamples} samples)") + self.__mad = 0 + self.__stddev = 0 + return + +- self._log(Log.INFO, "reducing %s" % self.__id) ++ self._log(Log.INFO, f"reducing {self.__id}") + total = 0 + keys = list(self.__samples.keys()) + keys.sort() +@@ -198,6 +201,7 @@ class RunData: + + + class Cyclictest(rtevalModulePrototype): ++ """ measurement module for rteval """ + def __init__(self, config, logger=None): + rtevalModulePrototype.__init__(self, 'measurement', 'cyclictest', logger) + self.__cfg = config +@@ -214,9 +218,12 @@ class Cyclictest(rtevalModulePrototype): + if self.__cfg.cpulist: + self.__cpulist = self.__cfg.cpulist + self.__cpus = expand_cpulist(self.__cpulist) ++ # Only include online cpus ++ self.__cpus = CpuList(self.__cpus).cpulist ++ self.__cpus = [str(c) for c in self.__cpus] + self.__sparse = True + else: +- self.__cpus = online_cpus() ++ self.__cpus = SysTopology().online_cpus_str() + # Get the cpuset from the environment + cpuset = os.sched_getaffinity(0) + # Convert the elements to strings +@@ -241,12 +248,12 @@ class Cyclictest(rtevalModulePrototype): + self.__cyclicdata['system'] = RunData('system', + 'system', self.__priority, + logfnc=self._log) +- self.__cyclicdata['system'].description = ("(%d cores) " % self.__numcores) + info['0']['model name'] ++ self.__cyclicdata['system'].description = (f"({self.__numcores} cores) ") + info['0']['model name'] + + if self.__sparse: +- self._log(Log.DEBUG, "system using %d cpu cores" % self.__numcores) ++ self._log(Log.DEBUG, f"system using {self.__numcores} cpu cores") + else: +- self._log(Log.DEBUG, "system has %d cpu cores" % self.__numcores) ++ self._log(Log.DEBUG, f"system has {self.__numcores} cpu cores") + self.__started = False + self.__cyclicoutput = None + self.__breaktraceval = None +@@ -273,30 +280,30 @@ class Cyclictest(rtevalModulePrototype): + + + def _WorkloadPrepare(self): +- self.__interval = 'interval' in self.__cfg and '-i%d' % int(self.__cfg.interval) or "" ++ self.__interval = 'interval' in self.__cfg and f'-i{int(self.__cfg.interval)}' or "" + + self.__cmd = ['cyclictest', + self.__interval, + '-qmu', +- '-h %d' % self.__buckets, +- "-p%d" % int(self.__priority), ++ f'-h {self.__buckets}', ++ f"-p{int(self.__priority)}", + ] + if self.__sparse: +- self.__cmd.append('-t%d' % self.__numcores) +- self.__cmd.append('-a%s' % self.__cpulist) ++ self.__cmd.append(f'-t{self.__numcores}') ++ self.__cmd.append(f'-a{self.__cpulist}') + else: + self.__cmd.append('-t') + self.__cmd.append('-a') + + if 'threads' in self.__cfg and self.__cfg.threads: +- self.__cmd.append("-t%d" % int(self.__cfg.threads)) ++ self.__cmd.append(f"-t{int(self.__cfg.threads)}") + + # Should have either breaktrace or threshold, not both + if 'breaktrace' in self.__cfg and self.__cfg.breaktrace: +- self.__cmd.append("-b%d" % int(self.__cfg.breaktrace)) ++ self.__cmd.append(f"-b{int(self.__cfg.breaktrace)}") + self.__cmd.append("--tracemark") + elif self.__cfg.threshold: +- self.__cmd.append("-b%d" % int(self.__cfg.threshold)) ++ self.__cmd.append(f"-b{int(self.__cfg.threshold)}") + + # Buffer for cyclictest data written to stdout + self.__cyclicoutput = tempfile.SpooledTemporaryFile(mode='w+b') +@@ -307,17 +314,16 @@ class Cyclictest(rtevalModulePrototype): + # Don't restart cyclictest if it is already runing + return + +- self._log(Log.DEBUG, "starting with cmd: %s" % " ".join(self.__cmd)) ++ self._log(Log.DEBUG, f'starting with cmd: {" ".join(self.__cmd)}') + self.__nullfp = os.open('/dev/null', os.O_RDWR) + + debugdir = self.__get_debugfs_mount() + if 'breaktrace' in self.__cfg and self.__cfg.breaktrace and debugdir: + # Ensure that the trace log is clean + trace = os.path.join(debugdir, 'tracing', 'trace') +- fp = open(os.path.join(trace), "w") +- fp.write("0") +- fp.flush() +- fp.close() ++ with open(os.path.join(trace), "w") as fp: ++ fp.write("0") ++ fp.flush() + + self.__cyclicoutput.seek(0) + try: +@@ -380,7 +386,7 @@ class Cyclictest(rtevalModulePrototype): + try: + index = int(vals[0]) + except: +- self._log(Log.DEBUG, "cyclictest: unexpected output: %s" % line) ++ self._log(Log.DEBUG, f"cyclictest: unexpected output: {line}") + continue + + for i, core in enumerate(self.__cpus): +@@ -420,8 +426,7 @@ class Cyclictest(rtevalModulePrototype): + + # Let the user know if max latency overshot the number of buckets + if self.__cyclicdata["system"].get_max() > self.__buckets: +- self._log(Log.ERR, "Max latency(%dus) exceeded histogram range(%dus). Skipping statistics" % +- (self.__cyclicdata["system"].get_max(), self.__buckets)) ++ self._log(Log.ERR, f'Max latency({self.__cyclicdata["system"].get_max()}us) exceeded histogram range({self.__buckets}us). Skipping statistics') + self._log(Log.ERR, "Increase number of buckets to avoid lost samples") + return rep_n + +-- +2.37.3 + diff --git a/rteval-Make-use-of-systopology-instead-of-misc-in-ha.patch b/rteval-Make-use-of-systopology-instead-of-misc-in-ha.patch new file mode 100644 index 0000000..cc3b4fe --- /dev/null +++ b/rteval-Make-use-of-systopology-instead-of-misc-in-ha.patch @@ -0,0 +1,127 @@ +From 39115f0a826de1b42fd2b879f7c702e2e9926f3c Mon Sep 17 00:00:00 2001 +From: John Kacur +Date: Mon, 25 Jul 2022 12:46:57 -0400 +Subject: [PATCH 05/19] rteval: Make use of systopology instead of misc in + hackbench + +- Make use of systopology instead of misc in hackbench +- Add a module docstring +- Make use of f-strings + +Signed-off-by: John Kacur +- Add missing "f" from f-string in __starton in hackbench.py +Signed-off-by: John Kacur +--- + rteval/modules/loads/hackbench.py | 30 ++++++++++++++---------------- + 1 file changed, 14 insertions(+), 16 deletions(-) + +diff --git a/rteval/modules/loads/hackbench.py b/rteval/modules/loads/hackbench.py +index ddd1378bac75..538f60f00282 100644 +--- a/rteval/modules/loads/hackbench.py ++++ b/rteval/modules/loads/hackbench.py +@@ -24,6 +24,7 @@ + # including keys needed to generate an equivalently functional executable + # are deemed to be part of the source code. + # ++""" Load module - run the hackbench program from rt-tests ad a load """ + + import sys + import os +@@ -34,8 +35,9 @@ import errno + from signal import SIGKILL + from rteval.modules.loads import CommandLineLoad + from rteval.Log import Log +-from rteval.misc import expand_cpulist +-from rteval.systopology import SysTopology ++from rteval.systopology import CpuList, SysTopology ++ ++expand_cpulist = CpuList.expand_cpulist + + class Hackbench(CommandLineLoad): + def __init__(self, config, logger): +@@ -46,7 +48,7 @@ class Hackbench(CommandLineLoad): + if self._donotrun: + return + +- 'calculate arguments based on input parameters' ++ # calculate arguments based on input parameters + (mem, units) = self.memsize + if units == 'KB': + mem = mem / (1024.0 * 1024.0) +@@ -58,9 +60,9 @@ class Hackbench(CommandLineLoad): + ratio = float(mem) / float(self.num_cpus) + if ratio < 0.75: + if self.__cfg.runlowmem: +- self._log(Log.WARN, "Low memory system (%f GB/core)!" % ratio) ++ self._log(Log.WARN, f"Low memory system ({ratio} GB/core)!") + else: +- self._log(Log.WARN, "Low memory system (%f GB/core)! Not running hackbench" % ratio) ++ self._log(Log.WARN, f"Low memory system ({ratio} GB/core)! Not running hackbench") + self._donotrun = True + + sysTop = SysTopology() +@@ -85,7 +87,7 @@ class Hackbench(CommandLineLoad): + for node, cpus in list(self.cpus.items()): + if not cpus: + self.nodes.remove(node) +- self._log(Log.DEBUG, "node %s has no available cpus, removing" % node) ++ self._log(Log.DEBUG, f"node {node} has no available cpus, removing") + + # setup jobs based on the number of cores available per node + self.jobs = biggest * 3 +@@ -95,7 +97,7 @@ class Hackbench(CommandLineLoad): + self.__multinodes = False + if len(self.nodes) > 1: + self.__multinodes = True +- self._log(Log.INFO, "running with multiple nodes (%d)" % len(self.nodes)) ++ self._log(Log.INFO, f"running with multiple nodes ({len(self.nodes)})") + if os.path.exists('/usr/bin/numactl') and not self.cpulist: + self.__usenumactl = True + self._log(Log.INFO, "using numactl for thread affinity") +@@ -121,7 +123,7 @@ class Hackbench(CommandLineLoad): + + self.tasks = {} + +- self._log(Log.DEBUG, "starting loop (jobs: %d)" % self.jobs) ++ self._log(Log.DEBUG, f"starting loop (jobs: {self.jobs})") + + self.started = False + +@@ -135,14 +137,14 @@ class Hackbench(CommandLineLoad): + else: + args = self.args + +- self._log(Log.DEBUG, "starting on node %s: args = %s" % (node, args)) ++ self._log(Log.DEBUG, f"starting on node {node}: args = {args}") + p = subprocess.Popen(args, + stdin=self.__nullfp, + stdout=self.__out, + stderr=self.__err) + if not p: +- self._log(Log.DEBUG, "hackbench failed to start on node %s" % node) +- raise RuntimeError("hackbench failed to start on node %s" % node) ++ self._log(Log.DEBUG, f"hackbench failed to start on node {node}") ++ raise RuntimeError(f"hackbench failed to start on node {node}") + return p + + def _WorkloadTask(self): +@@ -181,7 +183,7 @@ class Hackbench(CommandLineLoad): + + for node in self.nodes: + if node in self.tasks and self.tasks[node].poll() is None: +- self._log(Log.INFO, "cleaning up hackbench on node %s" % node) ++ self._log(Log.INFO, f"cleaning up hackbench on node {node}") + self.tasks[node].send_signal(SIGKILL) + if self.tasks[node].poll() is None: + time.sleep(2) +@@ -213,7 +215,3 @@ def ModuleParameters(): + def create(config, logger): + return Hackbench(config, logger) + +-# TODO: The following test is broken +-#if __name__ == '__main__': +-# h = Hackbench(params={'debugging':True, 'verbose':True}) +-# h.run() +-- +2.37.3 + diff --git a/rteval-Make-use-of-systopology-instead-of-misc-in-kc.patch b/rteval-Make-use-of-systopology-instead-of-misc-in-kc.patch new file mode 100644 index 0000000..7a3676a --- /dev/null +++ b/rteval-Make-use-of-systopology-instead-of-misc-in-kc.patch @@ -0,0 +1,257 @@ +From 42e5d22898571ddedd5a91bfefc47fd6354031c3 Mon Sep 17 00:00:00 2001 +From: John Kacur +Date: Mon, 25 Jul 2022 15:09:40 -0400 +Subject: [PATCH 06/19] rteval: Make use of systopology instead of misc in + kcompile + +- make use of systopology instead of misc in kcompile +- use f-strings where necessary +- Change """ to comment style # when not a docstring +- Add docstrings to functions +- Use python in (0, -2) for readability + +Signed-off-by: John Kacur +--- + rteval/modules/loads/kcompile.py | 86 +++++++++++++++++--------------- + 1 file changed, 46 insertions(+), 40 deletions(-) + +diff --git a/rteval/modules/loads/kcompile.py b/rteval/modules/loads/kcompile.py +index 2c93c794fe75..4a8659c042e6 100644 +--- a/rteval/modules/loads/kcompile.py ++++ b/rteval/modules/loads/kcompile.py +@@ -33,8 +33,10 @@ import subprocess + from rteval.modules import rtevalRuntimeError + from rteval.modules.loads import CommandLineLoad + from rteval.Log import Log +-from rteval.misc import expand_cpulist, compress_cpulist +-from rteval.systopology import SysTopology ++from rteval.systopology import CpuList, SysTopology ++ ++expand_cpulist = CpuList.expand_cpulist ++compress_cpulist = CpuList.compress_cpulist + + DEFAULT_KERNEL_PREFIX = "linux-5.18" + +@@ -48,21 +50,21 @@ class KBuildJob: + self.logger = logger + self.binder = None + self.builddir = os.path.dirname(kdir) +- self.objdir = "%s/node%d" % (self.builddir, int(node)) ++ self.objdir = f"{self.builddir}/node{int(node)}" + + if not os.path.isdir(self.objdir): + os.mkdir(self.objdir) + + if os.path.exists('/usr/bin/numactl') and not cpulist: +- """ Use numactl """ +- self.binder = 'numactl --cpunodebind %d' % int(self.node) ++ # Use numactl ++ self.binder = f'numactl --cpunodebind {int(self.node)}' + self.jobs = self.calc_jobs_per_cpu() * len(self.node) + elif cpulist: +- """ Use taskset """ ++ # Use taskset + self.jobs = self.calc_jobs_per_cpu() * len(cpulist) +- self.binder = 'taskset -c %s' % compress_cpulist(cpulist) ++ self.binder = f'taskset -c {compress_cpulist(cpulist)}' + else: +- """ Without numactl calculate number of jobs from the node """ ++ # Without numactl calculate number of jobs from the node + self.jobs = self.calc_jobs_per_cpu() * len(self.node) + + self.runcmd = f"make O={self.objdir} -C {self.kdir} -j{self.jobs}" +@@ -72,56 +74,61 @@ class KBuildJob: + self.runcmd = self.binder + " " + self.runcmd + self.cleancmd = self.binder + " " + self.cleancmd + +- self.log(Log.DEBUG, "node %d: jobs == %d" % (int(node), self.jobs)) ++ self.log(Log.DEBUG, f"node {int(node)}: jobs == {self.jobs}") + self.log(Log.DEBUG, f"cleancmd = {self.cleancmd}") +- self.log(Log.DEBUG, "node%d kcompile command: %s" \ +- % (int(node), self.runcmd)) ++ self.log(Log.DEBUG, f"node{int(node)} kcompile command: {self.runcmd}") + + def __str__(self): + return self.runcmd + + def log(self, logtype, msg): ++ """ starting logging for the kcompile module """ + if self.logger: +- self.logger.log(logtype, "[kcompile node%d] %s" % (int(self.node), msg)) ++ self.logger.log(logtype, f"[kcompile node{int(self.node)}] {msg}") + + def calc_jobs_per_cpu(self): ++ """ Calculate the number of kcompile jobs to do """ + mult = 2 +- self.log(Log.DEBUG, "calulating jobs for node %d" % int(self.node)) ++ self.log(Log.DEBUG, f"calulating jobs for node {int(self.node)}") + # get memory total in gigabytes + mem = int(self.node.meminfo['MemTotal']) / 1024.0 / 1024.0 / 1024.0 + # ratio of gigabytes to #cores + ratio = float(mem) / float(len(self.node)) +- if ratio < 1.0: +- ratio = 1.0 +- if ratio < 1.0 or ratio > 2.0: ++ ratio = max(ratio, 1.0) ++ if ratio > 2.0: + mult = 1 +- self.log(Log.DEBUG, "memory/cores ratio on node %d: %f" % (int(self.node), ratio)) +- self.log(Log.DEBUG, "returning jobs/core value of: %d" % int(ratio) * mult) ++ self.log(Log.DEBUG, f"memory/cores ratio on node {int(self.node)}: {ratio}") ++ self.log(Log.DEBUG, f"returning jobs/core value of: {int(ratio) * mult}") + return int(int(ratio) * int(mult)) + + def clean(self, sin=None, sout=None, serr=None): +- self.log(Log.DEBUG, "cleaning objdir %s" % self.objdir) ++ """ Runs command to clean any previous builds and configure kernel """ ++ self.log(Log.DEBUG, f"cleaning objdir {self.objdir}") + subprocess.call(self.cleancmd, shell=True, + stdin=sin, stdout=sout, stderr=serr) + + def run(self, sin=None, sout=None, serr=None): +- self.log(Log.INFO, "starting workload on node %d" % int(self.node)) +- self.log(Log.DEBUG, "running on node %d: %s" % (int(self.node), self.runcmd)) ++ """ Use Popen to launch a kcompile job """ ++ self.log(Log.INFO, f"starting workload on node {int(self.node)}") ++ self.log(Log.DEBUG, f"running on node {int(self.node)}: {self.runcmd}") + self.jobid = subprocess.Popen(self.runcmd, shell=True, + stdin=sin, stdout=sout, stderr=serr) + + def isrunning(self): ++ """ Query whether a job is running, returns True or False """ + if self.jobid is None: + return False + return self.jobid.poll() is None + + def stop(self): ++ """ stop a kcompile job """ + if not self.jobid: + return True + return self.jobid.terminate() + + + class Kcompile(CommandLineLoad): ++ """ class to compile the kernel as an rteval load """ + def __init__(self, config, logger): + self.buildjobs = {} + self.config = config +@@ -150,14 +157,14 @@ class Kcompile(CommandLineLoad): + def _remove_build_dirs(self): + if not os.path.isdir(self.builddir): + return +- self._log(Log.DEBUG, "removing kcompile directories in %s" % self.builddir) ++ self._log(Log.DEBUG, f"removing kcompile directories in {self.builddir}") + null = os.open("/dev/null", os.O_RDWR) + cmd = ["rm", "-rf", os.path.join(self.builddir, "kernel*"), + os.path.join(self.builddir, "node*")] + ret = subprocess.call(cmd, stdin=null, stdout=null, stderr=null) + if ret: + raise rtevalRuntimeError(self, \ +- "error removing builddir (%s) (ret=%d)" % (self.builddir, ret)) ++ f"error removing builddir ({self.buildir}) (ret={ret})") + + def _WorkloadSetup(self): + if self._donotrun: +@@ -167,15 +174,15 @@ class Kcompile(CommandLineLoad): + if self._cfg.source: + tarfile = os.path.join(self.srcdir, self._cfg.source) + if not os.path.exists(tarfile): +- raise rtevalRuntimeError(self, " tarfile %s does not exist!" % tarfile) ++ raise rtevalRuntimeError(self, f" tarfile {tarfile} does not exist!") + self.source = tarfile + kernel_prefix = re.search(r"linux-\d{1,2}\.\d{1,3}", self.source).group(0) + else: +- tarfiles = glob.glob(os.path.join(self.srcdir, "%s*" % DEFAULT_KERNEL_PREFIX)) ++ tarfiles = glob.glob(os.path.join(self.srcdir, f"{DEFAULT_KERNEL_PREFIX}*")) + if tarfiles: + self.source = tarfiles[0] + else: +- raise rtevalRuntimeError(self, " no kernel tarballs found in %s" % self.srcdir) ++ raise rtevalRuntimeError(self, f" no kernel tarballs found in {self.srcdir}") + kernel_prefix = DEFAULT_KERNEL_PREFIX + self._log(Log.DEBUG, f"kernel_prefix = {kernel_prefix}") + +@@ -190,15 +197,15 @@ class Kcompile(CommandLineLoad): + self._extract_tarball() + names = os.listdir(self.builddir) + for d in names: +- self._log(Log.DEBUG, "checking %s" % d) ++ self._log(Log.DEBUG, f"checking {d}") + if d.startswith(kernel_prefix): + kdir = d + break + if kdir is None: + raise rtevalRuntimeError(self, "Can't find kernel directory!") + self.mydir = os.path.join(self.builddir, kdir) +- self._log(Log.DEBUG, "mydir = %s" % self.mydir) +- self._log(Log.DEBUG, "systopology: %s" % self.topology) ++ self._log(Log.DEBUG, f"mydir = {self.mydir}") ++ self._log(Log.DEBUG, f"systopology: {self.topology}") + self.jobs = len(self.topology) + self.args = [] + +@@ -217,10 +224,10 @@ class Kcompile(CommandLineLoad): + for node, cpus in self.cpus.items(): + if not cpus: + self.nodes.remove(node) +- self._log(Log.DEBUG, "node %s has no available cpus, removing" % node) ++ self._log(Log.DEBUG, f"node {node} has no available cpus, removing") + + for n in self.nodes: +- self._log(Log.DEBUG, "Configuring build job for node %d" % int(n)) ++ self._log(Log.DEBUG, f"Configuring build job for node {int(n)}") + self.buildjobs[n] = KBuildJob(self.topology[n], self.mydir, \ + self.logger, self.cpus[n] if self.cpulist else None) + self.args.append(str(self.buildjobs[n])+";") +@@ -249,7 +256,7 @@ class Kcompile(CommandLineLoad): + ret = subprocess.call(cmd, stdin=null, stdout=out, stderr=err) + if ret: + # give up +- raise rtevalRuntimeError(self, "kcompile setup failed: %d" % ret) ++ raise rtevalRuntimeError(self, f"kcompile setup failed: {ret}") + except KeyboardInterrupt as m: + self._log(Log.DEBUG, "keyboard interrupt, aborting") + return +@@ -280,15 +287,14 @@ class Kcompile(CommandLineLoad): + def _WorkloadTask(self): + for n in self.nodes: + if not self.buildjobs[n]: +- raise RuntimeError("Build job not set up for node %d" % int(n)) ++ raise RuntimeError(f"Build job not set up for node {int(n)}") + if self.buildjobs[n].jobid is None or self.buildjobs[n].jobid.poll() is not None: + # A jobs was started, but now it finished. Check return code. + # -2 is returned when user forced stop of execution (CTRL-C). + if self.buildjobs[n].jobid is not None: +- if self.buildjobs[n].jobid.returncode != 0 and self.buildjobs[n].jobid.returncode != -2: +- raise RuntimeError("kcompile module failed to run (returned %d), please check logs for more detail" \ +- % self.buildjobs[n].jobid.returncode) +- self._log(Log.INFO, "Starting load on node %d" % n) ++ if self.buildjobs[n].jobid.returncode not in (0, -2): ++ raise RuntimeError(f"kcompile module failed to run (returned {self.buildjobs[n].jobid.returncode}), please check logs for more detail") ++ self._log(Log.INFO, f"Starting load on node {n}") + self.buildjobs[n].run(self.__nullfd, self.__outfd, self.__errfd) + + def WorkloadAlive(self): +@@ -296,8 +302,8 @@ class Kcompile(CommandLineLoad): + for n in self.nodes: + if self.buildjobs[n].jobid.poll() is not None: + # Check return code (see above). +- if self.buildjobs[n].jobid.returncode != 0 and self.buildjobs[n].jobid.returncode != -2: +- raise RuntimeError("kcompile module failed to run (returned %d), please check logs for more detail" % self.buildjobs[n].jobid.returncode) ++ if self.buildjobs[n].jobid.returncode not in (0, -2): ++ raise RuntimeError(f"kcompile module failed to run (returned {self.buildjobs[n].jobid.returncode}), please check logs for more detail") + return False + + return True +@@ -310,7 +316,7 @@ class Kcompile(CommandLineLoad): + self._log(Log.DEBUG, "out of stopevent loop") + for n in self.buildjobs: + if self.buildjobs[n].jobid.poll() is None: +- self._log(Log.DEBUG, "stopping job on node %d" % int(n)) ++ self._log(Log.DEBUG, f"stopping job on node {int(n)}") + self.buildjobs[n].jobid.terminate() + self.buildjobs[n].jobid.wait() + del self.buildjobs[n].jobid +-- +2.37.3 + diff --git a/rteval-Make-use-of-systopology-instead-of-misc-in-rt.patch b/rteval-Make-use-of-systopology-instead-of-misc-in-rt.patch new file mode 100644 index 0000000..24495aa --- /dev/null +++ b/rteval-Make-use-of-systopology-instead-of-misc-in-rt.patch @@ -0,0 +1,128 @@ +From c28822a02ebb88f61132163138b4decbe7b6d3d0 Mon Sep 17 00:00:00 2001 +From: John Kacur +Date: Mon, 25 Jul 2022 11:48:30 -0400 +Subject: [PATCH 04/19] rteval: Make use of systopology instead of misc in + rteval-cmd + +- convert rteval-cmd to use methods / functions in systopology instead of misc. +- strings converted to f-strings + +Signed-off-by: John Kacur +--- + rteval-cmd | 45 +++++++++++++++++++++++---------------------- + 1 file changed, 23 insertions(+), 22 deletions(-) + +diff --git a/rteval-cmd b/rteval-cmd +index c1a68bd5133b..24ff5df76883 100755 +--- a/rteval-cmd ++++ b/rteval-cmd +@@ -33,6 +33,7 @@ + # including keys needed to generate an equivalently functional executable + # are deemed to be part of the source code. + # ++""" Main module of the rteval program """ + + import sys + import os +@@ -48,9 +49,11 @@ from rteval import RtEval, rtevalConfig + from rteval.modules.loads import LoadModules + from rteval.modules.measurement import MeasurementModules + from rteval.version import RTEVAL_VERSION +-from rteval.misc import invert_cpulist, compress_cpulist ++from rteval.systopology import CpuList, SysTopology + from rteval.modules.loads.kcompile import ModuleParameters + ++compress_cpulist = CpuList.compress_cpulist ++ + def summarize(repfile, xslt): + """ Summarize an already existing XML report """ + isarchive = False +@@ -60,7 +63,7 @@ def summarize(repfile, xslt): + try: + t = tarfile.open(repfile) + except: +- print("Don't know how to summarize %s (tarfile open failed)" % repfile) ++ print(f"Don't know how to summarize {repfile} (tarfile open failed)") + return + element = None + for f in t.getnames(): +@@ -68,7 +71,7 @@ def summarize(repfile, xslt): + element = f + break + if element is None: +- print("No summary.xml found in tar archive %s" % repfile) ++ print(f"No summary.xml found in tar archive {repfile}") + return + tmp = tempfile.gettempdir() + t.extract(element, path=tmp) +@@ -172,7 +175,7 @@ def parse_options(cfg, parser, cmdargs): + + (cmd_opts, cmd_args) = parser.parse_args(args=cmdargs) + if cmd_opts.rteval___version: +- print(("rteval version %s" % RTEVAL_VERSION)) ++ print(f"rteval version {RTEVAL_VERSION}") + sys.exit(0) + + if cmd_opts.rteval___duration: +@@ -320,11 +323,13 @@ if __name__ == '__main__': + ldcfg = config.GetSection('loads') + msrcfg = config.GetSection('measurement') + if not ldcfg.cpulist and msrcfg.cpulist: +- ldcfg.cpulist = compress_cpulist(invert_cpulist(msrcfg.cpulist)) ++ invlist = SysTopology().invert_cpulist(msrcfg.cpulist) ++ ldcfg.cpulist = compress_cpulist(invlist) + if not msrcfg.cpulist and ldcfg.cpulist: +- msrcfg.cpulist = compress_cpulist(invert_cpulist(ldcfg.cpulist)) ++ invlist = SysTopology().invert_cpulist(ldcfg.cpulist) ++ msrcfg.cpulist = compress_cpulist(invlist) + +- logger.log(Log.DEBUG, "workdir: %s" % rtevcfg.workdir) ++ logger.log(Log.DEBUG, f"workdir: {rtevcfg.workdir}") + + # if --summarize was specified then just parse the XML, print it and exit + if rtevcfg.summarize or rtevcfg.rawhistogram: +@@ -343,22 +348,18 @@ if __name__ == '__main__': + print("Must be root to run rteval!") + sys.exit(-1) + +- logger.log(Log.DEBUG, '''rteval options: +- workdir: %s +- loaddir: %s +- reportdir: %s +- verbose: %s +- debugging: %s +- logging: %s +- duration: %f +- sysreport: %s''' % ( +- rtevcfg.workdir, rtevcfg.srcdir, +- rtevcfg.reportdir, rtevcfg.verbose, +- rtevcfg.debugging, rtevcfg.logging, +- rtevcfg.duration, rtevcfg.sysreport)) ++ logger.log(Log.DEBUG, f'''rteval options: ++ workdir: {rtevcfg.workdir} ++ loaddir: {rtevcfg.srcdir} ++ reportdir: {rtevcfg.reportdir} ++ verbose: {rtevcfg.verbose} ++ debugging: {rtevcfg.debugging} ++ logging: {rtevcfg.logging} ++ duration: {rtevcfg.duration} ++ sysreport: {rtevcfg.sysreport}''') + + if not os.path.isdir(rtevcfg.workdir): +- raise RuntimeError("work directory %s does not exist" % rtevcfg.workdir) ++ raise RuntimeError(f"work directory {rtevcfg.workdir} does not exist") + + + rteval = RtEval(config, loadmods, measuremods, logger) +@@ -378,7 +379,7 @@ if __name__ == '__main__': + else: + # ... otherwise, run the full measurement suite with loads + ec = rteval.Measure() +- logger.log(Log.DEBUG, "exiting with exit code: %d" % ec) ++ logger.log(Log.DEBUG, f"exiting with exit code: {ec}") + + sys.exit(ec) + except KeyboardInterrupt: +-- +2.37.3 + diff --git a/rteval-Make-use-of-systopology-instead-of-misc-in-st.patch b/rteval-Make-use-of-systopology-instead-of-misc-in-st.patch new file mode 100644 index 0000000..00e737e --- /dev/null +++ b/rteval-Make-use-of-systopology-instead-of-misc-in-st.patch @@ -0,0 +1,74 @@ +From 23684f4ab799934fed090593124a575e6ae0898a Mon Sep 17 00:00:00 2001 +From: John Kacur +Date: Mon, 25 Jul 2022 15:32:57 -0400 +Subject: [PATCH 07/19] rteval: Make use of systopology instead of misc in + stressng module + +- rteval: Make use of systopology instead of misc in stressng module +- make use of f-strings instead of regular strings + +Signed-off-by: John Kacur +--- + rteval/modules/loads/stressng.py | 15 ++++++++------- + 1 file changed, 8 insertions(+), 7 deletions(-) + +diff --git a/rteval/modules/loads/stressng.py b/rteval/modules/loads/stressng.py +index fe97189d3816..287f4e232d17 100644 +--- a/rteval/modules/loads/stressng.py ++++ b/rteval/modules/loads/stressng.py +@@ -6,8 +6,9 @@ import subprocess + import signal + from rteval.modules.loads import CommandLineLoad + from rteval.Log import Log +-from rteval.misc import expand_cpulist +-from rteval.systopology import SysTopology ++from rteval.systopology import CpuList, SysTopology ++ ++expand_cpulist = CpuList.expand_cpulist + + class Stressng(CommandLineLoad): + " This class creates a load module that runs stress-ng " +@@ -27,7 +28,7 @@ class Stressng(CommandLineLoad): + self._donotrun = False + else: + self._donotrun = True +- """ When this module runs, other load modules should not """ ++ # When this module runs, other load modules should not + self._exclusive = True + + def _WorkloadSetup(self): +@@ -50,7 +51,7 @@ class Stressng(CommandLineLoad): + + # stress-ng is only run if the user specifies an option + self.args = ['stress-ng'] +- self.args.append('--%s' % str(self.cfg.option)) ++ self.args.append(f'--{str(self.cfg.option)}') + if self.cfg.arg is not None: + self.args.append(self.cfg.arg) + if self.cfg.timeout is not None: +@@ -73,11 +74,11 @@ class Stressng(CommandLineLoad): + for node, cpu in cpus.items(): + if not cpu: + nodes.remove(node) +- self._log(Log.DEBUG, "node %s has no available cpus, removing" % node) ++ self._log(Log.DEBUG, f"node {node} has no available cpus, removing") + if self.cpulist: + for node in nodes: + cpulist = ",".join([str(n) for n in cpus[node]]) +- self.args.append('--taskset %s' % cpulist) ++ self.args.append(f'--taskset {cpulist}') + + def _WorkloadTask(self): + """ Kick of the workload here """ +@@ -85,7 +86,7 @@ class Stressng(CommandLineLoad): + # Only start the task once + return + +- self._log(Log.DEBUG, "starting with %s" % " ".join(self.args)) ++ self._log(Log.DEBUG, f'starting with {" ".join(self.args)}') + try: + self.process = subprocess.Popen(self.args, + stdout=self.__out, +-- +2.37.3 + diff --git a/rteval-Move-cpuinfo-to-systopology.py-and-delete-mis.patch b/rteval-Move-cpuinfo-to-systopology.py-and-delete-mis.patch new file mode 100644 index 0000000..0633e28 --- /dev/null +++ b/rteval-Move-cpuinfo-to-systopology.py-and-delete-mis.patch @@ -0,0 +1,207 @@ +From 29bfb90688feb6daf1d132a0c6fa784f499c9d79 Mon Sep 17 00:00:00 2001 +From: John Kacur +Date: Thu, 11 Aug 2022 13:05:42 -0400 +Subject: [PATCH 15/19] rteval: Move cpuinfo to systopology.py and delete + misc.py + +- Move the function cpuinfo() to systopology.py + Since this is the last remaining use of misc +- Delete misc.py to prevent it from being used again. + +Signed-off-by: John Kacur +--- + rteval/misc.py | 116 ----------------------- + rteval/modules/measurement/cyclictest.py | 2 +- + rteval/systopology.py | 40 ++++++++ + 3 files changed, 41 insertions(+), 117 deletions(-) + delete mode 100755 rteval/misc.py + +diff --git a/rteval/misc.py b/rteval/misc.py +deleted file mode 100755 +index a7c515b0d293..000000000000 +--- a/rteval/misc.py ++++ /dev/null +@@ -1,116 +0,0 @@ +-#!/usr/bin/python3 -tt +-# +-# Copyright (C) 2015 Clark Williams +-# Copyright (C) 2015 Red Hat, Inc. +-# +-# This program is free software; you can redistribute it and/or modify +-# it under the terms of the GNU General Public License as published by +-# the Free Software Foundation; version 2 of the License. +-# +-# This program is distributed in the hope that it will be useful, +-# but WITHOUT ANY WARRANTY; without even the implied warranty of +-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +-# GNU General Public License for more details. +-# +-# You should have received a copy of the GNU General Public License +-# along with this program; if not, write to the Free Software +-# Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA. +-''' Functions for operating on a cpulists ''' +- +- +-import os +-import glob +- +-# expand a string range into a list +-# don't error check against online cpus +-def expand_cpulist(cpulist): +- '''expand a range string into an array of cpu numbers''' +- result = [] +- for part in cpulist.split(','): +- if '-' in part: +- a, b = part.split('-') +- a, b = int(a), int(b) +- result.extend(list(range(a, b + 1))) +- else: +- a = int(part) +- result.append(a) +- return [str(i) for i in list(set(result))] +- +-def online_cpus(): +- ''' Collapse a list of cpu numbers into a string range ''' +- online_cpus = [] +- # Check for the online file with cpu1 since cpu0 can't always be offlined +- if os.path.exists('/sys/devices/system/cpu/cpu1/online'): +- for c in glob.glob('/sys/devices/system/cpu/cpu[0-9]*'): +- num = str(c.replace('/sys/devices/system/cpu/cpu', '')) +- # On some machine you can't turn off cpu0 +- if not os.path.exists(c + '/online') and num == "0": +- online_cpus.append(num) +- else: +- with open(c + '/online') as f: +- is_online = f.read().rstrip('\n') +- if is_online == "1": +- online_cpus.append(num) +- else: # use the old heuristic +- for c in glob.glob('/sys/devices/system/cpu/cpu[0-9]*'): +- num = str(c.replace('/sys/devices/system/cpu/cpu', '')) +- online_cpus.append(num) +- return online_cpus +- +-def invert_cpulist(cpulist): +- ''' return a list of online cpus not in cpulist ''' +- return [c for c in online_cpus() if c not in cpulist] +- +-def compress_cpulist(cpulist): +- ''' return a string representation of cpulist ''' +- if isinstance(cpulist[0], int): +- return ",".join(str(e) for e in cpulist) +- return ",".join(cpulist) +- +-def cpuinfo(): +- ''' return a dictionary of cpu keys with various cpu information ''' +- core = -1 +- info = {} +- with open('/proc/cpuinfo') as fp: +- for l in fp: +- l = l.strip() +- if not l: +- continue +- # Split a maximum of one time. In case a model name has ':' in it +- key, val = [i.strip() for i in l.split(':', 1)] +- if key == 'processor': +- core = val +- info[core] = {} +- continue +- info[core][key] = val +- +- for (core, pcdict) in info.items(): +- if not 'model name' in pcdict: +- # On Arm CPU implementer is present +- # Construct the model_name from the following fields +- if 'CPU implementer' in pcdict: +- model_name = [pcdict.get('CPU implementer')] +- model_name.append(pcdict.get('CPU architecture')) +- model_name.append(pcdict.get('CPU variant')) +- model_name.append(pcdict.get('CPU part')) +- model_name.append(pcdict.get('CPU revision')) +- +- # If a list item is None, remove it +- model_name = [name for name in model_name if name] +- +- # Convert the model_name list into a string +- model_name = " ".join(model_name) +- pcdict['model name'] = model_name +- else: +- pcdict['model name'] = 'Unknown' +- +- return info +- +-if __name__ == "__main__": +- +- info = cpuinfo() +- idx = sorted(info.keys()) +- for i in idx: +- print("%s: %s" % (i, info[i])) +- +- print("0: %s" % (info['0']['model name'])) +diff --git a/rteval/modules/measurement/cyclictest.py b/rteval/modules/measurement/cyclictest.py +index e235b83b49f7..ace8db438c52 100644 +--- a/rteval/modules/measurement/cyclictest.py ++++ b/rteval/modules/measurement/cyclictest.py +@@ -35,7 +35,7 @@ import math + import libxml2 + from rteval.Log import Log + from rteval.modules import rtevalModulePrototype +-from rteval.misc import cpuinfo ++from rteval.systopology import cpuinfo + from rteval.systopology import CpuList, SysTopology, collapse_cpulist + + expand_cpulist = CpuList.expand_cpulist +diff --git a/rteval/systopology.py b/rteval/systopology.py +index 26332c30bb0e..c8f85c5837aa 100644 +--- a/rteval/systopology.py ++++ b/rteval/systopology.py +@@ -54,6 +54,46 @@ def sysread(path, obj): + with open(os.path.join(path, obj), "r") as fp: + return fp.readline().strip() + ++def cpuinfo(): ++ ''' return a dictionary of cpu keys with various cpu information ''' ++ core = -1 ++ info = {} ++ with open('/proc/cpuinfo') as fp: ++ for l in fp: ++ l = l.strip() ++ if not l: ++ continue ++ # Split a maximum of one time. In case a model name has ':' in it ++ key, val = [i.strip() for i in l.split(':', 1)] ++ if key == 'processor': ++ core = val ++ info[core] = {} ++ continue ++ info[core][key] = val ++ ++ for (core, pcdict) in info.items(): ++ if not 'model name' in pcdict: ++ # On Arm CPU implementer is present ++ # Construct the model_name from the following fields ++ if 'CPU implementer' in pcdict: ++ model_name = [pcdict.get('CPU implementer')] ++ model_name.append(pcdict.get('CPU architecture')) ++ model_name.append(pcdict.get('CPU variant')) ++ model_name.append(pcdict.get('CPU part')) ++ model_name.append(pcdict.get('CPU revision')) ++ ++ # If a list item is None, remove it ++ model_name = [name for name in model_name if name] ++ ++ # Convert the model_name list into a string ++ model_name = " ".join(model_name) ++ pcdict['model name'] = model_name ++ else: ++ pcdict['model name'] = 'Unknown' ++ ++ return info ++ ++ + # + # class to provide access to a list of cpus + # +-- +2.37.3 + diff --git a/rteval-cyclictest-Reset-cpulist-from-newly-calculate.patch b/rteval-cyclictest-Reset-cpulist-from-newly-calculate.patch new file mode 100644 index 0000000..f3dbe65 --- /dev/null +++ b/rteval-cyclictest-Reset-cpulist-from-newly-calculate.patch @@ -0,0 +1,41 @@ +From 7399c8f9fdf8e1c152dde1ca86a757559dfc72ab Mon Sep 17 00:00:00 2001 +From: John Kacur +Date: Tue, 26 Jul 2022 20:58:46 -0400 +Subject: [PATCH 09/19] rteval: cyclictest: Reset cpulist from newly calculated + cpus + +After we recalculate self.__cpus to only include online cpus, we also +need to reset the self.__cpulist which is the collapsed form of the list +that we pass as parameters to cyclictest, otherwise we can potentially +pass an offline cpu as a parameter to cyclictest which will then fail + +Signed-off-by: John Kacur +--- + rteval/modules/measurement/cyclictest.py | 4 +++- + 1 file changed, 3 insertions(+), 1 deletion(-) + +diff --git a/rteval/modules/measurement/cyclictest.py b/rteval/modules/measurement/cyclictest.py +index 9e7f4ba25eab..e235b83b49f7 100644 +--- a/rteval/modules/measurement/cyclictest.py ++++ b/rteval/modules/measurement/cyclictest.py +@@ -36,7 +36,7 @@ import libxml2 + from rteval.Log import Log + from rteval.modules import rtevalModulePrototype + from rteval.misc import cpuinfo +-from rteval.systopology import CpuList, SysTopology ++from rteval.systopology import CpuList, SysTopology, collapse_cpulist + + expand_cpulist = CpuList.expand_cpulist + +@@ -220,6 +220,8 @@ class Cyclictest(rtevalModulePrototype): + self.__cpus = expand_cpulist(self.__cpulist) + # Only include online cpus + self.__cpus = CpuList(self.__cpus).cpulist ++ # Reset cpulist from the newly calculated self.__cpus ++ self.__cpulist = collapse_cpulist(self.__cpus) + self.__cpus = [str(c) for c in self.__cpus] + self.__sparse = True + else: +-- +2.37.3 + diff --git a/rteval.spec b/rteval.spec index dea5c2a..67290a3 100644 --- a/rteval.spec +++ b/rteval.spec @@ -1,6 +1,6 @@ Name: rteval Version: 3.4 -Release: 2%{?dist} +Release: 3%{?dist} Summary: Utility to evaluate system suitability for RT Linux Group: Development/Tools @@ -13,6 +13,7 @@ BuildRequires: python3-devel Requires: python3-ethtool Requires: python3-lxml Requires: python3-dmidecode >= 3.10 +Requires: python3-requests Requires: realtime-tests >= 2.1-1 Requires: rteval-loads >= 1.6-1 Requires: sysstat @@ -31,6 +32,16 @@ BuildArch: noarch #Patches Patch1: rteval-Add-option-for-downloading-kernel.patch Patch2: rteval-Add-man-page-entry-for-S-source-download-opti.patch +Patch3: rteval-Create-common-functions-in-CpuList-and-SysTop.patch +Patch4: rteval-Make-use-of-systopology-instead-of-misc-in-rt.patch +Patch5: rteval-Make-use-of-systopology-instead-of-misc-in-ha.patch +Patch6: rteval-Make-use-of-systopology-instead-of-misc-in-kc.patch +Patch7: rteval-Make-use-of-systopology-instead-of-misc-in-st.patch +Patch8: rteval-Make-use-of-systopology-instead-of-misc-in-cy.patch +Patch9: rteval-cyclictest-Reset-cpulist-from-newly-calculate.patch +Patch10: rteval-Fix-loads-cpulist-restriction.patch +Patch11: rteval-Allow-user-to-enter-compressed-cpu-lists-fix.patch +Patch12: rteval-Move-cpuinfo-to-systopology.py-and-delete-mis.patch %description The rteval script is a utility for measuring various aspects of @@ -45,6 +56,16 @@ to the screen. %setup -q %patch1 -p1 %patch2 -p1 +%patch3 -p1 +%patch4 -p1 +%patch5 -p1 +%patch6 -p1 +%patch7 -p1 +%patch8 -p1 +%patch9 -p1 +%patch10 -p1 +%patch11 -p1 +%patch12 -p1 %build %{__python3} setup.py build @@ -66,6 +87,11 @@ to the screen. %{_bindir}/rteval %changelog +* Tue Sep 13 2022 John Kacur - 3.4-3 +- Make use of systopology instead of misc everywhere +- Allow user to enter compressed form of cpulist +Resolves: rhbz#2121534 + * Mon Sep 12 2022 John Kacur - 3.4-2 - Add option for downloading the kernel to compile as a load - Add a manpage entry for the kernel download option