From 26ed496843fe85f0ac9a165cb958e7563aba90e6 Mon Sep 17 00:00:00 2001 From: John Kacur Date: Tue, 7 May 2024 14:19:11 -0400 Subject: [PATCH] Rebase rteval to upstream rteval-3.8 Resolves: RHEL-30165 Signed-off-by: John Kacur --- .gitignore | 1 + rteval-Add-relative-cpulists-for-loads.patch | 70 -- ...d-relative-cpulists-for-measurements.patch | 278 -------- ...tla-timerlat-as-a-measurement-module.patch | 189 ------ ...l-Add-summary-reporting-for-timerlat.patch | 178 ----- ...ange-the-default-kernel-for-kcompile.patch | 65 -- ...al-Convert-CpuList-class-to-a-module.patch | 616 ------------------ ...t.py-Make-standalone-file-work-again.patch | 88 --- rteval-Disable-use-of-python-dmidecode.patch | 87 --- ...raw-histogram-data-for-an-existing-t.patch | 69 -- ...-Implement-initial-dmidecode-support.patch | 299 --------- ...-output-work-with-new-dmiinfo-format.patch | 60 -- rteval-Makefile-More-rpm-cleanups.patch | 64 -- ...-Minor-improvements-to-CpuList-class.patch | 85 --- ...ctor-collapse_cpulist-in-systopology.patch | 225 ------- rteval-Remove-upstream-spec-file.patch | 566 ---------------- ...nstead-of-setdefault-for-calculating.patch | 45 -- ...t.py-Fix-the-description-in-the-xml-.patch | 45 -- ...ictest.py-Fix-the-median-calculation.patch | 60 -- ...ctest.py-Remove-unused-method-sample.patch | 32 - ...t.py-reduce-Fix-exception-with-missi.patch | 35 - ...t.py-Add-statistics-and-generate-xml.patch | 358 ---------- rteval.spec | 50 +- sources | 4 +- 24 files changed, 8 insertions(+), 3561 deletions(-) delete mode 100644 rteval-Add-relative-cpulists-for-loads.patch delete mode 100644 rteval-Add-relative-cpulists-for-measurements.patch delete mode 100644 rteval-Add-rtla-timerlat-as-a-measurement-module.patch delete mode 100644 rteval-Add-summary-reporting-for-timerlat.patch delete mode 100644 rteval-Change-the-default-kernel-for-kcompile.patch delete mode 100644 rteval-Convert-CpuList-class-to-a-module.patch delete mode 100644 rteval-Cyclictest.py-Make-standalone-file-work-again.patch delete mode 100644 rteval-Disable-use-of-python-dmidecode.patch delete mode 100644 rteval-Generate-raw-histogram-data-for-an-existing-t.patch delete mode 100644 rteval-Implement-initial-dmidecode-support.patch delete mode 100644 rteval-Make-output-work-with-new-dmiinfo-format.patch delete mode 100644 rteval-Makefile-More-rpm-cleanups.patch delete mode 100644 rteval-Minor-improvements-to-CpuList-class.patch delete mode 100644 rteval-Refactor-collapse_cpulist-in-systopology.patch delete mode 100644 rteval-Remove-upstream-spec-file.patch delete mode 100644 rteval-Use-get-instead-of-setdefault-for-calculating.patch delete mode 100644 rteval-cyclictest.py-Fix-the-description-in-the-xml-.patch delete mode 100644 rteval-cyclictest.py-Fix-the-median-calculation.patch delete mode 100644 rteval-cyclictest.py-Remove-unused-method-sample.patch delete mode 100644 rteval-cyclictest.py-reduce-Fix-exception-with-missi.patch delete mode 100644 rteval-timerlat.py-Add-statistics-and-generate-xml.patch diff --git a/.gitignore b/.gitignore index 6f910b8..1224ee2 100644 --- a/.gitignore +++ b/.gitignore @@ -6,3 +6,4 @@ /rteval-3.7.tar.xz /0001-rteval-Change-the-default-kernel-for-kcompile-to-lin.patch /0002-rteval-Remove-upstream-spec-file.patch +/rteval-3.8.tar.xz diff --git a/rteval-Add-relative-cpulists-for-loads.patch b/rteval-Add-relative-cpulists-for-loads.patch deleted file mode 100644 index 2f45c89..0000000 --- a/rteval-Add-relative-cpulists-for-loads.patch +++ /dev/null @@ -1,70 +0,0 @@ -From 636701e66cb98b979948b7a47320809c734e2a9e Mon Sep 17 00:00:00 2001 -From: Tomas Glozar -Date: Thu, 4 Apr 2024 10:14:38 +0200 -Subject: [PATCH] rteval: Add relative cpulists for loads - -Relative cpulists were added for measurements in 64ce7848 ("rteval: Add -relative cpulists for measurements"). It was observed since that this -feature would also be useful for load cpulists, for example when the -measurements are performed externally and rteval is only used to run -loads. - -Add support for relative cpulists also for loads. This works the same -way as for measurements using parse_cpulist_from_config, only difference -is there is no --loads-run-on-isolcpus option. That is, --loads-cpulist -now also takes lists with addition (+) and removal (-) of CPUs against -the default list, e.g. +0,1,-7,8. - -Signed-off-by: Tomas Glozar -Signed-off-by: John Kacur ---- - rteval-cmd | 17 ++++++++++------- - 1 file changed, 10 insertions(+), 7 deletions(-) - -diff --git a/rteval-cmd b/rteval-cmd -index ed13af3..c72bc61 100755 ---- a/rteval-cmd -+++ b/rteval-cmd -@@ -331,29 +331,32 @@ if __name__ == '__main__': - - ldcfg = config.GetSection('loads') - msrcfg = config.GetSection('measurement') -+ # Remember if cpulists were explicitly set by the user before running -+ # parse_cpulist_from_config, which generates default value for them - msrcfg_cpulist_present = msrcfg.cpulist != "" -- # Parse measurement cpulist using parse_cpulist_from_config to account for run-on-isolcpus -- # and relative cpusets -+ ldcfg_cpulist_present = ldcfg.cpulist != "" -+ # Parse cpulists using parse_cpulist_from_config to account for -+ # run-on-isolcpus and relative cpusets - cpulist = parse_cpulist_from_config(msrcfg.cpulist, msrcfg.run_on_isolcpus) - if msrcfg_cpulist_present and not cpulist_utils.is_relative(msrcfg.cpulist) and msrcfg.run_on_isolcpus: - logger.log(Log.WARN, "ignoring --measurement-run-on-isolcpus, since cpulist is specified") - msrcfg.cpulist = collapse_cpulist(cpulist) -- if ldcfg.cpulist: -- ldcfg.cpulist = remove_offline(ldcfg.cpulist) -+ cpulist = parse_cpulist_from_config(ldcfg.cpulist) -+ ldcfg.cpulist = collapse_cpulist(cpulist) - # if we only specified one set of cpus (loads or measurement) - # default the other to the inverse of the specified list -- if not ldcfg.cpulist and msrcfg_cpulist_present: -+ if not ldcfg_cpulist_present and msrcfg_cpulist_present: - tmplist = expand_cpulist(msrcfg.cpulist) - tmplist = SysTopology().invert_cpulist(tmplist) - tmplist = cpulist_utils.online_cpulist(tmplist) - ldcfg.cpulist = collapse_cpulist(tmplist) -- if not msrcfg_cpulist_present and ldcfg.cpulist: -+ if not msrcfg_cpulist_present and ldcfg_cpulist_present: - tmplist = expand_cpulist(ldcfg.cpulist) - tmplist = SysTopology().invert_cpulist(tmplist) - tmplist = cpulist_utils.online_cpulist(tmplist) - msrcfg.cpulist = collapse_cpulist(tmplist) - -- if ldcfg.cpulist: -+ if ldcfg_cpulist_present: - logger.log(Log.DEBUG, f"loads cpulist: {ldcfg.cpulist}") - # if --onlyload is specified msrcfg.cpulist is unused - if msrcfg_cpulist_present and not rtevcfg.onlyload: --- -2.44.0 - diff --git a/rteval-Add-relative-cpulists-for-measurements.patch b/rteval-Add-relative-cpulists-for-measurements.patch deleted file mode 100644 index 933255f..0000000 --- a/rteval-Add-relative-cpulists-for-measurements.patch +++ /dev/null @@ -1,278 +0,0 @@ -From 64ce7848dfabd2056d35c8a60f3354db45e36286 Mon Sep 17 00:00:00 2001 -From: Tomas Glozar -Date: Thu, 18 Jan 2024 10:18:10 +0100 -Subject: [PATCH 2/4] rteval: Add relative cpulists for measurements - -Instead of specifying an absolute list of CPUs to run measurements on -in --measurement-cpulist, implement an option to specify a relative list -with respect to the current cpuset of rteval. - -The relative cpulist can include CPUs both for addition and for removal, -e.g. +0,1,-7,8. - -Also move the logic for processing cpulists specified by the user as -a string into cpulists usable by rteval to a single function. - -Signed-off-by: Tomas Glozar -Signed-off-by: John Kacur ---- - rteval-cmd | 26 +++++++----- - rteval/cpulist_utils.py | 33 ++++++++++++++++ - rteval/modules/measurement/__init__.py | 9 +---- - rteval/modules/measurement/cyclictest.py | 50 +++--------------------- - rteval/systopology.py | 33 ++++++++++++++++ - 5 files changed, 90 insertions(+), 61 deletions(-) - -diff --git a/rteval-cmd b/rteval-cmd -index d224728..a5e8746 100755 ---- a/rteval-cmd -+++ b/rteval-cmd -@@ -30,7 +30,7 @@ from rteval import RtEval, rtevalConfig - from rteval.modules.loads import LoadModules - from rteval.modules.measurement import MeasurementModules - from rteval.version import RTEVAL_VERSION --from rteval.systopology import SysTopology -+from rteval.systopology import SysTopology, parse_cpulist_from_config - from rteval.modules.loads.kcompile import ModuleParameters - import rteval.cpulist_utils as cpulist_utils - -@@ -339,26 +339,32 @@ if __name__ == '__main__': - - ldcfg = config.GetSection('loads') - msrcfg = config.GetSection('measurement') -- if ldcfg.cpulist and msrcfg.cpulist: -+ msrcfg_cpulist_present = msrcfg.cpulist != "" -+ # Parse measurement cpulist using parse_cpulist_from_config to account for run-on-isolcpus -+ # and relative cpusets -+ cpulist = parse_cpulist_from_config(msrcfg.cpulist, msrcfg.run_on_isolcpus) -+ if msrcfg_cpulist_present and not cpulist_utils.is_relative(msrcfg.cpulist) and msrcfg.run_on_isolcpus: -+ logger.log(Log.WARN, "ignoring --measurement-run-on-isolcpus, since cpulist is specified") -+ msrcfg.cpulist = collapse_cpulist(cpulist) -+ if ldcfg.cpulist: - ldcfg.cpulist = remove_offline(ldcfg.cpulist) -- msrcfg.cpulist = remove_offline(msrcfg.cpulist) - # if we only specified one set of cpus (loads or measurement) - # default the other to the inverse of the specified list -- if not ldcfg.cpulist and msrcfg.cpulist: -+ if not ldcfg.cpulist and msrcfg_cpulist_present: - tmplist = expand_cpulist(msrcfg.cpulist) - tmplist = SysTopology().invert_cpulist(tmplist) -- ldcfg.cpulist = compress_cpulist(tmplist) -- msrcfg.cpulist = remove_offline(msrcfg.cpulist) -- if not msrcfg.cpulist and ldcfg.cpulist: -+ tmplist = cpulist_utils.online_cpulist(tmplist) -+ ldcfg.cpulist = collapse_cpulist(tmplist) -+ if not msrcfg_cpulist_present and ldcfg.cpulist: - tmplist = expand_cpulist(ldcfg.cpulist) - tmplist = SysTopology().invert_cpulist(tmplist) -- msrcfg.cpulist = compress_cpulist(tmplist) -- ldcfg.cpulist = remove_offline(ldcfg.cpulist) -+ tmplist = cpulist_utils.online_cpulist(tmplist) -+ msrcfg.cpulist = collapse_cpulist(tmplist) - - if ldcfg.cpulist: - logger.log(Log.DEBUG, f"loads cpulist: {ldcfg.cpulist}") - # if --onlyload is specified msrcfg.cpulist is unused -- if msrcfg.cpulist and not rtevcfg.onlyload: -+ if msrcfg_cpulist_present and not rtevcfg.onlyload: - logger.log(Log.DEBUG, f"measurement cpulist: {msrcfg.cpulist}") - logger.log(Log.DEBUG, f"workdir: {rtevcfg.workdir}") - -diff --git a/rteval/cpulist_utils.py b/rteval/cpulist_utils.py -index 402d579..7abc45a 100644 ---- a/rteval/cpulist_utils.py -+++ b/rteval/cpulist_utils.py -@@ -126,3 +126,36 @@ def nonisolated_cpulist(cpulist): - isolated_cpulist = sysread(cpupath, "isolated") - isolated_cpulist = expand_cpulist(isolated_cpulist) - return list(set(cpulist).difference(set(isolated_cpulist))) -+ -+ -+def is_relative(cpulist): -+ return cpulist.startswith("+") or cpulist.startswith("-") -+ -+ -+def expand_relative_cpulist(cpulist): -+ """ -+ Expand a relative cpulist into a tuple of lists. -+ :param cpulist: Relative cpulist of form +1,2,3,-4,5,6 -+ :return: Tuple of two lists, one for added CPUs, one for removed CPUs -+ """ -+ added_cpus = [] -+ removed_cpus = [] -+ -+ if not cpulist: -+ return added_cpus, removed_cpus -+ -+ cpus = None -+ -+ for part in cpulist.split(','): -+ if part.startswith('+') or part.startswith('-'): -+ cpus = added_cpus if part[0] == '+' else removed_cpus -+ part = part[1:] -+ if '-' in part: -+ a, b = part.split('-') -+ a, b = int(a), int(b) -+ cpus.extend(list(range(a, b + 1))) -+ else: -+ a = int(part) -+ cpus.append(a) -+ -+ return list(set(added_cpus)), list(set(removed_cpus)) -diff --git a/rteval/modules/measurement/__init__.py b/rteval/modules/measurement/__init__.py -index 66dc9c5..11bd7b0 100644 ---- a/rteval/modules/measurement/__init__.py -+++ b/rteval/modules/measurement/__init__.py -@@ -5,7 +5,7 @@ - - import libxml2 - from rteval.modules import RtEvalModules, ModuleContainer --from rteval.systopology import SysTopology as SysTop -+from rteval.systopology import parse_cpulist_from_config - import rteval.cpulist_utils as cpulist_utils - - class MeasurementProfile(RtEvalModules): -@@ -183,12 +183,7 @@ measurement profiles, based on their characteristics""" - rep_n = libxml2.newNode("Measurements") - cpulist = self.__cfg.GetSection("measurement").cpulist - run_on_isolcpus = self.__cfg.GetSection("measurement").run_on_isolcpus -- if cpulist: -- # Convert str to list and remove offline cpus -- cpulist = cpulist_utils.expand_cpulist(cpulist) -- cpulist = cpulist_utils.online_cpulist(cpulist) -- else: -- cpulist = SysTop().online_cpus() if run_on_isolcpus else SysTop().default_cpus() -+ cpulist = parse_cpulist_from_config(cpulist, run_on_isolcpus) - rep_n.newProp("measurecpus", cpulist_utils.collapse_cpulist(cpulist)) - - for mp in self.__measureprofiles: -diff --git a/rteval/modules/measurement/cyclictest.py b/rteval/modules/measurement/cyclictest.py -index fdca257..7224225 100644 ---- a/rteval/modules/measurement/cyclictest.py -+++ b/rteval/modules/measurement/cyclictest.py -@@ -16,8 +16,7 @@ import math - import libxml2 - from rteval.Log import Log - from rteval.modules import rtevalModulePrototype --from rteval.systopology import cpuinfo --from rteval.systopology import SysTopology -+from rteval.systopology import cpuinfo, parse_cpulist_from_config - import rteval.cpulist_utils as cpulist_utils - - expand_cpulist = cpulist_utils.expand_cpulist -@@ -193,39 +192,9 @@ class Cyclictest(rtevalModulePrototype): - self.__priority = int(self.__cfg.setdefault('priority', 95)) - self.__buckets = int(self.__cfg.setdefault('buckets', 2000)) - self.__numcores = 0 -- self.__cpus = [] - self.__cyclicdata = {} -- self.__sparse = False -- self.__run_on_isolcpus = bool(self.__cfg.setdefault('run-on-isolcpus', False)) -- -- if self.__cfg.cpulist: -- self.__cpulist = self.__cfg.cpulist -- self.__cpus = expand_cpulist(self.__cpulist) -- # Only include online cpus -- self.__cpus = cpulist_utils.online_cpulist(self.__cpus) -- # Reset cpulist from the newly calculated self.__cpus -- self.__cpulist = cpulist_utils.collapse_cpulist(self.__cpus) -- self.__cpus = [str(c) for c in self.__cpus] -- self.__sparse = True -- if self.__run_on_isolcpus: -- self._log(Log.WARN, "ignoring --measurement-run-on-isolcpus, since cpulist is specified") -- else: -- self.__cpus = SysTopology().online_cpus_str() -- # Get the cpuset from the environment -- cpuset = os.sched_getaffinity(0) -- # Convert the elements to strings -- cpuset = [str(c) for c in cpuset] -- # Get isolated CPU list -- isolcpus = [str(c) for c in SysTopology().isolated_cpus()] -- # Only include cpus that are in the cpuset and isolated CPUs if run_on_isolcpus is enabled -- self.__cpus = [c for c in self.__cpus if c in cpuset or self.__run_on_isolcpus and c in isolcpus] -- if self.__run_on_isolcpus: -- self.__sparse = True -- self.__cpulist = cpulist_utils.collapse_cpulist([int(c) for c in self.__cpus]) -- -- # Sort the list of cpus to align with the order reported by cyclictest -- self.__cpus.sort(key=int) -- -+ self.__cpulist = self.__cfg.cpulist -+ self.__cpus = [str(c) for c in expand_cpulist(self.__cpulist)] - self.__numcores = len(self.__cpus) - - info = cpuinfo() -@@ -242,10 +211,7 @@ class Cyclictest(rtevalModulePrototype): - logfnc=self._log) - self.__cyclicdata['system'].description = (f"({self.__numcores} cores) ") + info['0']['model name'] - -- if self.__sparse: -- self._log(Log.DEBUG, f"system using {self.__numcores} cpu cores") -- else: -- self._log(Log.DEBUG, f"system has {self.__numcores} cpu cores") -+ self._log(Log.DEBUG, f"system using {self.__numcores} cpu cores") - self.__started = False - self.__cyclicoutput = None - self.__breaktraceval = None -@@ -280,12 +246,8 @@ class Cyclictest(rtevalModulePrototype): - f'-h {self.__buckets}', - f"-p{int(self.__priority)}", - ] -- if self.__sparse: -- self.__cmd.append(f'-t{self.__numcores}') -- self.__cmd.append(f'-a{self.__cpulist}') -- else: -- self.__cmd.append('-t') -- self.__cmd.append('-a') -+ self.__cmd.append(f'-t{self.__numcores}') -+ self.__cmd.append(f'-a{self.__cpulist}') - - if 'threads' in self.__cfg and self.__cfg.threads: - self.__cmd.append(f"-t{int(self.__cfg.threads)}") -diff --git a/rteval/systopology.py b/rteval/systopology.py -index 9e45762..6bcfc77 100644 ---- a/rteval/systopology.py -+++ b/rteval/systopology.py -@@ -241,6 +241,39 @@ class SysTopology: - """ return a list of online cpus in cpulist """ - return [c for c in self.online_cpus() if c in cpulist] - -+ -+def parse_cpulist_from_config(cpulist, run_on_isolcpus=False): -+ """ -+ Generates a cpulist based on --*-cpulist argument given by user -+ :param cpulist: Value of --*-cpulist argument -+ :param run_on_isolcpus: Value of --*-run-on-isolcpus argument -+ :return: Sorted list of CPUs as integers -+ """ -+ if cpulist and not cpulist_utils.is_relative(cpulist): -+ result = cpulist_utils.expand_cpulist(cpulist) -+ # Only include online cpus -+ result = cpulist_utils.online_cpulist(result) -+ else: -+ result = SysTopology().online_cpus() -+ # Get the cpuset from the environment -+ cpuset = os.sched_getaffinity(0) -+ # Get isolated CPU list -+ isolcpus = SysTopology().isolated_cpus() -+ if cpulist and cpulist_utils.is_relative(cpulist): -+ # Include cpus that are not removed in relative cpuset and are either in cpuset from affinity, -+ # isolcpus (with run_on_isolcpus enabled, or added by relative cpuset -+ added_cpus, removed_cpus = cpulist_utils.expand_relative_cpulist(cpulist) -+ result = [c for c in result -+ if (c in cpuset or -+ c in added_cpus or -+ run_on_isolcpus and c in isolcpus) and -+ c not in removed_cpus] -+ else: -+ # Only include cpus that are in the cpuset and isolated CPUs if run_on_isolcpus is enabled -+ result = [c for c in result if c in cpuset or run_on_isolcpus and c in isolcpus] -+ return result -+ -+ - if __name__ == "__main__": - - def unit_test(): --- -2.43.0 - diff --git a/rteval-Add-rtla-timerlat-as-a-measurement-module.patch b/rteval-Add-rtla-timerlat-as-a-measurement-module.patch deleted file mode 100644 index 50e7ff7..0000000 --- a/rteval-Add-rtla-timerlat-as-a-measurement-module.patch +++ /dev/null @@ -1,189 +0,0 @@ -From 393c95ee69ec98816a26aa0583f6b1cac4acb5e7 Mon Sep 17 00:00:00 2001 -From: John Kacur -Date: Fri, 12 Apr 2024 14:40:37 -0400 -Subject: [PATCH 02/13] rteval: Add rtla timerlat as a measurement module - -This is the first step to adding timerlat as a measurement module -With this change you can run timerlat as a standalone rteval file like -this (as root) - -python rteval/modules/measurement/timerlat.py - -You can also modify your rteval.conf to list timerlat in the -[measurement] section, for example like this - -[measurement] -cyclictest: module -timerlat: module - -and then both measurement moduels will be run from rteval, for example - -rteval -D -d5m --measurement-cpulist=1-5 - -Will run rteval with Debug info, for 5m and cyclictest and timerlat will -run on cpus 1-5 and load modules will run on the other available cpus. - -Currently MakeReport just prints to standard out the same information -that timerlat outputs, in otherwords, there is no processing into xml -yet. Also, there is no way to invoke tracing at the time, but that will -be added soon! - -Signed-off-by: John Kacur ---- - rteval-cmd | 1 + - rteval/modules/measurement/timerlat.py | 131 +++++++++++++++++++++++++ - 2 files changed, 132 insertions(+) - create mode 100644 rteval/modules/measurement/timerlat.py - -diff --git a/rteval-cmd b/rteval-cmd -index c72bc614ad78..5cb6d7a44523 100755 ---- a/rteval-cmd -+++ b/rteval-cmd -@@ -247,6 +247,7 @@ if __name__ == '__main__': - if not config.HasSection('measurement'): - config.AppendConfig('measurement', { - 'cyclictest' : 'module', -+ 'timerlat' : 'module', - 'sysstat' : 'module'}) - - # Prepare log levels before loading modules, not to have unwanted log messages -diff --git a/rteval/modules/measurement/timerlat.py b/rteval/modules/measurement/timerlat.py -new file mode 100644 -index 000000000000..d4e78de8d2a2 ---- /dev/null -+++ b/rteval/modules/measurement/timerlat.py -@@ -0,0 +1,131 @@ -+# SPDX-License-Identifier: GPL-2.0-or-later -+# -+# Copyright 2024 John Kacur -+# -+""" timerlat.py - objectd to manage rtla timerlat """ -+import os -+import subprocess -+import signal -+import time -+import tempfile -+import libxml2 -+from rteval.Log import Log -+from rteval.modules import rtevalModulePrototype -+from rteval.systopology import cpuinfo, SysTopology -+from rteval.cpulist_utils import expand_cpulist, collapse_cpulist -+ -+class Timerlat(rtevalModulePrototype): -+ """ measurement modules for rteval """ -+ def __init__(self, config, logger=None): -+ rtevalModulePrototype.__init__(self, 'measurement', 'timerlat', logger) -+ -+ self.__cfg = config -+ -+ self.__numanodes = int(self.__cfg.setdefault('numanodes', 0)) -+ self.__priority = int(self.__cfg.setdefault('priority', 95)) -+ -+ self.__cpulist = self.__cfg.setdefault('cpulist', "") -+ self.__cpus = [str(c) for c in expand_cpulist(self.__cpulist)] -+ self.__numcores = len(self.__cpus) -+ -+ self.__timerlat_out = None -+ self.__timerlat_err = None -+ self.__started = False -+ self._log(Log.DEBUG, f"system using {self.__numcores} cpu cores") -+ -+ -+ def _WorkloadSetup(self): -+ self.__timerlat_process = None -+ -+ def _WorkloadBuild(self): -+ self._setReady() -+ -+ def _WorkloadPrepare(self): -+ self.__cmd = ['rtla', 'timerlat', 'hist', '-P', f'f:{int(self.__priority)}', '-u'] -+ self.__cmd.append(f'-c{self.__cpulist}') -+ self._log(Log.DEBUG, f'self.__cmd = {self.__cmd}') -+ self.__timerlat_out = tempfile.SpooledTemporaryFile(mode='w+b') -+ self.__timerlat_err = tempfile.SpooledTemporaryFile(mode='w+b') -+ -+ def _WorkloadTask(self): -+ if self.__started: -+ return -+ -+ self._log(Log.DEBUG, f'starting with cmd: {" ".join(self.__cmd)}') -+ -+ self.__timerlat_out.seek(0) -+ self.__timerlat_err.seek(0) -+ try: -+ self.__timerlat_process = subprocess.Popen(self.__cmd, -+ stdout=self.__timerlat_out, -+ stderr=self.__timerlat_err, -+ stdin=None) -+ self.__started = True -+ except OSError: -+ self.__started = False -+ -+ def WorkloadAlive(self): -+ if self.__started: -+ return self.__timerlat_process.poll() is None -+ return False -+ -+ def _WorkloadCleanup(self): -+ if not self.__started: -+ return -+ while self.__timerlat_process.poll() is None: -+ self._log(Log.DEBUG, "Sending SIGINT") -+ os.kill(self.__timerlat_process.pid, signal.SIGINT) -+ time.sleep(2) -+ -+ self._setFinished() -+ self.__started = False -+ -+ def MakeReport(self): -+ self.__timerlat_out.seek(0) -+ for line in self.__timerlat_out: -+ line = bytes.decode(line) -+ print(line) -+ self.__timerlat_out.close() -+ -+ -+def ModuleInfo(): -+ """ Required measurement module information """ -+ return {"parallel": True, -+ "loads": True} -+ -+def ModuleParameters(): -+ """ default parameters """ -+ return {"priority": {"descr": "Run rtla timerlat with this priority", -+ "default": 95, -+ "metavar": "PRIO" } -+ } -+ -+def create(params, logger): -+ """ Instantiate a Timerlat measurement module object""" -+ return Timerlat(params, logger) -+ -+if __name__ == '__main__': -+ from rteval.rtevalConfig import rtevalConfig -+ -+ l = Log() -+ l.SetLogVerbosity(Log.INFO|Log.DEBUG|Log.ERR|Log.WARN) -+ -+ cfg = rtevalConfig({}, logger=l) -+ prms = {} -+ modprms = ModuleParameters() -+ for c, p in list(modprms.items()): -+ prms[c] = p['default'] -+ cfg.AppendConfig('timerlat', prms) -+ -+ cfg_tl = cfg.GetSection('timerlat') -+ cfg_tl.cpulist = collapse_cpulist(SysTopology().online_cpus()) -+ -+ RUNTIME = 10 -+ -+ tl = Timerlat(cfg_tl, l) -+ tl._WorkloadSetup() -+ tl._WorkloadPrepare() -+ tl._WorkloadTask() -+ time.sleep(RUNTIME) -+ tl._WorkloadCleanup() -+ tl.MakeReport() --- -2.44.0 - diff --git a/rteval-Add-summary-reporting-for-timerlat.patch b/rteval-Add-summary-reporting-for-timerlat.patch deleted file mode 100644 index 2a94208..0000000 --- a/rteval-Add-summary-reporting-for-timerlat.patch +++ /dev/null @@ -1,178 +0,0 @@ -From 20fbcc1547d77a77a3e333bac28b7f28632d8707 Mon Sep 17 00:00:00 2001 -From: John Kacur -Date: Fri, 26 Apr 2024 14:45:48 -0400 -Subject: [PATCH 08/13] rteval: Add summary reporting for timerlat - -This adds an rteval section to the xsl file and generates a timerlat -report at the end of a run. To use it edit your rteval.conf file -to comment out cyclictest and uncomment timerlat - -Another interesting thing you can do is uncomment both of them and -get a report from both in one run. - -You can also use this with the summary report, for example - -rteval -Z rteval-20240426-3/summary.xml - -will work with timerlat now. (or both cyclictest and timerlat at the -same time.) - -Signed-off-by: John Kacur ---- - rteval.conf | 1 + - rteval/rteval_text.xsl | 103 +++++++++++++++++++++++++++++++++++++++-- - 2 files changed, 100 insertions(+), 4 deletions(-) - -diff --git a/rteval.conf b/rteval.conf -index 4c32fcf4d842..601410b51c28 100644 ---- a/rteval.conf -+++ b/rteval.conf -@@ -7,6 +7,7 @@ report_interval: 600 - - [measurement] - cyclictest: module -+# timerlat: module - - [loads] - kcompile: module -diff --git a/rteval/rteval_text.xsl b/rteval/rteval_text.xsl -index f526526d4d49..1e9c0f0d26c5 100644 ---- a/rteval/rteval_text.xsl -+++ b/rteval/rteval_text.xsl -@@ -201,11 +201,11 @@ - - - -- -+ - - - -- -+ - - Latency test - -@@ -237,7 +237,7 @@ - - - -- -+ - - CPU core - -@@ -300,6 +300,101 @@ - - - -+ -+ -+ Latency test -+ -+ Started: -+ -+ -+ -+ Stopped: -+ -+ -+ -+ Command: -+ -+ -+ -+ -+ -+ System: -+ -+ -+ -+ Statistics: -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ CPU core -+ -+ Priority: -+ -+ -+ Statistics: -+ -+ -+ -+ -+ -+ -+ -+ Samples: -+ -+ -+ -+ -+ Mean: -+ -+ -+ -+ -+ Median: -+ -+ -+ -+ -+ Mode: -+ -+ -+ -+ -+ Range: -+ -+ -+ -+ -+ Min: -+ -+ -+ -+ -+ Max: -+ -+ -+ -+ -+ Mean Absolute Dev: -+ -+ -+ -+ -+ Std.dev: -+ -+ -+ -+ -+ -+ -+ - - - -@@ -340,7 +435,7 @@ - us - - -- -+ - - sysstat measurements - --- -2.44.0 - diff --git a/rteval-Change-the-default-kernel-for-kcompile.patch b/rteval-Change-the-default-kernel-for-kcompile.patch deleted file mode 100644 index 8af26cf..0000000 --- a/rteval-Change-the-default-kernel-for-kcompile.patch +++ /dev/null @@ -1,65 +0,0 @@ -From c0ee73f00f6868e0ead5ace958a88a6a23db6ad3 Mon Sep 17 00:00:00 2001 -From: John Kacur -Date: Thu, 9 Nov 2023 15:43:53 -0500 -Subject: [PATCH] rteval: Change the default kernel for kcompile to linux-6.6.1 - -Change the default kernel for kcompile to linux-6.6.1 - -Signed-off-by: John Kacur ---- - Makefile | 2 +- - rteval/modules/loads/kcompile.py | 4 ++-- - rteval/rteval.conf | 2 +- - 3 files changed, 4 insertions(+), 4 deletions(-) - -diff --git a/Makefile b/Makefile -index b73e8c13f3e5..14f74e087eff 100644 ---- a/Makefile -+++ b/Makefile -@@ -18,7 +18,7 @@ PREFIX := /usr - DATADIR := $(DESTDIR)/$(PREFIX)/share - LOADDIR := loadsource - --KLOAD := $(LOADDIR)/linux-6.1.8.tar.xz -+KLOAD := $(LOADDIR)/linux-6.6.1.tar.xz - BLOAD := $(LOADDIR)/dbench-4.0.tar.gz - LOADS := $(KLOAD) $(BLOAD) - -diff --git a/rteval/modules/loads/kcompile.py b/rteval/modules/loads/kcompile.py -index 8be79ce630d5..0d025771e90e 100644 ---- a/rteval/modules/loads/kcompile.py -+++ b/rteval/modules/loads/kcompile.py -@@ -20,7 +20,7 @@ expand_cpulist = CpuList.expand_cpulist - compress_cpulist = CpuList.compress_cpulist - nonisolated_cpulist = CpuList.nonisolated_cpulist - --DEFAULT_KERNEL_PREFIX = "linux-6.1" -+DEFAULT_KERNEL_PREFIX = "linux-6.6" - - class KBuildJob: - '''Class to manage a build job bound to a particular node''' -@@ -334,7 +334,7 @@ class Kcompile(CommandLineLoad): - - def ModuleParameters(): - return {"source": {"descr": "Source tar ball", -- "default": "linux-6.1.8.tar.xz", -+ "default": "linux-6.6.1.tar.xz", - "metavar": "TARBALL"}, - "jobspercore": {"descr": "Number of working threads per core", - "default": 2, -diff --git a/rteval/rteval.conf b/rteval/rteval.conf -index 79e28032dc6b..a4aad33e264f 100644 ---- a/rteval/rteval.conf -+++ b/rteval/rteval.conf -@@ -18,7 +18,7 @@ dbench: external - stressng: module - - [kcompile] --source: linux-6.1.8.xz -+source: linux-6.6.1.xz - jobspercore: 2 - - [hackbench] --- -2.41.0 - diff --git a/rteval-Convert-CpuList-class-to-a-module.patch b/rteval-Convert-CpuList-class-to-a-module.patch deleted file mode 100644 index b5bbcfb..0000000 --- a/rteval-Convert-CpuList-class-to-a-module.patch +++ /dev/null @@ -1,616 +0,0 @@ -From a788ea3ec976e51309f0bfae1a41cce704a77f2d Mon Sep 17 00:00:00 2001 -From: Tomas Glozar -Date: Thu, 18 Jan 2024 10:18:09 +0100 -Subject: [PATCH 1/4] rteval: Convert CpuList class to a module - -Move out code from CpuList class in rteval.systopology into a separate -module named rteval.cpulist_utils and avoid wrapping CPU lists in -a CpuList object. - -Almost all uses of CpuList in the code either use the static methods of -the class or its constructor to filter online CPUs by running -CpuList(...).cpulist. The only exception to this are NumaNode and -SimNumaNode classes in systopology; these store a CpuList object, -however their .getcpulist() method extracts the list out. Thus, -the class is completely unnecessary. - -Note: A better name for the module would be cpulist, consistent with the -original name of the class, but this name is already used for variables -throughout the code, hence cpulist_utils is used instead. - -Signed-off-by: Tomas Glozar -Signed-off-by: John Kacur ---- - rteval-cmd | 10 +- - rteval/cpulist_utils.py | 128 ++++++++++++++++++ - rteval/modules/loads/__init__.py | 8 +- - rteval/modules/loads/hackbench.py | 9 +- - rteval/modules/loads/kcompile.py | 11 +- - rteval/modules/loads/stressng.py | 8 +- - rteval/modules/measurement/__init__.py | 8 +- - rteval/modules/measurement/cyclictest.py | 11 +- - rteval/systopology.py | 165 ++--------------------- - 9 files changed, 177 insertions(+), 181 deletions(-) - create mode 100644 rteval/cpulist_utils.py - -diff --git a/rteval-cmd b/rteval-cmd -index 7c41429..d224728 100755 ---- a/rteval-cmd -+++ b/rteval-cmd -@@ -30,11 +30,13 @@ from rteval import RtEval, rtevalConfig - from rteval.modules.loads import LoadModules - from rteval.modules.measurement import MeasurementModules - from rteval.version import RTEVAL_VERSION --from rteval.systopology import CpuList, SysTopology -+from rteval.systopology import SysTopology - from rteval.modules.loads.kcompile import ModuleParameters -+import rteval.cpulist_utils as cpulist_utils - --compress_cpulist = CpuList.compress_cpulist --expand_cpulist = CpuList.expand_cpulist -+compress_cpulist = cpulist_utils.compress_cpulist -+expand_cpulist = cpulist_utils.expand_cpulist -+collapse_cpulist = cpulist_utils.collapse_cpulist - - def summarize(repfile, xslt): - """ Summarize an already existing XML report """ -@@ -211,7 +213,7 @@ def remove_offline(cpulist): - """ return cpulist in collapsed compressed form with only online cpus """ - tmplist = expand_cpulist(cpulist) - tmplist = SysTopology().online_cpulist(tmplist) -- return CpuList.collapse_cpulist(tmplist) -+ return collapse_cpulist(tmplist) - - - if __name__ == '__main__': -diff --git a/rteval/cpulist_utils.py b/rteval/cpulist_utils.py -new file mode 100644 -index 0000000..402d579 ---- /dev/null -+++ b/rteval/cpulist_utils.py -@@ -0,0 +1,128 @@ -+# -*- coding: utf-8 -*- -+# SPDX-License-Identifier: GPL-2.0-or-later -+# -+# Copyright 2016 - Clark Williams -+# Copyright 2021 - John Kacur -+# Copyright 2023 - Tomas Glozar -+# -+"""Module providing utility functions for working with CPU lists""" -+ -+import os -+ -+ -+cpupath = "/sys/devices/system/cpu" -+ -+ -+def sysread(path, obj): -+ """ Helper function for reading system files """ -+ with open(os.path.join(path, obj), "r") as fp: -+ return fp.readline().strip() -+ -+ -+def _online_file_exists(): -+ """ Check whether machine / kernel is configured with online file """ -+ # Note: some machines do not have cpu0/online so we check cpu1/online. -+ # In the case of machines with a single CPU, there is no cpu1, but -+ # that is not a problem, since a single CPU cannot be offline -+ return os.path.exists(os.path.join(cpupath, "cpu1/online")) -+ -+ -+def _isolated_file_exists(): -+ """ Check whether machine / kernel is configured with isolated file """ -+ return os.path.exists(os.path.join(cpupath, "isolated")) -+ -+ -+def collapse_cpulist(cpulist): -+ """ -+ Collapse a list of cpu numbers into a string range -+ of cpus (e.g. 0-5, 7, 9) -+ """ -+ cur_range = [None, None] -+ result = [] -+ for cpu in cpulist + [None]: -+ if cur_range[0] is None: -+ cur_range[0] = cur_range[1] = cpu -+ continue -+ if cpu is not None and cpu == cur_range[1] + 1: -+ # Extend currently processed range -+ cur_range[1] += 1 -+ else: -+ # Range processing finished, add range to string -+ result.append(f"{cur_range[0]}-{cur_range[1]}" -+ if cur_range[0] != cur_range[1] -+ else str(cur_range[0])) -+ # Reset -+ cur_range[0] = cur_range[1] = cpu -+ return ",".join(result) -+ -+ -+def compress_cpulist(cpulist): -+ """ return a string representation of cpulist """ -+ if not cpulist: -+ return "" -+ if isinstance(cpulist[0], int): -+ return ",".join(str(e) for e in cpulist) -+ return ",".join(cpulist) -+ -+ -+def expand_cpulist(cpulist): -+ """ expand a range string into an array of cpu numbers -+ don't error check against online cpus -+ """ -+ result = [] -+ -+ if not cpulist: -+ return result -+ -+ for part in cpulist.split(','): -+ if '-' in part: -+ a, b = part.split('-') -+ a, b = int(a), int(b) -+ result.extend(list(range(a, b + 1))) -+ else: -+ a = int(part) -+ result.append(a) -+ return [int(i) for i in list(set(result))] -+ -+ -+def is_online(n): -+ """ check whether cpu n is online """ -+ path = os.path.join(cpupath, f'cpu{n}') -+ -+ # Some hardware doesn't allow cpu0 to be turned off -+ if not os.path.exists(path + '/online') and n == 0: -+ return True -+ -+ return sysread(path, "online") == "1" -+ -+ -+def online_cpulist(cpulist): -+ """ Given a cpulist, return a cpulist of online cpus """ -+ # This only works if the sys online files exist -+ if not _online_file_exists(): -+ return cpulist -+ newlist = [] -+ for cpu in cpulist: -+ if not _online_file_exists() and cpu == '0': -+ newlist.append(cpu) -+ elif is_online(int(cpu)): -+ newlist.append(cpu) -+ return newlist -+ -+ -+def isolated_cpulist(cpulist): -+ """Given a cpulist, return a cpulist of isolated CPUs""" -+ if not _isolated_file_exists(): -+ return cpulist -+ isolated_cpulist = sysread(cpupath, "isolated") -+ isolated_cpulist = expand_cpulist(isolated_cpulist) -+ return list(set(isolated_cpulist) & set(cpulist)) -+ -+ -+def nonisolated_cpulist(cpulist): -+ """Given a cpulist, return a cpulist of non-isolated CPUs""" -+ if not _isolated_file_exists(): -+ return cpulist -+ isolated_cpulist = sysread(cpupath, "isolated") -+ isolated_cpulist = expand_cpulist(isolated_cpulist) -+ return list(set(cpulist).difference(set(isolated_cpulist))) -diff --git a/rteval/modules/loads/__init__.py b/rteval/modules/loads/__init__.py -index 13fba1e..0845742 100644 ---- a/rteval/modules/loads/__init__.py -+++ b/rteval/modules/loads/__init__.py -@@ -11,7 +11,8 @@ import libxml2 - from rteval.Log import Log - from rteval.rtevalConfig import rtevalCfgSection - from rteval.modules import RtEvalModules, rtevalModulePrototype --from rteval.systopology import CpuList, SysTopology as SysTop -+from rteval.systopology import SysTopology as SysTop -+import rteval.cpulist_utils as cpulist_utils - - class LoadThread(rtevalModulePrototype): - def __init__(self, name, config, logger=None): -@@ -117,10 +118,11 @@ class LoadModules(RtEvalModules): - cpulist = self._cfg.GetSection(self._module_config).cpulist - if cpulist: - # Convert str to list and remove offline cpus -- cpulist = CpuList(cpulist).cpulist -+ cpulist = cpulist_utils.expand_cpulist(cpulist) -+ cpulist = cpulist_utils.online_cpulist(cpulist) - else: - cpulist = SysTop().default_cpus() -- rep_n.newProp("loadcpus", CpuList.collapse_cpulist(cpulist)) -+ rep_n.newProp("loadcpus", cpulist_utils.collapse_cpulist(cpulist)) - - return rep_n - -diff --git a/rteval/modules/loads/hackbench.py b/rteval/modules/loads/hackbench.py -index cfc7063..a70fdb3 100644 ---- a/rteval/modules/loads/hackbench.py -+++ b/rteval/modules/loads/hackbench.py -@@ -16,10 +16,11 @@ import errno - from signal import SIGKILL - from rteval.modules.loads import CommandLineLoad - from rteval.Log import Log --from rteval.systopology import CpuList, SysTopology -+from rteval.systopology import SysTopology -+import rteval.cpulist_utils as cpulist_utils - --expand_cpulist = CpuList.expand_cpulist --isolated_cpulist = CpuList.isolated_cpulist -+expand_cpulist = cpulist_utils.expand_cpulist -+isolated_cpulist = cpulist_utils.isolated_cpulist - - class Hackbench(CommandLineLoad): - def __init__(self, config, logger): -@@ -61,7 +62,7 @@ class Hackbench(CommandLineLoad): - self.cpus[n] = [c for c in self.cpus[n] if c in expand_cpulist(self.cpulist)] - # if a cpulist was not specified, exclude isolated cpus - else: -- self.cpus[n] = CpuList.nonisolated_cpulist(self.cpus[n]) -+ self.cpus[n] = cpulist_utils.nonisolated_cpulist(self.cpus[n]) - - # track largest number of cpus used on a node - node_biggest = len(self.cpus[n]) -diff --git a/rteval/modules/loads/kcompile.py b/rteval/modules/loads/kcompile.py -index 0d02577..b606f7a 100644 ---- a/rteval/modules/loads/kcompile.py -+++ b/rteval/modules/loads/kcompile.py -@@ -14,11 +14,12 @@ import subprocess - from rteval.modules import rtevalRuntimeError - from rteval.modules.loads import CommandLineLoad - from rteval.Log import Log --from rteval.systopology import CpuList, SysTopology -+from rteval.systopology import SysTopology -+import rteval.cpulist_utils as cpulist_utils - --expand_cpulist = CpuList.expand_cpulist --compress_cpulist = CpuList.compress_cpulist --nonisolated_cpulist = CpuList.nonisolated_cpulist -+expand_cpulist = cpulist_utils.expand_cpulist -+compress_cpulist = cpulist_utils.compress_cpulist -+nonisolated_cpulist = cpulist_utils.nonisolated_cpulist - - DEFAULT_KERNEL_PREFIX = "linux-6.6" - -@@ -38,7 +39,7 @@ class KBuildJob: - os.mkdir(self.objdir) - - # Exclude isolated CPUs if cpulist not set -- cpus_available = len(nonisolated_cpulist(self.node.cpus.cpulist)) -+ cpus_available = len(nonisolated_cpulist(self.node.cpus)) - - if os.path.exists('/usr/bin/numactl') and not cpulist: - # Use numactl -diff --git a/rteval/modules/loads/stressng.py b/rteval/modules/loads/stressng.py -index 7c9e63f..cbcf6b7 100644 ---- a/rteval/modules/loads/stressng.py -+++ b/rteval/modules/loads/stressng.py -@@ -7,10 +7,10 @@ import subprocess - import signal - from rteval.modules.loads import CommandLineLoad - from rteval.Log import Log --from rteval.systopology import CpuList, SysTopology -+from rteval.systopology import SysTopology -+import rteval.cpulist_utils as cpulist_utils - --expand_cpulist = CpuList.expand_cpulist --nonisolated_cpulist = CpuList.nonisolated_cpulist -+expand_cpulist = cpulist_utils.expand_cpulist - - class Stressng(CommandLineLoad): - " This class creates a load module that runs stress-ng " -@@ -73,7 +73,7 @@ class Stressng(CommandLineLoad): - cpus[n] = [c for c in cpus[n] if c in expand_cpulist(self.cpulist)] - # if a cpulist was not specified, exclude isolated cpus - else: -- cpus[n] = CpuList.nonisolated_cpulist(cpus[n]) -+ cpus[n] = cpulist_utils.nonisolated_cpulist(cpus[n]) - - - # remove nodes with no cpus available for running -diff --git a/rteval/modules/measurement/__init__.py b/rteval/modules/measurement/__init__.py -index 41b8022..66dc9c5 100644 ---- a/rteval/modules/measurement/__init__.py -+++ b/rteval/modules/measurement/__init__.py -@@ -5,7 +5,8 @@ - - import libxml2 - from rteval.modules import RtEvalModules, ModuleContainer --from rteval.systopology import CpuList, SysTopology as SysTop -+from rteval.systopology import SysTopology as SysTop -+import rteval.cpulist_utils as cpulist_utils - - class MeasurementProfile(RtEvalModules): - """Keeps and controls all the measurement modules with the same measurement profile""" -@@ -184,10 +185,11 @@ measurement profiles, based on their characteristics""" - run_on_isolcpus = self.__cfg.GetSection("measurement").run_on_isolcpus - if cpulist: - # Convert str to list and remove offline cpus -- cpulist = CpuList(cpulist).cpulist -+ cpulist = cpulist_utils.expand_cpulist(cpulist) -+ cpulist = cpulist_utils.online_cpulist(cpulist) - else: - cpulist = SysTop().online_cpus() if run_on_isolcpus else SysTop().default_cpus() -- rep_n.newProp("measurecpus", CpuList.collapse_cpulist(cpulist)) -+ rep_n.newProp("measurecpus", cpulist_utils.collapse_cpulist(cpulist)) - - for mp in self.__measureprofiles: - mprep_n = mp.MakeReport() -diff --git a/rteval/modules/measurement/cyclictest.py b/rteval/modules/measurement/cyclictest.py -index 1b14e7e..fdca257 100644 ---- a/rteval/modules/measurement/cyclictest.py -+++ b/rteval/modules/measurement/cyclictest.py -@@ -17,9 +17,10 @@ import libxml2 - from rteval.Log import Log - from rteval.modules import rtevalModulePrototype - from rteval.systopology import cpuinfo --from rteval.systopology import CpuList, SysTopology -+from rteval.systopology import SysTopology -+import rteval.cpulist_utils as cpulist_utils - --expand_cpulist = CpuList.expand_cpulist -+expand_cpulist = cpulist_utils.expand_cpulist - - class RunData: - '''class to keep instance data from a cyclictest run''' -@@ -201,9 +202,9 @@ class Cyclictest(rtevalModulePrototype): - self.__cpulist = self.__cfg.cpulist - self.__cpus = expand_cpulist(self.__cpulist) - # Only include online cpus -- self.__cpus = CpuList(self.__cpus).cpulist -+ self.__cpus = cpulist_utils.online_cpulist(self.__cpus) - # Reset cpulist from the newly calculated self.__cpus -- self.__cpulist = CpuList.collapse_cpulist(self.__cpus) -+ self.__cpulist = cpulist_utils.collapse_cpulist(self.__cpus) - self.__cpus = [str(c) for c in self.__cpus] - self.__sparse = True - if self.__run_on_isolcpus: -@@ -220,7 +221,7 @@ class Cyclictest(rtevalModulePrototype): - self.__cpus = [c for c in self.__cpus if c in cpuset or self.__run_on_isolcpus and c in isolcpus] - if self.__run_on_isolcpus: - self.__sparse = True -- self.__cpulist = CpuList.collapse_cpulist([int(c) for c in self.__cpus]) -+ self.__cpulist = cpulist_utils.collapse_cpulist([int(c) for c in self.__cpus]) - - # Sort the list of cpus to align with the order reported by cyclictest - self.__cpus.sort(key=int) -diff --git a/rteval/systopology.py b/rteval/systopology.py -index bed5192..9e45762 100644 ---- a/rteval/systopology.py -+++ b/rteval/systopology.py -@@ -9,12 +9,8 @@ - import os - import os.path - import glob -- -- --def sysread(path, obj): -- """ Helper function for reading system files """ -- with open(os.path.join(path, obj), "r") as fp: -- return fp.readline().strip() -+import rteval.cpulist_utils as cpulist_utils -+from rteval.cpulist_utils import sysread - - def cpuinfo(): - ''' return a dictionary of cpu keys with various cpu information ''' -@@ -56,145 +52,6 @@ def cpuinfo(): - return info - - --# --# class to provide access to a list of cpus --# -- --class CpuList: -- "Object that represents a group of system cpus" -- -- cpupath = '/sys/devices/system/cpu' -- -- def __init__(self, cpulist): -- if isinstance(cpulist, list): -- self.cpulist = cpulist -- elif isinstance(cpulist, str): -- self.cpulist = self.expand_cpulist(cpulist) -- self.cpulist = self.online_cpulist(self.cpulist) -- self.cpulist.sort() -- -- def __str__(self): -- return self.collapse_cpulist(self.cpulist) -- -- def __contains__(self, cpu): -- return cpu in self.cpulist -- -- def __len__(self): -- return len(self.cpulist) -- -- def getcpulist(self): -- """ return the list of cpus tracked """ -- return self.cpulist -- -- @staticmethod -- def online_file_exists(): -- """ Check whether machine / kernel is configured with online file """ -- # Note: some machines do not have cpu0/online so we check cpu1/online. -- # In the case of machines with a single CPU, there is no cpu1, but -- # that is not a problem, since a single CPU cannot be offline -- return os.path.exists(os.path.join(CpuList.cpupath, "cpu1/online")) -- -- @staticmethod -- def isolated_file_exists(): -- """ Check whether machine / kernel is configured with isolated file """ -- return os.path.exists(os.path.join(CpuList.cpupath, "isolated")) -- -- @staticmethod -- def collapse_cpulist(cpulist): -- """ -- Collapse a list of cpu numbers into a string range -- of cpus (e.g. 0-5, 7, 9) -- """ -- cur_range = [None, None] -- result = [] -- for cpu in cpulist + [None]: -- if cur_range[0] is None: -- cur_range[0] = cur_range[1] = cpu -- continue -- if cpu is not None and cpu == cur_range[1] + 1: -- # Extend currently processed range -- cur_range[1] += 1 -- else: -- # Range processing finished, add range to string -- result.append(f"{cur_range[0]}-{cur_range[1]}" -- if cur_range[0] != cur_range[1] -- else str(cur_range[0])) -- # Reset -- cur_range[0] = cur_range[1] = cpu -- return ",".join(result) -- -- @staticmethod -- def compress_cpulist(cpulist): -- """ return a string representation of cpulist """ -- if not cpulist: -- return "" -- if isinstance(cpulist[0], int): -- return ",".join(str(e) for e in cpulist) -- return ",".join(cpulist) -- -- @staticmethod -- def expand_cpulist(cpulist): -- """ expand a range string into an array of cpu numbers -- don't error check against online cpus -- """ -- result = [] -- -- if not cpulist: -- return result -- -- for part in cpulist.split(','): -- if '-' in part: -- a, b = part.split('-') -- a, b = int(a), int(b) -- result.extend(list(range(a, b + 1))) -- else: -- a = int(part) -- result.append(a) -- return [int(i) for i in list(set(result))] -- -- @staticmethod -- def is_online(n): -- """ check whether cpu n is online """ -- path = os.path.join(CpuList.cpupath, f'cpu{n}') -- -- # Some hardware doesn't allow cpu0 to be turned off -- if not os.path.exists(path + '/online') and n == 0: -- return True -- -- return sysread(path, "online") == "1" -- -- @staticmethod -- def online_cpulist(cpulist): -- """ Given a cpulist, return a cpulist of online cpus """ -- # This only works if the sys online files exist -- if not CpuList.online_file_exists(): -- return cpulist -- newlist = [] -- for cpu in cpulist: -- if not CpuList.online_file_exists() and cpu == '0': -- newlist.append(cpu) -- elif CpuList.is_online(int(cpu)): -- newlist.append(cpu) -- return newlist -- -- @staticmethod -- def isolated_cpulist(cpulist): -- """Given a cpulist, return a cpulist of isolated CPUs""" -- if not CpuList.isolated_file_exists(): -- return cpulist -- isolated_cpulist = sysread(CpuList.cpupath, "isolated") -- isolated_cpulist = CpuList.expand_cpulist(isolated_cpulist) -- return list(set(isolated_cpulist) & set(cpulist)) -- -- @staticmethod -- def nonisolated_cpulist(cpulist): -- """Given a cpulist, return a cpulist of non-isolated CPUs""" -- if not CpuList.isolated_file_exists(): -- return cpulist -- isolated_cpulist = sysread(CpuList.cpupath, "isolated") -- isolated_cpulist = CpuList.expand_cpulist(isolated_cpulist) -- return list(set(cpulist).difference(set(isolated_cpulist))) -- - # - # class to abstract access to NUMA nodes in /sys filesystem - # -@@ -208,7 +65,8 @@ class NumaNode: - """ - self.path = path - self.nodeid = int(os.path.basename(path)[4:].strip()) -- self.cpus = CpuList(sysread(self.path, "cpulist")) -+ self.cpus = cpulist_utils.expand_cpulist(sysread(self.path, "cpulist")) -+ self.cpus = cpulist_utils.online_cpulist(self.cpus) - self.getmeminfo() - - def __contains__(self, cpu): -@@ -240,11 +98,11 @@ class NumaNode: - - def getcpustr(self): - """ return list of cpus for this node as a string """ -- return str(self.cpus) -+ return cpulist_utils.collapse_cpulist(self.cpus) - - def getcpulist(self): - """ return list of cpus for this node """ -- return self.cpus.getcpulist() -+ return self.cpus - - class SimNumaNode(NumaNode): - """class representing a simulated NUMA node. -@@ -257,7 +115,8 @@ class SimNumaNode(NumaNode): - - def __init__(self): - self.nodeid = 0 -- self.cpus = CpuList(sysread(SimNumaNode.cpupath, "possible")) -+ self.cpus = cpulist_utils.expand_cpulist(sysread(SimNumaNode.cpupath, "possible")) -+ self.cpus = cpulist_utils.online_cpulist(self.cpus) - self.getmeminfo() - - def getmeminfo(self): -@@ -339,7 +198,7 @@ class SysTopology: - """ return a list of integers of all online cpus """ - cpulist = [] - for n in self.nodes: -- cpulist += self.getcpus(n) -+ cpulist += cpulist_utils.online_cpulist(self.getcpus(n)) - cpulist.sort() - return cpulist - -@@ -347,7 +206,7 @@ class SysTopology: - """ return a list of integers of all isolated cpus """ - cpulist = [] - for n in self.nodes: -- cpulist += CpuList.isolated_cpulist(self.getcpus(n)) -+ cpulist += cpulist_utils.isolated_cpulist(self.getcpus(n)) - cpulist.sort() - return cpulist - -@@ -355,7 +214,7 @@ class SysTopology: - """ return a list of integers of all default schedulable cpus, i.e. online non-isolated cpus """ - cpulist = [] - for n in self.nodes: -- cpulist += CpuList.nonisolated_cpulist(self.getcpus(n)) -+ cpulist += cpulist_utils.nonisolated_cpulist(self.getcpus(n)) - cpulist.sort() - return cpulist - -@@ -403,7 +262,7 @@ if __name__ == "__main__": - - onlcpus = s.online_cpus() - print(f'onlcpus = {onlcpus}') -- onlcpus = CpuList.collapse_cpulist(onlcpus) -+ onlcpus = cpulist_utils.collapse_cpulist(onlcpus) - print(f'onlcpus = {onlcpus}') - - onlcpus_str = s.online_cpus_str() --- -2.43.0 - diff --git a/rteval-Cyclictest.py-Make-standalone-file-work-again.patch b/rteval-Cyclictest.py-Make-standalone-file-work-again.patch deleted file mode 100644 index fb6354f..0000000 --- a/rteval-Cyclictest.py-Make-standalone-file-work-again.patch +++ /dev/null @@ -1,88 +0,0 @@ -From ea72c99ead85d1fcf156b9a3825a547ba7f4d0f8 Mon Sep 17 00:00:00 2001 -From: John Kacur -Date: Fri, 12 Apr 2024 14:34:11 -0400 -Subject: [PATCH 01/13] rteval: Cyclictest.py: Make standalone file work again - -Make standalone Cyclictest.py work again for testing purposes -- remove unused parse_cpulist_from_config -- Instead of "import as", use "from" and the requested functionality -- Obtain the default buckets from the ModuleParameters to use if a - number is not otherwise provided -- set the cpulist to "" if not otherwise provided -- add a few docstrings to functions -- obtain a default cpulist from online_cpus for the standalone test - -Signed-off-by: John Kacur ---- - rteval/modules/measurement/cyclictest.py | 19 ++++++++----------- - 1 file changed, 8 insertions(+), 11 deletions(-) - -diff --git a/rteval/modules/measurement/cyclictest.py b/rteval/modules/measurement/cyclictest.py -index 722422589d62..671426e13c4d 100644 ---- a/rteval/modules/measurement/cyclictest.py -+++ b/rteval/modules/measurement/cyclictest.py -@@ -16,10 +16,8 @@ import math - import libxml2 - from rteval.Log import Log - from rteval.modules import rtevalModulePrototype --from rteval.systopology import cpuinfo, parse_cpulist_from_config --import rteval.cpulist_utils as cpulist_utils -- --expand_cpulist = cpulist_utils.expand_cpulist -+from rteval.systopology import cpuinfo, SysTopology -+from rteval.cpulist_utils import expand_cpulist, collapse_cpulist - - class RunData: - '''class to keep instance data from a cyclictest run''' -@@ -190,10 +188,11 @@ class Cyclictest(rtevalModulePrototype): - # Create a RunData object per CPU core - self.__numanodes = int(self.__cfg.setdefault('numanodes', 0)) - self.__priority = int(self.__cfg.setdefault('priority', 95)) -- self.__buckets = int(self.__cfg.setdefault('buckets', 2000)) -+ default_buckets = ModuleParameters()["buckets"]["default"] -+ self.__buckets = int(self.__cfg.setdefault('buckets', default_buckets)) - self.__numcores = 0 - self.__cyclicdata = {} -- self.__cpulist = self.__cfg.cpulist -+ self.__cpulist = self.__cfg.setdefault('cpulist', "") - self.__cpus = [str(c) for c in expand_cpulist(self.__cpulist)] - self.__numcores = len(self.__cpus) - -@@ -393,14 +392,13 @@ class Cyclictest(rtevalModulePrototype): - return rep_n - - -- - def ModuleInfo(): - return {"parallel": True, - "loads": True} - - -- - def ModuleParameters(): -+ """ default parameters """ - return {"interval": {"descr": "Base interval of the threads in microseconds", - "default": 100, - "metavar": "INTV_US"}, -@@ -421,6 +419,7 @@ def ModuleParameters(): - - - def create(params, logger): -+ """ Instantiate a Cyclictest measurement module object """ - return Cyclictest(params, logger) - - -@@ -438,9 +437,7 @@ if __name__ == '__main__': - cfg.AppendConfig('cyclictest', prms) - - cfg_ct = cfg.GetSection('cyclictest') -- cfg_ct.reportdir = "." -- cfg_ct.buckets = 200 -- # cfg_ct.breaktrace = 30 -+ cfg_ct.cpulist = collapse_cpulist(SysTopology().online_cpus()) - - runtime = 10 - --- -2.44.0 - diff --git a/rteval-Disable-use-of-python-dmidecode.patch b/rteval-Disable-use-of-python-dmidecode.patch deleted file mode 100644 index ceed1b5..0000000 --- a/rteval-Disable-use-of-python-dmidecode.patch +++ /dev/null @@ -1,87 +0,0 @@ -From d142f0d23d8df1cede3573c3d6cfbf16535b3475 Mon Sep 17 00:00:00 2001 -From: John Kacur -Date: Wed, 20 Dec 2023 17:55:21 -0500 -Subject: [PATCH 2/2] rteval: Disable use of python-dmidecode - -python-dmidecode is not being maintained upstream. -For now just disable it's use in rteval since it is useful but not -essential information for running rteval - -In the future look at generating this info directly using dmidecode. - -Signed-off-by: John Kacur ---- - README | 6 ------ - doc/installing.txt | 9 --------- - rteval/sysinfo/dmi.py | 4 ++-- - 3 files changed, 2 insertions(+), 17 deletions(-) - -diff --git a/README b/README -index a5cf98344a46..b352d7f66ad2 100644 ---- a/README -+++ b/README -@@ -19,15 +19,9 @@ Rteval requires the following packages to run: - Python >= 3.0 - http://www.python.org/download/ - --python-ethtool -- git://git.kernel.org/pub/scm/linux/kernel/git/acme/python-ethtool.git -- - python-lxml - http://lxml.de/ - --python-dmidecode -- http://www.ohloh.net/p/python-dmidecode -- - libxml2-python - http://xmlsoft.org/ - -diff --git a/doc/installing.txt b/doc/installing.txt -index ff2d43cb9481..227249bbc9ed 100644 ---- a/doc/installing.txt -+++ b/doc/installing.txt -@@ -1,18 +1,10 @@ - The rteval utility requires some external software libraries to run - properly. These are: - --python-ethtool -- A python library to query network interfaces -- git://git.kernel.org/pub/scm/linux/kernel/git/acme/python-ethtool.git -- - python-lxml - A python library to parse XML files and XSLT stylesheets - http://lxml.de/ - --python-dmidecode -- A python library used to access DMI table information -- http://www.autonomy.net.au/display/pydmi/Home -- - libxml2-python - A python library to parse XML files - http://xmlsoft.org/ -@@ -22,7 +14,6 @@ rt-tests - git://git.kernel.org/pub/scm/utils/rt-tests/rt-tests.git - - --$ sudo yum install python-{dmidecode,ethtool) - $ git clone \ - git://git.kernel.org/pub/scm/utils/rt-tests/rt-tests.git - $ cd rt-tests && sudo make prefix=/usr install -diff --git a/rteval/sysinfo/dmi.py b/rteval/sysinfo/dmi.py -index e8285d263fe6..c01a0eef1435 100644 ---- a/rteval/sysinfo/dmi.py -+++ b/rteval/sysinfo/dmi.py -@@ -15,8 +15,8 @@ from rteval import xmlout - from rteval import rtevalConfig - - try: -- import dmidecode -- dmidecode_avail = True -+ # import dmidecode -+ dmidecode_avail = False - except ModuleNotFoundError: - dmidecode_avail = False - --- -2.42.0 - diff --git a/rteval-Generate-raw-histogram-data-for-an-existing-t.patch b/rteval-Generate-raw-histogram-data-for-an-existing-t.patch deleted file mode 100644 index 13d5e7a..0000000 --- a/rteval-Generate-raw-histogram-data-for-an-existing-t.patch +++ /dev/null @@ -1,69 +0,0 @@ -From 843ff713340ab701aa8859ca8b3d8a776608175c Mon Sep 17 00:00:00 2001 -From: John Kacur -Date: Mon, 29 Apr 2024 10:47:41 -0400 -Subject: [PATCH 09/13] rteval: Generate raw histogram data for an existing - timerlat report - -Generate raw histogram data for an existing timerlat report. - -Signed-off-by: John Kacur ---- - rteval/rteval_histogram_raw.xsl | 29 ++++++++++++++++++++++++++++- - 1 file changed, 28 insertions(+), 1 deletion(-) - -diff --git a/rteval/rteval_histogram_raw.xsl b/rteval/rteval_histogram_raw.xsl -index 038419bf03b7..00b2be34f305 100644 ---- a/rteval/rteval_histogram_raw.xsl -+++ b/rteval/rteval_histogram_raw.xsl -@@ -12,15 +12,26 @@ - - - -- -+ - - - -+ -+ -+ -+ -+ - - - - - -+ -+ -+ -+ -+ -+ - - - -@@ -43,4 +54,20 @@ - - - -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ -+ - --- -2.44.0 - diff --git a/rteval-Implement-initial-dmidecode-support.patch b/rteval-Implement-initial-dmidecode-support.patch deleted file mode 100644 index 4ed3fe8..0000000 --- a/rteval-Implement-initial-dmidecode-support.patch +++ /dev/null @@ -1,299 +0,0 @@ -From 43c45ba7dae488c16beb359834b3a71ebf6068d6 Mon Sep 17 00:00:00 2001 -From: Tomas Glozar -Date: Mon, 4 Mar 2024 11:26:03 +0100 -Subject: [PATCH] rteval: Implement initial dmidecode support - -Previously rteval used python-dmidecode to gather DMI data from a -system. Since python-dmidecode is without a maintainer, its support was -removed in d142f0d2 ("rteval: Disable use of python-dmidecode"). - -Add get_dmidecode_xml() function into rteval/sysinfo/dmi.py that does -simple parsing of dmidecode command-line tool output without any -structure changes and include it into the rteval report. - -Notes: -- ProcessWarnings() in rteval.sysinfo.dmi was reworked into a class - method of DMIinfo and to use the class's __log field as logger. It - now also does not ignore warnings that appear when running rteval as - non-root, since that is no longer supported. Additionally, - a duplicate call in rteval-cmd was removed. -- rteval/rteval_dmi.xsl XSLT template was left untouched and is - currectly not used. In a future commit, it is expected to be rewritten - to transform the XML format outputted by get_dmidecode_xml() into the - same format that was used with python-dmidecode. - -Signed-off-by: Tomas Glozar -Signed-off-by: John Kacur ---- - rteval-cmd | 2 - - rteval/sysinfo/__init__.py | 2 +- - rteval/sysinfo/dmi.py | 178 ++++++++++++++++++++++++------------- - 3 files changed, 118 insertions(+), 64 deletions(-) - -diff --git a/rteval-cmd b/rteval-cmd -index a5e8746..018a414 100755 ---- a/rteval-cmd -+++ b/rteval-cmd -@@ -268,8 +268,6 @@ if __name__ == '__main__': - | (rtevcfg.debugging and Log.DEBUG) - logger.SetLogVerbosity(loglev) - -- dmi.ProcessWarnings(logger=logger) -- - # Load modules - loadmods = LoadModules(config, logger=logger) - measuremods = MeasurementModules(config, logger=logger) -diff --git a/rteval/sysinfo/__init__.py b/rteval/sysinfo/__init__.py -index d3f9efb..09af52e 100644 ---- a/rteval/sysinfo/__init__.py -+++ b/rteval/sysinfo/__init__.py -@@ -30,7 +30,7 @@ class SystemInfo(KernelInfo, SystemServices, dmi.DMIinfo, CPUtopology, - NetworkInfo.__init__(self, logger=logger) - - # Parse initial DMI decoding errors -- dmi.ProcessWarnings(logger=logger) -+ self.ProcessWarnings() - - # Parse CPU info - CPUtopology._parse(self) -diff --git a/rteval/sysinfo/dmi.py b/rteval/sysinfo/dmi.py -index c01a0ee..f1aab9f 100644 ---- a/rteval/sysinfo/dmi.py -+++ b/rteval/sysinfo/dmi.py -@@ -3,6 +3,7 @@ - # Copyright 2009 - 2013 Clark Williams - # Copyright 2009 - 2013 David Sommerseth - # Copyright 2022 John Kacur -+# Copyright 2024 Tomas Glozar - # - """ dmi.py class to wrap DMI Table Information """ - -@@ -10,65 +11,125 @@ import sys - import os - import libxml2 - import lxml.etree -+import shutil -+import re -+from subprocess import Popen, PIPE, SubprocessError - from rteval.Log import Log - from rteval import xmlout - from rteval import rtevalConfig - --try: -- # import dmidecode -- dmidecode_avail = False --except ModuleNotFoundError: -- dmidecode_avail = False -- --def set_dmidecode_avail(val): -- """ Used to set global variable dmidecode_avail from a function """ -- global dmidecode_avail -- dmidecode_avail = val -- --def ProcessWarnings(logger=None): -- """ Process Warnings from dmidecode """ -- -- if not dmidecode_avail: -- return -- -- if not hasattr(dmidecode, 'get_warnings'): -- return -- -- warnings = dmidecode.get_warnings() -- if warnings is None: -- return -- -- ignore1 = '/dev/mem: Permission denied' -- ignore2 = 'No SMBIOS nor DMI entry point found, sorry.' -- ignore3 = 'Failed to open memory buffer (/dev/mem): Permission denied' -- ignore = (ignore1, ignore2, ignore3) -- for warnline in warnings.split('\n'): -- # Ignore these warnings, as they are "valid" if not running as root -- if warnline in ignore: -- continue - -- # All other warnings will be printed -- if len(warnline) > 0: -- logger.log(Log.DEBUG, f"** DMI WARNING ** {warnline}") -- set_dmidecode_avail(False) -+def get_dmidecode_xml(dmidecode_executable): -+ """ -+ Transform human-readable dmidecode output into machine-processable XML format -+ :param dmidecode_executable: Path to dmidecode tool executable -+ :return: Tuple of values with resulting XML and dmidecode warnings -+ """ -+ proc = Popen(dmidecode_executable, text=True, stdout=PIPE, stderr=PIPE) -+ outs, errs = proc.communicate() -+ parts = outs.split("\n\n") -+ if len(parts) < 2: -+ raise RuntimeError("Parsing dmidecode output failed") -+ header = parts[0] -+ handles = parts[1:] -+ root = lxml.etree.Element("dmidecode") -+ # Parse dmidecode output header -+ # Note: Only supports SMBIOS data currently -+ regex = re.compile(r"# dmidecode (\d+\.\d+)\n" -+ r"Getting SMBIOS data from sysfs\.\n" -+ r"SMBIOS ((?:\d+\.)+\d+) present\.\n" -+ r"(?:(\d+) structures occupying (\d+) bytes\.\n)?" -+ r"Table at (0x[0-9A-Fa-f]+)\.", re.MULTILINE) -+ match = re.match(regex, header) -+ if match is None: -+ raise RuntimeError("Parsing dmidecode output failed") -+ root.attrib["dmidecodeversion"] = match.group(1) -+ root.attrib["smbiosversion"] = match.group(2) -+ if match.group(3) is not None: -+ root.attrib["structures"] = match.group(3) -+ if match.group(4) is not None: -+ root.attrib["size"] = match.group(4) -+ root.attrib["address"] = match.group(5) -+ -+ # Generate element per handle in dmidecode output -+ for handle_text in handles: -+ if not handle_text: -+ # Empty line -+ continue - -- dmidecode.clear_warnings() -+ handle = lxml.etree.Element("Handle") -+ lines = handle_text.splitlines() -+ # Parse handle header -+ if len(lines) < 2: -+ raise RuntimeError("Parsing dmidecode handle failed") -+ header, name, content = lines[0], lines[1], lines[2:] -+ match = re.match(r"Handle (0x[0-9A-Fa-f]{4}), " -+ r"DMI type (\d+), (\d+) bytes", header) -+ if match is None: -+ raise RuntimeError("Parsing dmidecode handle failed") -+ handle.attrib["address"] = match.group(1) -+ handle.attrib["type"] = match.group(2) -+ handle.attrib["bytes"] = match.group(3) -+ handle.attrib["name"] = name -+ -+ # Parse all fields in handle and create an element for each -+ list_field = None -+ for index, line in enumerate(content): -+ line = content[index] -+ if line.rfind("\t") > 0: -+ # We are inside a list field, add value to it -+ value = lxml.etree.Element("Value") -+ value.text = line.strip() -+ list_field.append(value) -+ continue -+ line = line.lstrip().split(":", 1) -+ if len(line) != 2: -+ raise RuntimeError("Parsing dmidecode field failed") -+ if not line[1] or (index + 1 < len(content) and -+ content[index + 1].rfind("\t") > 0): -+ # No characters after : or next line is inside list field -+ # means a list field -+ # Note: there are list fields which specify a number of -+ # items, for example "Installable Languages", so merely -+ # checking for no characters after : is not enough -+ list_field = lxml.etree.Element("List") -+ list_field.attrib["Name"] = line[0].strip() -+ handle.append(list_field) -+ else: -+ # Regular field -+ field = lxml.etree.Element("Field") -+ field.attrib["Name"] = line[0].strip() -+ field.text = line[1].strip() -+ handle.append(field) -+ -+ root.append(handle) -+ -+ return root, errs - - - class DMIinfo: -- '''class used to obtain DMI info via python-dmidecode''' -+ '''class used to obtain DMI info via dmidecode''' - - def __init__(self, logger=None): - self.__version = '0.6' - self._log = logger - -- if not dmidecode_avail: -- logger.log(Log.DEBUG, "DMI info unavailable, ignoring DMI tables") -+ dmidecode_executable = shutil.which("dmidecode") -+ if dmidecode_executable is None: -+ logger.log(Log.DEBUG, "DMI info unavailable," -+ " ignoring DMI tables") - self.__fake = True - return - - self.__fake = False -- self.__dmixml = dmidecode.dmidecodeXML() -+ try: -+ self.__dmixml, self.__warnings = get_dmidecode_xml( -+ dmidecode_executable) -+ except (RuntimeError, OSError, SubprocessError) as error: -+ logger.log(Log.DEBUG, "DMI info unavailable: {};" -+ " ignoring DMI tables".format(str(error))) -+ self.__fake = True -+ return - - self.__xsltparser = self.__load_xslt('rteval_dmi.xsl') - -@@ -88,30 +149,25 @@ class DMIinfo: - - raise RuntimeError(f'Could not locate XSLT template for DMI data ({fname})') - -+ def ProcessWarnings(self): -+ """Prints out warnings from dmidecode into log if there were any""" -+ if self.__fake or self._log is None: -+ return -+ for warnline in self.__warnings.split('\n'): -+ if len(warnline) > 0: -+ self._log.log(Log.DEBUG, f"** DMI WARNING ** {warnline}") -+ - def MakeReport(self): - """ Add DMI information to final report """ -- rep_n = libxml2.newNode("DMIinfo") -- rep_n.newProp("version", self.__version) - if self.__fake: -+ rep_n = libxml2.newNode("DMIinfo") -+ rep_n.newProp("version", self.__version) - rep_n.addContent("No DMI tables available") - rep_n.newProp("not_available", "1") -- else: -- self.__dmixml.SetResultType(dmidecode.DMIXML_DOC) -- try: -- dmiqry = xmlout.convert_libxml2_to_lxml_doc(self.__dmixml.QuerySection('all')) -- except Exception as ex1: -- self._log.log(Log.DEBUG, f'** EXCEPTION {str(ex1)}, will query BIOS only') -- try: -- # If we can't query 'all', at least query 'bios' -- dmiqry = xmlout.convert_libxml2_to_lxml_doc(self.__dmixml.QuerySection('bios')) -- except Exception as ex2: -- rep_n.addContent("No DMI tables available") -- rep_n.newProp("not_available", "1") -- self._log.log(Log.DEBUG, f'** EXCEPTION {str(ex2)}, dmi info will not be reported') -- return rep_n -- resdoc = self.__xsltparser(dmiqry) -- dmi_n = xmlout.convert_lxml_to_libxml2_nodes(resdoc.getroot()) -- rep_n.addChild(dmi_n) -+ return rep_n -+ rep_n = xmlout.convert_lxml_to_libxml2_nodes(self.__dmixml) -+ rep_n.setName("DMIinfo") -+ rep_n.newProp("version", self.__version) - return rep_n - - def unit_test(rootdir): -@@ -130,12 +186,12 @@ def unit_test(rootdir): - log = Log() - log.SetLogVerbosity(Log.DEBUG|Log.INFO) - -- ProcessWarnings(logger=log) - if os.getuid() != 0: - print("** ERROR ** Must be root to run this unit_test()") - return 1 - - d = DMIinfo(logger=log) -+ d.ProcessWarnings() - dx = d.MakeReport() - x = libxml2.newDoc("1.0") - x.setRootElement(dx) --- -2.44.0 - diff --git a/rteval-Make-output-work-with-new-dmiinfo-format.patch b/rteval-Make-output-work-with-new-dmiinfo-format.patch deleted file mode 100644 index 452011d..0000000 --- a/rteval-Make-output-work-with-new-dmiinfo-format.patch +++ /dev/null @@ -1,60 +0,0 @@ -From 701ab9f5d50fb0e1308ad8ea163d49525d3d90af Mon Sep 17 00:00:00 2001 -From: Tomas Glozar -Date: Mon, 29 Apr 2024 10:47:20 +0200 -Subject: [PATCH 10/13] rteval: Make output work with new dmiinfo format - -Commit 43c45ba7 ("rteval: Implement initial dmidecode support") -re-introduced DMI information into rteval XML summary, but in a -different format, which is not correctly picked up in rteval_text.xsl -for formatting BIOS and vendor information into rteval text output. - -This lead to the information not being displayed correctly: - Model: - - BIOS version: (ver: , rev :, release date: ) - -Modify the relevant section of rteval_text.xsl to make it understand the -new format. Also, adjust the incorrect spacing around "rev". - -New output (in QEMU VM): - Model: QEMU - Standard PC (Q35 + ICH9, 2009) - BIOS version: SeaBIOS (ver: 1.16.3-1.fc39, rev: 0.0, release date: 04/01/2014) - -Signed-off-by: Tomas Glozar -Signed-off-by: John Kacur ---- - rteval/rteval_text.xsl | 14 +++++++------- - 1 file changed, 7 insertions(+), 7 deletions(-) - -diff --git a/rteval/rteval_text.xsl b/rteval/rteval_text.xsl -index 1e9c0f0d26c5..2f03bda0bb55 100644 ---- a/rteval/rteval_text.xsl -+++ b/rteval/rteval_text.xsl -@@ -39,18 +39,18 @@ - - - Model: -- -- - -+ -+ - - - - BIOS version: -- -+ - (ver: -- -- , rev : -- -+ -+ , rev: -+ - , release date: -- -+ - ) - - --- -2.44.0 - diff --git a/rteval-Makefile-More-rpm-cleanups.patch b/rteval-Makefile-More-rpm-cleanups.patch deleted file mode 100644 index abca54c..0000000 --- a/rteval-Makefile-More-rpm-cleanups.patch +++ /dev/null @@ -1,64 +0,0 @@ -From 2d2e85c459d240926c99b1961bbef090aa80a1fc Mon Sep 17 00:00:00 2001 -From: John Kacur -Date: Wed, 20 Dec 2023 17:22:22 -0500 -Subject: [PATCH 1/2] rteval: Makefile: More rpm cleanups - -Afer having removed the upstream specfile, there were still a few -references to rpms in the Makefile. These are not necessary because any -work with rpms can be done with modern rpm tools and are typically done -by distributions - -Also test whether directory 'run' exists since it may have been removed -by make realclean, and create it if it does not - -Signed-off-by: John Kacur ---- - Makefile | 11 +++-------- - 1 file changed, 3 insertions(+), 8 deletions(-) - -diff --git a/Makefile b/Makefile -index ee4cca555b95..b8bed643f760 100644 ---- a/Makefile -+++ b/Makefile -@@ -31,6 +31,7 @@ load: - $(PYTHON) rteval-cmd --onlyload -D -L -v --workdir=./run --loaddir=$(HERE)/loadsource -f $(HERE)/rteval/rteval.conf -i $(HERE)/rteval - - sysreport: -+ [ -d $(HERE)/run ] || mkdir run - $(PYTHON) rteval-cmd -D -v --workdir=$(HERE)/run --loaddir=$(HERE)/loadsource --duration=$(D) -i $(HERE)/rteval --sysreport - - clean: -@@ -39,7 +40,7 @@ clean: - - realclean: clean - [ -f $(XMLRPCDIR)/Makefile ] && make -C $(XMLRPCDIR) maintainer-clean || echo -n -- rm -rf run rpm -+ rm -rf run - - install: install_loads install_rteval - -@@ -73,13 +74,6 @@ rteval-xmlrpc-$(XMLRPCVER).tar.gz : - make distcheck - cp $(XMLRPCDIR)/rteval-xmlrpc-$(XMLRPCVER).tar.gz $(HERE)/ - --rpm_prep: -- rm -rf rpm -- mkdir -p rpm/{BUILD,RPMS,SRPMS,SOURCES,SPECS} -- --rpms rpm: rpm_prep rtevalrpm loadrpm -- -- - help: - @echo "" - @echo "rteval Makefile targets:" -@@ -88,6 +82,7 @@ help: - @echo " tarfile: create the source tarball" - @echo " install: install rteval locally" - @echo " clean: cleanup generated files" -+ @echo " realclean: Same as clean plus directory run" - @echo " sysreport: do a short testrun and generate sysreport data" - @echo " tags: generate a ctags file" - @echo " cleantags: remove the ctags file" --- -2.42.0 - diff --git a/rteval-Minor-improvements-to-CpuList-class.patch b/rteval-Minor-improvements-to-CpuList-class.patch deleted file mode 100644 index 44430e0..0000000 --- a/rteval-Minor-improvements-to-CpuList-class.patch +++ /dev/null @@ -1,85 +0,0 @@ -From 768daa63ec8e2299da53afe081e7304b37fc91cf Mon Sep 17 00:00:00 2001 -From: Tomas Glozar -Date: Wed, 29 Nov 2023 10:34:55 +0100 -Subject: [PATCH 2/9] rteval: Minor improvements to CpuList class - -- Remove unnecessary if-else from online_file_exists -- Use cpupath in online_file_exists -- In is_online, remove check for n in cpuset and make it static -- Mark also the remaining methods static since they do not rely on -any fields of the class - -Signed-off-by: Tomas Glozar -- Removed incorrect line from commit message -Signed-off-by: John Kacur ---- - rteval/systopology.py | 29 +++++++++++++++-------------- - 1 file changed, 15 insertions(+), 14 deletions(-) - -diff --git a/rteval/systopology.py b/rteval/systopology.py -index ea8e242..60ac8e8 100644 ---- a/rteval/systopology.py -+++ b/rteval/systopology.py -@@ -82,12 +82,17 @@ class CpuList: - def __len__(self): - return len(self.cpulist) - -+ def getcpulist(self): -+ """ return the list of cpus tracked """ -+ return self.cpulist -+ - @staticmethod - def online_file_exists(): - """ Check whether machine / kernel is configured with online file """ -- if os.path.exists('/sys/devices/system/cpu/cpu1/online'): -- return True -- return False -+ # Note: some machines do not have cpu0/online so we check cpu1/online. -+ # In the case of machines with a single CPU, there is no cpu1, but -+ # that is not a problem, since a single CPU cannot be offline -+ return os.path.exists(os.path.join(CpuList.cpupath, "cpu1/online")) - - @staticmethod - def isolated_file_exists(): -@@ -147,14 +152,9 @@ class CpuList: - result.append(a) - return [int(i) for i in list(set(result))] - -- def getcpulist(self): -- """ return the list of cpus tracked """ -- return self.cpulist -- -- def is_online(self, n): -+ @staticmethod -+ def is_online(n): - """ check whether cpu n is online """ -- if n not in self.cpulist: -- raise RuntimeError(f"invalid cpu number {n}") - path = os.path.join(CpuList.cpupath, f'cpu{n}') - - # Some hardware doesn't allow cpu0 to be turned off -@@ -163,16 +163,17 @@ class CpuList: - - return sysread(path, "online") == "1" - -- def online_cpulist(self, cpulist): -+ @staticmethod -+ def online_cpulist(cpulist): - """ Given a cpulist, return a cpulist of online cpus """ - # This only works if the sys online files exist -- if not self.online_file_exists(): -+ if not CpuList.online_file_exists(): - return cpulist - newlist = [] - for cpu in cpulist: -- if not self.online_file_exists() and cpu == '0': -+ if not CpuList.online_file_exists() and cpu == '0': - newlist.append(cpu) -- elif self.is_online(int(cpu)): -+ elif CpuList.is_online(int(cpu)): - newlist.append(cpu) - return newlist - --- -2.43.0 - diff --git a/rteval-Refactor-collapse_cpulist-in-systopology.patch b/rteval-Refactor-collapse_cpulist-in-systopology.patch deleted file mode 100644 index cb43c85..0000000 --- a/rteval-Refactor-collapse_cpulist-in-systopology.patch +++ /dev/null @@ -1,225 +0,0 @@ -From 4b4f50900e38a931ed9585c6cf200b74c0120a20 Mon Sep 17 00:00:00 2001 -From: Tomas Glozar -Date: Wed, 29 Nov 2023 10:34:54 +0100 -Subject: [PATCH 1/9] rteval: Refactor collapse_cpulist in systopology - -Instead of having duplicate code in two functions, one top-level and -one member function of CpuList, have only one static function in -CpuList. - -Additionally re-write the implementation to use a more straight forward -one-pass algorithm. - -Signed-off-by: Tomas Glozar -Signed-off-by: John Kacur ---- - rteval-cmd | 4 +- - rteval/modules/loads/__init__.py | 4 +- - rteval/modules/measurement/__init__.py | 4 +- - rteval/modules/measurement/cyclictest.py | 6 +-- - rteval/systopology.py | 68 ++++++++---------------- - 5 files changed, 30 insertions(+), 56 deletions(-) - -diff --git a/rteval-cmd b/rteval-cmd -index 6f613a3..7c41429 100755 ---- a/rteval-cmd -+++ b/rteval-cmd -@@ -30,7 +30,7 @@ from rteval import RtEval, rtevalConfig - from rteval.modules.loads import LoadModules - from rteval.modules.measurement import MeasurementModules - from rteval.version import RTEVAL_VERSION --from rteval.systopology import CpuList, SysTopology, collapse_cpulist -+from rteval.systopology import CpuList, SysTopology - from rteval.modules.loads.kcompile import ModuleParameters - - compress_cpulist = CpuList.compress_cpulist -@@ -211,7 +211,7 @@ def remove_offline(cpulist): - """ return cpulist in collapsed compressed form with only online cpus """ - tmplist = expand_cpulist(cpulist) - tmplist = SysTopology().online_cpulist(tmplist) -- return collapse_cpulist(tmplist) -+ return CpuList.collapse_cpulist(tmplist) - - - if __name__ == '__main__': -diff --git a/rteval/modules/loads/__init__.py b/rteval/modules/loads/__init__.py -index aca0c9f..13fba1e 100644 ---- a/rteval/modules/loads/__init__.py -+++ b/rteval/modules/loads/__init__.py -@@ -11,7 +11,7 @@ import libxml2 - from rteval.Log import Log - from rteval.rtevalConfig import rtevalCfgSection - from rteval.modules import RtEvalModules, rtevalModulePrototype --from rteval.systopology import collapse_cpulist, CpuList, SysTopology as SysTop -+from rteval.systopology import CpuList, SysTopology as SysTop - - class LoadThread(rtevalModulePrototype): - def __init__(self, name, config, logger=None): -@@ -120,7 +120,7 @@ class LoadModules(RtEvalModules): - cpulist = CpuList(cpulist).cpulist - else: - cpulist = SysTop().default_cpus() -- rep_n.newProp("loadcpus", collapse_cpulist(cpulist)) -+ rep_n.newProp("loadcpus", CpuList.collapse_cpulist(cpulist)) - - return rep_n - -diff --git a/rteval/modules/measurement/__init__.py b/rteval/modules/measurement/__init__.py -index 2a0556b..41b8022 100644 ---- a/rteval/modules/measurement/__init__.py -+++ b/rteval/modules/measurement/__init__.py -@@ -5,7 +5,7 @@ - - import libxml2 - from rteval.modules import RtEvalModules, ModuleContainer --from rteval.systopology import collapse_cpulist, CpuList, SysTopology as SysTop -+from rteval.systopology import CpuList, SysTopology as SysTop - - class MeasurementProfile(RtEvalModules): - """Keeps and controls all the measurement modules with the same measurement profile""" -@@ -187,7 +187,7 @@ measurement profiles, based on their characteristics""" - cpulist = CpuList(cpulist).cpulist - else: - cpulist = SysTop().online_cpus() if run_on_isolcpus else SysTop().default_cpus() -- rep_n.newProp("measurecpus", collapse_cpulist(cpulist)) -+ rep_n.newProp("measurecpus", CpuList.collapse_cpulist(cpulist)) - - for mp in self.__measureprofiles: - mprep_n = mp.MakeReport() -diff --git a/rteval/modules/measurement/cyclictest.py b/rteval/modules/measurement/cyclictest.py -index 0af1d31..1b14e7e 100644 ---- a/rteval/modules/measurement/cyclictest.py -+++ b/rteval/modules/measurement/cyclictest.py -@@ -17,7 +17,7 @@ import libxml2 - from rteval.Log import Log - from rteval.modules import rtevalModulePrototype - from rteval.systopology import cpuinfo --from rteval.systopology import CpuList, SysTopology, collapse_cpulist -+from rteval.systopology import CpuList, SysTopology - - expand_cpulist = CpuList.expand_cpulist - -@@ -203,7 +203,7 @@ class Cyclictest(rtevalModulePrototype): - # Only include online cpus - self.__cpus = CpuList(self.__cpus).cpulist - # Reset cpulist from the newly calculated self.__cpus -- self.__cpulist = collapse_cpulist(self.__cpus) -+ self.__cpulist = CpuList.collapse_cpulist(self.__cpus) - self.__cpus = [str(c) for c in self.__cpus] - self.__sparse = True - if self.__run_on_isolcpus: -@@ -220,7 +220,7 @@ class Cyclictest(rtevalModulePrototype): - self.__cpus = [c for c in self.__cpus if c in cpuset or self.__run_on_isolcpus and c in isolcpus] - if self.__run_on_isolcpus: - self.__sparse = True -- self.__cpulist = collapse_cpulist(self.__cpus) -+ self.__cpulist = CpuList.collapse_cpulist([int(c) for c in self.__cpus]) - - # Sort the list of cpus to align with the order reported by cyclictest - self.__cpus.sort(key=int) -diff --git a/rteval/systopology.py b/rteval/systopology.py -index 62ad355..ea8e242 100644 ---- a/rteval/systopology.py -+++ b/rteval/systopology.py -@@ -10,25 +10,6 @@ import os - import os.path - import glob - --# Utility version of collapse_cpulist that doesn't require a CpuList object --def collapse_cpulist(cpulist): -- """ Collapse a list of cpu numbers into a string range -- of cpus (e.g. 0-5, 7, 9) """ -- if len(cpulist) == 0: -- return "" -- idx = CpuList.longest_sequence(cpulist) -- if idx == 0: -- seq = str(cpulist[0]) -- else: -- if idx == 1: -- seq = f"{cpulist[0]},{cpulist[idx]}" -- else: -- seq = f"{cpulist[0]}-{cpulist[idx]}" -- -- rest = collapse_cpulist(cpulist[idx+1:]) -- if rest == "": -- return seq -- return ",".join((seq, rest)) - - def sysread(path, obj): - """ Helper function for reading system files """ -@@ -93,7 +74,7 @@ class CpuList: - self.cpulist.sort() - - def __str__(self): -- return self.__collapse_cpulist(self.cpulist) -+ return self.collapse_cpulist(self.cpulist) - - def __contains__(self, cpu): - return cpu in self.cpulist -@@ -114,35 +95,28 @@ class CpuList: - return os.path.exists(os.path.join(CpuList.cpupath, "isolated")) - - @staticmethod -- def longest_sequence(cpulist): -- """ return index of last element of a sequence that steps by one """ -- lim = len(cpulist) -- for idx, _ in enumerate(cpulist): -- if idx+1 == lim: -- break -- if int(cpulist[idx+1]) != (int(cpulist[idx])+1): -- return idx -- return lim - 1 -- -- def __collapse_cpulist(self, cpulist): -- """ Collapse a list of cpu numbers into a string range -+ def collapse_cpulist(cpulist): -+ """ -+ Collapse a list of cpu numbers into a string range - of cpus (e.g. 0-5, 7, 9) - """ -- if len(cpulist) == 0: -- return "" -- idx = self.longest_sequence(cpulist) -- if idx == 0: -- seq = str(cpulist[0]) -- else: -- if idx == 1: -- seq = f"{cpulist[0]},{cpulist[idx]}" -+ cur_range = [None, None] -+ result = [] -+ for cpu in cpulist + [None]: -+ if cur_range[0] is None: -+ cur_range[0] = cur_range[1] = cpu -+ continue -+ if cpu is not None and cpu == cur_range[1] + 1: -+ # Extend currently processed range -+ cur_range[1] += 1 - else: -- seq = f"{cpulist[0]}-{cpulist[idx]}" -- -- rest = self.__collapse_cpulist(cpulist[idx+1:]) -- if rest == "": -- return seq -- return ",".join((seq, rest)) -+ # Range processing finished, add range to string -+ result.append(f"{cur_range[0]}-{cur_range[1]}" -+ if cur_range[0] != cur_range[1] -+ else str(cur_range[0])) -+ # Reset -+ cur_range[0] = cur_range[1] = cpu -+ return ",".join(result) - - @staticmethod - def compress_cpulist(cpulist): -@@ -428,7 +402,7 @@ if __name__ == "__main__": - - onlcpus = s.online_cpus() - print(f'onlcpus = {onlcpus}') -- onlcpus = collapse_cpulist(onlcpus) -+ onlcpus = CpuList.collapse_cpulist(onlcpus) - print(f'onlcpus = {onlcpus}') - - onlcpus_str = s.online_cpus_str() --- -2.43.0 - diff --git a/rteval-Remove-upstream-spec-file.patch b/rteval-Remove-upstream-spec-file.patch deleted file mode 100644 index d35f4cc..0000000 --- a/rteval-Remove-upstream-spec-file.patch +++ /dev/null @@ -1,566 +0,0 @@ -From de8e25ff3a30dbdbba6fb1b68ea0921dff55cd91 Mon Sep 17 00:00:00 2001 -From: John Kacur -Date: Mon, 13 Nov 2023 14:32:19 -0500 -Subject: [PATCH] rteval: Remove upstream spec file - -Specfiles should be maintained by distributions and not in the upstream -code. In practice they are not maintained upstream except for version -numbers, so just remove the specfile. - -This also moves a lot of functionality around rpms in the Makefile, but -this functionality exists in tools such as rpmbuild and friends anyway. - -Signed-off-by: John Kacur ---- - Makefile | 37 ---- - rteval.spec | 484 ---------------------------------------------------- - 2 files changed, 521 deletions(-) - delete mode 100644 rteval.spec - -diff --git a/Makefile b/Makefile -index 14f74e087eff..ee4cca555b95 100644 ---- a/Makefile -+++ b/Makefile -@@ -79,49 +79,12 @@ rpm_prep: - - rpms rpm: rpm_prep rtevalrpm loadrpm - --rtevalrpm: rteval-$(VERSION).tar.bz2 -- cp $^ rpm/SOURCES -- cp rteval.spec rpm/SPECS -- rpmbuild -ba --define "_topdir $(HERE)/rpm" rpm/SPECS/rteval.spec -- --rtevalsrpm: rteval-$(VERSION).tar.bz2 -- cp $^ rpm/SOURCES -- cp rteval.spec rpm/SPECS -- rpmbuild -bs --define "_topdir $(HERE)/rpm" rpm/SPECS/rteval.spec -- -- --xmlrpcrpm: rteval-xmlrpc-$(XMLRPCVER).tar.gz -- cp rteval-xmlrpc-$(XMLRPCVER).tar.gz rpm/SOURCES/ -- cp server/rteval-parser.spec rpm/SPECS/ -- rpmbuild -ba --define "_topdir $(HERE)/rpm" rpm/SPECS/rteval-parser.spec -- --xmlsrpm: rteval-xmlrpc-$(XMLRPCVER).tar.gz -- cp rteval-xmlrpc-$(XMLRPCVER).tar.gz rpm/SOURCES/ -- cp server/rteval-parser.spec rpm/SPECS/ -- rpmbuild -bs --define "_topdir $(HERE)/rpm" rpm/SPECS/rteval-parser.spec -- --loadrpm: -- rm -rf rpm-loads -- mkdir -p rpm-loads/{BUILD,RPMS,SRPMS,SOURCES,SPECS} -- cp rteval-loads.spec rpm-loads/SPECS -- cp $(LOADS) rpm-loads/SOURCES -- rpmbuild -ba --define "_topdir $(HERE)/rpm-loads" rpm-loads/SPECS/rteval-loads.spec -- --rpmlint: rpms -- @echo "===============" -- @echo "running rpmlint" -- rpmlint -v $(shell find ./rpm -type f -name "*.rpm") \ -- $(shell find ./rpm-loads -type f -name "*.rpm") \ -- $(shell find ./rpm/SPECS -type f -name "rteval*.spec") \ -- $(shell find ./rpm-loads/SPECS -type f -name "rteval*.spec" ) - - help: - @echo "" - @echo "rteval Makefile targets:" - @echo "" - @echo " runit: do a short testrun locally [default]" -- @echo " rpm: run rpmbuild for all rpms" -- @echo " rpmlint: run rpmlint against all rpms/srpms/specfiles" - @echo " tarfile: create the source tarball" - @echo " install: install rteval locally" - @echo " clean: cleanup generated files" -diff --git a/rteval.spec b/rteval.spec -deleted file mode 100644 -index b5842f0d8206..000000000000 ---- a/rteval.spec -+++ /dev/null -@@ -1,484 +0,0 @@ --%{!?python_sitelib: %define python_sitelib %(%{__python} -c "from distutils.sysconfig import get_python_lib; print get_python_lib()")} --%{!?python_ver: %define python_ver %(%{__python} -c "import sys ; print sys.version[:3]")} -- --Name: rteval --Version: 3.7 --Release: 1%{?dist} --Summary: Utility to evaluate system suitability for RT Linux -- --Group: Development/Tools --License: GPLv2 --URL: http://git.kernel.org/?p=linux/kernel/git/clrkwllms/rteval.git --Source0: rteval-%{version}.tar.bz2 --BuildRoot: %{_tmppath}/%{name}-%{version}-%{release}-root-%(%{__id_u} -n) -- --BuildRequires: python3-devel --Requires: platform-python --Requires: python3-ethtool python3-lxml --Requires: python3-dmidecode >= 3.10 --Requires: rt-tests >= 0.97 --Requires: rteval-loads >= 1.4 --Requires: rteval-common => %{version}-%{release} --Requires: sysstat --Requires: bzip2 --Requires: kernel-headers --Requires: sos --BuildArch: noarch --Obsoletes: rteval <= 1.7 --Requires: numactl -- --%description --The rteval script is a utility for measuring various aspects of --realtime behavior on a system under load. The script unpacks the --kernel source, and then goes into a loop, running hackbench and --compiling a kernel tree. During that loop the cyclictest program --is run to measure event response time. After the run time completes, --a statistical analysis of the event response times is done and printed --to the screen. -- -- --%package common --Summary: Common rteval files --BuildArch: noarch -- --%description common --Common files used by rteval, rteval-xmlrpc and rteval-parser -- --%prep --%setup -q -- --# version sanity check (make sure specfile and rteval.py match) --cp rteval/version.py rtevalversion.py --srcver=$(%{__python} -c "from rtevalversion import RTEVAL_VERSION; print RTEVAL_VERSION") --rm -rf rtevalversion.py --if [ $srcver != %{version} ]; then -- printf "\n***\n*** rteval spec file version do not match the rteval/rteval.py version\n***\n\n" -- exit -1 --fi -- --%build --%{__python} setup.py build -- --%install --%{__python} setup.py install --root=$RPM_BUILD_ROOT -- --%clean --rm -rf $RPM_BUILD_ROOT -- --%files common --%doc COPYING --%dir %{_datadir}/%{name} --%{python_sitelib}/rteval/rtevalclient.py* --%{python_sitelib}/rteval/rtevalConfig.py* --%{python_sitelib}/rteval/rtevalXMLRPC.py* --%{python_sitelib}/rteval/version.py* --%{python_sitelib}/rteval/Log.py* --%{python_sitelib}/rteval/misc.py* --%{python_sitelib}/rteval/systopology.py* -- --%files --%defattr(-,root,root,-) --%if "%{python_ver}" >= "2.5" --%{python_sitelib}/*.egg-info --%endif -- --%doc COPYING README doc/rteval.txt --%{_mandir}/man8/rteval.8.gz --%config(noreplace) %{_sysconfdir}/rteval.conf --%{_datadir}/%{name}/rteval_*.xsl --%{python_sitelib}/rteval/__init__.py* --%{python_sitelib}/rteval/rtevalMailer.py* --%{python_sitelib}/rteval/rtevalReport.py* --%{python_sitelib}/rteval/xmlout.py* --%{python_sitelib}/rteval/modules --%{python_sitelib}/rteval/sysinfo --/usr/bin/rteval -- --%changelog --* Thu Mar 16 2017 Clark Williams - 2.14-1 --- removed leftover import of systopology from sysinfo -- --* Wed Mar 15 2017 Clark Williams - 2.13-2 --- Updated specfile to correct version and bz [1382155] -- --* Tue Sep 20 2016 Clark Williams - 2.12-1 --- handle empty environment variables SUDO_USER and USER [1312057] -- --* Tue Aug 30 2016 Clark Williams - 2.11-1 --- make sure we return non-zero for early exit from tests -- --* Wed Aug 3 2016 Clark Williams - 2.10-1 --- bumped version for RHEL 7.3 release -- --* Mon May 9 2016 Clark Williams - 2.9.1 --- default cpulist for modules if only one specified [1333831] -- --* Tue Apr 26 2016 Clark Williams - 2.8.1 --- add the --version option to print the rteval version --- made the --cyclictest-breaktrace option work properly [1209986] -- --* Fri Apr 1 2016 Clark Williams - 2.7.1 --- treat SIGINT and SIGTERM as valid end-of-run events [1278757] --- added cpulist options to man page -- --* Thu Feb 11 2016 Clark Williams - 2.6.1 --- update to make --loads-cpulist and --measurement-cpulist work [1306437] -- --* Thu Dec 10 2015 Clark Williams - 2.5-1 --- stop using old numactl --cpubind argument -- --* Wed Dec 9 2015 Clark Williams - 2.4.2 --- added Require of package numactl -- --* Tue Nov 17 2015 Clark Williams - 2.4.1 --- rework hackbench load to not generate cross-node traffic [1282826] -- --* Wed Aug 12 2015 Clark Williams - 2.3-1 --- comment out HWLatDetect module from default config [1245699] -- --* Wed Jun 10 2015 Clark Williams - 2.2-1 --- add --loads-cpulist and --measurement-cpulist to allow cpu placement [1230401] -- --* Thu Apr 23 2015 Luis Claudio R. Goncalves - 2.1-8 --- load default configs when no config file is specified (Jiri kastner) [1212452] -- --* Wed Jan 14 2015 Clark Williams - 2.1-7 --- added requires of bzip2 to specfile [1151567] -- --* Thu Jan 8 2015 Clark Williams - 2.1-6 --- cleaned up product documentation [1173315] -- --* Mon Nov 10 2014 Luis Claudio R. Goncalves - 2.1-5 --- rebuild for RHEL-7.1 (1151567) -- --* Thu Mar 27 2014 Clark Williams - 2.1-4 --- cherry-picked old commit to deal with installdir problem -- --* Wed Mar 26 2014 Clark Williams - 2.1-3 --- added sysstat requires to specfile -- --* Tue Mar 12 2013 David Sommerseth - 2.1-2 --- Migrated from libxslt-python to python-lxml -- --* Fri Jan 18 2013 David Sommerseth - 2.1-1 --- Made some log lines clearer --- cyclictest: Added --cyclictest-breaktrace feature --- cyclictest: Removed --cyclictest-distance option --- cyclictest: Use a tempfile buffer for cyclictest's stdout data --- cyclictest: Report if breaktrace was triggered --- cyclictest: Make the unit test work again --- cyclictest: Only log and show statistic data when samples are collected --- Copyright updates -- --* Thu Jan 17 2013 David Sommerseth - 2.0.1-1 --- Fix up type casting in the core module code --- hwlatdetect: Add some more useful debug info --- Reworked the run logic for modules - allow them to flag they won't run --- Fixed a few log messages in load modules --- Add a 30 seconds sleep before unleashing the measurement threads -- --* Thu Jan 10 2013 David Sommerseth - 2.0-3 --- Separate out RTEVAL_VERSION into rteval.version, to avoid -- massive BuildRequirements -- --* Fri Dec 21 2012 David Sommerseth - 2.0-2 --- Split out common files into rteval-common -- --* Fri Dec 21 2012 David Sommerseth - 2.0-1 --- Updated to rteval v2.0 and reworked spec file to use setup.py directly -- --* Tue Oct 23 2012 Clark Williams - 1.36-1 --- deal with system not having dmidecode python module --- make sure to cast priority parameter to int --- from Raphaël Beamonte : -- - Rewrite of the get_kthreads method to make it cross-distribution -- - Adds getcmdpath method to use which to locate the used commands -- - Rewrite of the get_services method to make it cross-distribution -- --* Mon Apr 2 2012 Clark Williams - 1.35-1 --- fix thinko where SIGINT and SIGTERM handlers were commented out -- --* Thu Jan 12 2012 Clark Williams - 1.34-1 --- fix missing config merge in rteval.py to pass parameters -- down to cyclictest --- modify hackbench to use helper function to start process -- --* Sat May 14 2011 Clark Williams - 1.33-1 --- modify hackbench cutoff to be 0.75GB/core -- --* Mon Aug 23 2010 Clark Williams - 1.32-1 --- update docs --- refactor some RTEval methods to utility functions --- modify hackbench.py not to run under low memory conditions --- clean up XML generation to deal with new hackbench code --- clean up XSL code to deal with new XML 'run' attribute --- from David Sommerseth : -- - improve CPU socket counting logic -- - delay log directory creation until actually needed --- from Gowrishankar : -- - check if the core id really exists (multithreading fix) -- --* Mon Jul 26 2010 Clark Williams - 1.31-1 --- from David Sommerseth : -- - Updated hackbench implementation to avoid overusing resources -- - Don't show NUMA node information if it's missing in the summary.xml -- - Show CPU cores properly -- --* Wed Jul 21 2010 Clark Williams - 1.30-1 --- added code to hackbench to try to detect and ease memory pressure -- --* Fri Jul 16 2010 Clark Williams - 1.29-1 --- fixed incorrect type value in kcompile.py -- --* Fri Jul 16 2010 Clark Williams - 1.28-1 --- added logic to loads to adjust number of jobs based on ratio -- of memory per core -- --* Wed Jul 14 2010 Clark Williams - 1.27-1 --- modified hackbench to go back to using threads rather than -- processes for units of work --- added memory size, number of numa nodes and run duration to the -- parameter dictionary passed to all loads and cyclictest -- --* Tue Jul 13 2010 Clark Williams - 1.26-1 --- modified hackbench parameters to reduce memory consumption -- --* Mon Jul 12 2010 Clark Williams - 1.25-1 --- fixed cyclictest bug that caused everything to be uniprocessor --- updated source copyrights to 2010 -- --* Fri Jul 9 2010 Clark Williams - 1.24-1 --- modified hackbench arguments and added new parameters for -- hackbench in rteval.conf -- --* Thu Jul 8 2010 Clark Williams - 1.23-1 --- version bump to deal with out-of-sync cvs issue -- --* Thu Jul 8 2010 Clark Williams - 1.22-1 --- merged David Sommerseth changes to use -- hackbench from rt-tests packages rather than carry local copy --- converted all loads and cyclictest to pass __init__ parameters -- in a dictionary rather than as discrete parameters --- added logging for load output -- --* Tue Apr 13 2010 Clark Williams - 1.21-1 --- from Luis Claudio Goncalves : -- - remove unecessary wait() call in cyclictest.py -- - close /dev/null after using it -- - call subprocess.wait() when needed -- - remove delayloop code in hackbench.py --- from David Sommerseth : -- - add SIGINT handler -- - handle non-root user case -- - process DMI warnings before command line arguments -- - added --annotate feature to rteval -- - updates to xmlrpc code -- --* Tue Apr 6 2010 Clark Williams - 1.20-1 --- code fixes from Luis Claudio Goncalves --- from David Sommerseth : -- - xmlrpc server updates -- - cputopology.py for recording topology in xml -- - added NUMA node recording for run data -- - rpmlint fixes --- added start of rteval whitepaper in docs dir -- --* Tue Mar 16 2010 Clark Williams - 1.19-1 --- add ability for --summarize to read tarfiles --- from David Sommerseth -- - gather info about loaded kernel modules for XML file -- - added child tracking to hackbench to prevent zombies -- --* Tue Feb 16 2010 Clark Williams - 1.18-1 --- fix usage of python 2.6 features on RHEL5 (python 2.4) -- --* Tue Feb 16 2010 Clark Williams - 1.17-1 --- added logic to filter non-printables from service status output -- so that we have legal XML output --- added logic to hackbench.py to cleanup properly at the end -- of the test -- --* Thu Feb 11 2010 Clark Williams - 1.16-1 --- fix errors in show_remaining_time() introduced because -- time values are floats rather than ints -- --* Thu Feb 11 2010 Clark Williams - 1.15-1 --- added logic to use --numa and --smp options of new cyclictest --- added countdown report for time remaining in a run -- --* Tue Feb 9 2010 Clark Williams - 1.14-1 --- David Sommerseth : -- merged XMLReport() changes for hwcert suite -- --* Tue Dec 22 2009 Clark Williams - 1.13-1 --- added cyclictest default initializers --- added sanity checks to statistics reduction code --- updated release checklist to include origin push --- updated Makefile clean and help targets --- davids updates (mainly for v7 integration): -- - Add explicit sys.path directory to the python sitelib+ -- '/rteval' -- - Send program arguments via RtEval() constructor -- - Added more DMI data into the summary.xml report -- - Fixed issue with not including all devices in the -- OnBoardDeviceInfo tag -- --* Thu Dec 3 2009 David Sommerseth - 1.12-2 --- fixed Makefile and specfile to include and install the -- rteval/rteval_histogram_raw.py source file for gaining -- raw access to histogram data --- Removed xmlrpc package during merge against master_ipv4 branch -- --* Wed Nov 25 2009 Clark Williams - 1.12-1 --- fix incorrect reporting of measurement thread priorities -- --* Mon Nov 16 2009 Clark Williams - 1.11-5 --- ensure that no double-slashes ("//") appear in the symlink -- path for /usr/bin/rteval (problem with rpmdiff) -- --* Tue Nov 10 2009 Clark Williams - 1.11-4 --- changed symlink back to install and tracked by %%files -- --* Mon Nov 9 2009 Clark Williams - 1.11-3 --- changed symlink generation from %%post to %%posttrans -- --* Mon Nov 9 2009 Clark Williams - 1.11-2 --- fixed incorrect dependency for libxslt -- --* Fri Nov 6 2009 Clark Williams - 1.11-1 --- added base OS info to XML file and XSL report --- created new package rteval-loads for the load source code -- --* Wed Nov 4 2009 Clark Williams - 1.10-1 --- added config file section for cyclictest and two settable -- parameters, buckets and interval -- --* Thu Oct 29 2009 Clark Williams - 1.9-1 --- merged davids updates: -- -H option (raw histogram data) -- cleaned up xsl files -- fixed cpu sorting -- --* Mon Oct 26 2009 David Sommerseth - 1.8-3 --- Fixed rpmlint complaints -- --* Mon Oct 26 2009 David Sommerseth - 1.8-2 --- Added xmlrpc package, containing the XML-RPC mod_python modules -- --* Tue Oct 20 2009 Clark Williams - 1.8-1 --- split kcompile and hackbench into sub-packages --- reworked Makefile (and specfile) install/uninstall logic --- fixed sysreport incorrect plugin option --- catch failure when running on root-squashed NFS -- --* Tue Oct 13 2009 Clark Williams - 1.7-1 --- added kthread status to xml file --- merged davids changes for option processing and additions -- to xml summary -- --* Tue Oct 13 2009 Clark Williams - 1.6-1 --- changed stat calculation to loop less --- added methods to grab service and kthread status -- --* Mon Oct 12 2009 Clark Williams - 1.5-1 --- changed cyclictest to use less memory when doing statisics -- calculations --- updated debug output to use module name prefixes --- changed option processing to only process config file once -- --* Fri Oct 9 2009 Clark Williams - 1.4-1 --- changed cyclictest to use histogram rather than sample array --- calcuated statistics directly from histogram --- changed sample interval to 100us --- added -a (affinity) argument to force cpu affinity for -- measurement threads -- --* Thu Sep 24 2009 David Sommerseth - 1.3-3 --- Cleaned up the spec file and made rpmlint happy -- --* Wed Sep 23 2009 David Sommerseth - 1.3-2 --- Removed version number from /usr/share/rteval path -- --* Tue Sep 22 2009 Clark Williams - 1.3-1 --- changes from davids: -- * changed report code to sort by processor id -- * added report submission retry logic -- * added emailer class -- --* Fri Sep 18 2009 Clark Williams - 1.2-1 --- added config file handling for modifying load behavior and -- setting defaults --- added units in report per IBM request -- --* Wed Aug 26 2009 Clark Williams - 1.1-2 --- missed a version change in rteval/rteval.py -- --* Wed Aug 26 2009 Clark Williams - 1.1-1 --- modified cyclictest.py to start cyclictest threads with a -- 'distance' of zero, meaning they all have the same measurement -- interval -- --* Tue Aug 25 2009 Clark Williams - 1.0-1 --- merged davids XMLRPC fixes --- fixed --workdir option --- verion bump to 1.0 -- --* Thu Aug 13 2009 Clark Williams - 0.9-2 --- fixed problem with incorrect version in rteval.py -- --* Tue Aug 4 2009 Clark Williams - 0.9-1 --- merged dsommers XMLRPC and database changes --- Specify minimum python-dmidecode version, which got native XML support --- Added rteval_dmi.xsl --- Fixed permission issues in /usr/share/rteval-x.xx -- --* Wed Jul 22 2009 Clark Williams - 0.8-1 --- added code to capture clocksource info --- added code to copy dmesg info to report directory --- added code to display clocksource info in report --- added --summarize option to display summary of existing report --- added helpfile target to Makefile -- --* Thu Mar 26 2009 Clark Williams - 0.7-1 --- added require for python-schedutils to specfile --- added default for cyclictest output file --- added help parameter to option parser data --- renamed xml output file to summary.xml --- added routine to create tarfile of result files -- --* Wed Mar 18 2009 Clark Williams - 0.6-6 --- added code to handle binary data coming from DMI tables -- --* Wed Mar 18 2009 Clark Williams - 0.6-5 --- fixed logic for locating XSL template (williams) --- fixed another stupid typo in specfile (williams) -- --* Wed Mar 18 2009 Clark Williams - 0.6-4 --- fixed specfile to install rteval_text.xsl in /usr/share directory -- --* Wed Mar 18 2009 Clark Williams - 0.6-3 --- added Requires for libxslt-python (williams) --- fixed race condition in xmlout constructor/destructor (williams) -- --* Wed Mar 18 2009 Clark Williams - 0.6-2 --- added Requires for libxslt (williams) --- fixed stupid typo in rteval/rteval.py (williams) -- --* Wed Mar 18 2009 Clark Williams - 0.6-1 --- added xml output logic (williams, dsommers) --- added xlst template for report generator (dsommers) --- added dmi/smbios output to report (williams) --- added __del__ method to hackbench to cleanup after run (williams) --- modified to always keep run data (williams) -- --* Fri Feb 20 2009 Clark Williams - 0.5-1 --- fixed tab/space mix problem --- added report path line to report -- --* Fri Feb 20 2009 Clark Williams - 0.4-1 --- reworked report output --- handle keyboard interrupt better --- removed duration mismatch between rteval and cyclictest -- --* Mon Feb 2 2009 Clark Williams - 0.3-1 --- initial checkin --- -2.41.0 - diff --git a/rteval-Use-get-instead-of-setdefault-for-calculating.patch b/rteval-Use-get-instead-of-setdefault-for-calculating.patch deleted file mode 100644 index 195f6bf..0000000 --- a/rteval-Use-get-instead-of-setdefault-for-calculating.patch +++ /dev/null @@ -1,45 +0,0 @@ -From a5f3289e2ee6577cbbb78661bea58b821942ba56 Mon Sep 17 00:00:00 2001 -From: John Kacur -Date: Thu, 2 May 2024 13:09:01 -0400 -Subject: [PATCH 12/13] rteval: Use get instead of setdefault for calculating - range - -As Crystal Wood points out, there is no -need to setdefault when calculating the range, just use get -if there is no value. - -Signed-off-by: John Kacur ---- - rteval/modules/measurement/cyclictest.py | 2 +- - rteval/modules/measurement/timerlat.py | 2 +- - 2 files changed, 2 insertions(+), 2 deletions(-) - -diff --git a/rteval/modules/measurement/cyclictest.py b/rteval/modules/measurement/cyclictest.py -index 925065367eaf..3301e1b45e11 100644 ---- a/rteval/modules/measurement/cyclictest.py -+++ b/rteval/modules/measurement/cyclictest.py -@@ -115,7 +115,7 @@ class RunData: - low = i - break - high = keys[-1] -- while high and self.__samples.setdefault(high, 0) == 0: -+ while high and self.__samples.get(high, 0) == 0: - high -= 1 - self.__range = high - low - -diff --git a/rteval/modules/measurement/timerlat.py b/rteval/modules/measurement/timerlat.py -index 9fa931043e40..e8345fab1ad7 100644 ---- a/rteval/modules/measurement/timerlat.py -+++ b/rteval/modules/measurement/timerlat.py -@@ -106,7 +106,7 @@ class TLRunData: - low = i - break - high = keys[-1] -- while high and self.__samples.setdefault(high, 0) == 0: -+ while high and self.__samples.get(high, 0) == 0: - high -= 1 - self.__range = high - low - --- -2.44.0 - diff --git a/rteval-cyclictest.py-Fix-the-description-in-the-xml-.patch b/rteval-cyclictest.py-Fix-the-description-in-the-xml-.patch deleted file mode 100644 index 8969421..0000000 --- a/rteval-cyclictest.py-Fix-the-description-in-the-xml-.patch +++ /dev/null @@ -1,45 +0,0 @@ -From 0a431278cc681270b54efea4eebce9571c237624 Mon Sep 17 00:00:00 2001 -From: John Kacur -Date: Thu, 18 Apr 2024 12:39:33 -0400 -Subject: [PATCH 03/13] rteval: cyclictest.py: Fix the description in the xml - report - -If the __type is 'system' we should add the description to the xml -The output should be something like this - - - -Before this fix it was - - - -Signed-off-by: John Kacur ---- - rteval/modules/measurement/cyclictest.py | 4 ++-- - 1 file changed, 2 insertions(+), 2 deletions(-) - -diff --git a/rteval/modules/measurement/cyclictest.py b/rteval/modules/measurement/cyclictest.py -index 671426e13c4d..48b86a03cbfc 100644 ---- a/rteval/modules/measurement/cyclictest.py -+++ b/rteval/modules/measurement/cyclictest.py -@@ -25,7 +25,7 @@ class RunData: - self.__id = coreid - self.__type = datatype - self.__priority = int(priority) -- self.__description = '' -+ self.description = '' - # histogram of data - self.__samples = {} - self.__numsamples = 0 -@@ -130,7 +130,7 @@ class RunData: - def MakeReport(self): - rep_n = libxml2.newNode(self.__type) - if self.__type == 'system': -- rep_n.newProp('description', self.__description) -+ rep_n.newProp('description', self.description) - else: - rep_n.newProp('id', str(self.__id)) - rep_n.newProp('priority', str(self.__priority)) --- -2.44.0 - diff --git a/rteval-cyclictest.py-Fix-the-median-calculation.patch b/rteval-cyclictest.py-Fix-the-median-calculation.patch deleted file mode 100644 index ce0c8e3..0000000 --- a/rteval-cyclictest.py-Fix-the-median-calculation.patch +++ /dev/null @@ -1,60 +0,0 @@ -From 071bd1be404f1b5c71f8b2f2d9fa1a85bd485ce9 Mon Sep 17 00:00:00 2001 -From: John Kacur -Date: Thu, 25 Apr 2024 13:48:05 -0400 -Subject: [PATCH 05/13] rteval: cyclictest.py: Fix the median calculation - -Fix the calculation of the median in rteval for cyclictest - -Signed-off-by: John Kacur ---- - rteval/modules/measurement/cyclictest.py | 20 ++++++++++++++------ - 1 file changed, 14 insertions(+), 6 deletions(-) - -diff --git a/rteval/modules/measurement/cyclictest.py b/rteval/modules/measurement/cyclictest.py -index 1ba8b8b2323c..cef73abd1b4b 100644 ---- a/rteval/modules/measurement/cyclictest.py -+++ b/rteval/modules/measurement/cyclictest.py -@@ -80,26 +80,34 @@ class RunData: - return - - self._log(Log.INFO, f"reducing {self.__id}") -- total = 0 -+ total = 0 # total number of samples -+ total_us = 0 - keys = list(self.__samples.keys()) - keys.sort() - -- mid = self.__numsamples / 2 -+ # if numsamples is odd, then + 1 gives us the actual mid -+ # if numsamples is even, we avg mid and mid + 1, so we actually -+ # want to know mid + 1 since we will combine it with mid and -+ # the lastkey if the last key is at the end of a previous bucket -+ mid = int(self.__numsamples / 2) + 1 - - # mean, mode, and median - occurances = 0 - lastkey = -1 - for i in keys: -- if mid > total and mid <= (total + self.__samples[i]): -- if self.__numsamples & 1 and mid == total+1: -+ if mid > total and mid <= total + self.__samples[i]: -+ # Test if numsamples is even and if mid+1 is the next bucket -+ if self.__numsamples & 1 != 0 and mid == total+1: - self.__median = (lastkey + i) / 2 - else: - self.__median = i -- total += (i * self.__samples[i]) -+ lastkey = i -+ total += self.__samples[i] -+ total_us += (i * self.__samples[i]) - if self.__samples[i] > occurances: - occurances = self.__samples[i] - self.__mode = i -- self.__mean = float(total) / float(self.__numsamples) -+ self.__mean = float(total_us) / float(self.__numsamples) - - # range - for i in keys: --- -2.44.0 - diff --git a/rteval-cyclictest.py-Remove-unused-method-sample.patch b/rteval-cyclictest.py-Remove-unused-method-sample.patch deleted file mode 100644 index 0c77182..0000000 --- a/rteval-cyclictest.py-Remove-unused-method-sample.patch +++ /dev/null @@ -1,32 +0,0 @@ -From 6a185e50bb9b0903e034a2361456b5742e60c47b Mon Sep 17 00:00:00 2001 -From: John Kacur -Date: Fri, 19 Apr 2024 12:17:31 -0400 -Subject: [PATCH 04/13] rteval: cyclictest.py: Remove unused method sample - -Remove the unused method sample() in class RunData - -Signed-off-by: John Kacur ---- - rteval/modules/measurement/cyclictest.py | 6 ------ - 1 file changed, 6 deletions(-) - -diff --git a/rteval/modules/measurement/cyclictest.py b/rteval/modules/measurement/cyclictest.py -index 48b86a03cbfc..1ba8b8b2323c 100644 ---- a/rteval/modules/measurement/cyclictest.py -+++ b/rteval/modules/measurement/cyclictest.py -@@ -61,12 +61,6 @@ class RunData: - if value < self.__min: - self.__min = value - -- def sample(self, value): -- self.__samples[value] += self.__samples.setdefault(value, 0) + 1 -- self.update_max(value) -- self.update_min(value) -- self.__numsamples += 1 -- - def bucket(self, index, value): - self.__samples[index] = self.__samples.setdefault(index, 0) + value - if value: --- -2.44.0 - diff --git a/rteval-cyclictest.py-reduce-Fix-exception-with-missi.patch b/rteval-cyclictest.py-reduce-Fix-exception-with-missi.patch deleted file mode 100644 index d3fcd62..0000000 --- a/rteval-cyclictest.py-reduce-Fix-exception-with-missi.patch +++ /dev/null @@ -1,35 +0,0 @@ -From eb344bffd1525bbb8cd95fb4f4501e467cdc21ce Mon Sep 17 00:00:00 2001 -From: Crystal Wood -Date: Wed, 1 May 2024 15:09:22 -0500 -Subject: [PATCH 11/13] rteval: cyclictest.py: reduce: Fix exception with - missing samples - -If cyclictest omits a line of zeroes rather than printing it, -__samples[high] will not exist (rather than be zero). Handle this -in preparation for cyclictest doing this. - -Note that get() seems more suitable than setdefault(), but the latter -is used in order to match the code in timerlat.py. - -Signed-off-by: Crystal Wood -Signed-off-by: John Kacur ---- - rteval/modules/measurement/cyclictest.py | 2 +- - 1 file changed, 1 insertion(+), 1 deletion(-) - -diff --git a/rteval/modules/measurement/cyclictest.py b/rteval/modules/measurement/cyclictest.py -index cef73abd1b4b..925065367eaf 100644 ---- a/rteval/modules/measurement/cyclictest.py -+++ b/rteval/modules/measurement/cyclictest.py -@@ -115,7 +115,7 @@ class RunData: - low = i - break - high = keys[-1] -- while high and self.__samples[high] == 0: -+ while high and self.__samples.setdefault(high, 0) == 0: - high -= 1 - self.__range = high - low - --- -2.44.0 - diff --git a/rteval-timerlat.py-Add-statistics-and-generate-xml.patch b/rteval-timerlat.py-Add-statistics-and-generate-xml.patch deleted file mode 100644 index 7f1749f..0000000 --- a/rteval-timerlat.py-Add-statistics-and-generate-xml.patch +++ /dev/null @@ -1,358 +0,0 @@ -From 53248cbad7b0b353e31a18b40181f38da0a721aa Mon Sep 17 00:00:00 2001 -From: John Kacur -Date: Thu, 25 Apr 2024 13:59:27 -0400 -Subject: [PATCH 06/13] rteval: timerlat.py Add statistics and generate xml - -This adds calculating statitics such as samples (count), min, max, mean, -median, mode, range, absolute and standard deviations to xml - -Currently it combines the kernel thread, IRQs and userspace threads into -one statistic like it does with cyclictest, but in the future we can -separate this out too. - -To see this functioning from git, do this, - -su -c 'python rteval/modules/measurement/timerlat.py ' - -Signed-off-by: John Kacur ---- - rteval/modules/measurement/timerlat.py | 266 ++++++++++++++++++++++++- - 1 file changed, 259 insertions(+), 7 deletions(-) - -diff --git a/rteval/modules/measurement/timerlat.py b/rteval/modules/measurement/timerlat.py -index d4e78de8d2a2..9fa931043e40 100644 ---- a/rteval/modules/measurement/timerlat.py -+++ b/rteval/modules/measurement/timerlat.py -@@ -8,12 +8,170 @@ import subprocess - import signal - import time - import tempfile -+import math -+import sys - import libxml2 - from rteval.Log import Log - from rteval.modules import rtevalModulePrototype - from rteval.systopology import cpuinfo, SysTopology - from rteval.cpulist_utils import expand_cpulist, collapse_cpulist - -+class TLRunData: -+ ''' class to store instance data from a timerlat run ''' -+ def __init__(self, coreid, datatype, priority, logfnc): -+ self.__id = coreid -+ self.__type = datatype -+ self.__priority = int(priority) -+ self.description = '' -+ self._log = logfnc -+ self.duration = '' -+ # histogram data, irqs, kernel threads and user threads per core -+ self.irqs = {} -+ self.thrs = {} -+ self.usrs = {} -+ self.__samples = {} -+ self.__numsamples = 0 -+ self.min = 100000000 -+ self.max = 0 -+ self.__stddev = 0.0 -+ self.__mean = 0.0 -+ self.__mode = 0.0 -+ self.__median = 0.0 -+ self.__range = 0.0 -+ -+ def update_max(self, value): -+ """ highest bucket with a value """ -+ if value > self.max: -+ self.max = value -+ -+ def update_min(self, value): -+ """ lowest bucket with a value """ -+ if value < self.min: -+ self.min = value -+ -+ def bucket(self, index, val1, val2, val3): -+ """ Store results index=bucket number, val1=IRQ, val2=thr, val3=usr """ -+ values = val1 + val2 + val3 -+ self.__samples[index] = self.__samples.setdefault(index, 0) + values -+ self.irqs[index] = val1 -+ self.thrs[index] = val2 -+ self.usrs[index] = val3 -+ if values: -+ self.update_max(index) -+ self.update_min(index) -+ self.__numsamples += values -+ -+ def reduce(self): -+ """ Calculate statistics """ -+ # Check to see if we have any samples. If there are 1 or 0, return -+ if self.__numsamples <= 1: -+ self._log(Log.DEBUG, f"skipping {self.__id} ({self.__numsamples} sampples)") -+ self.__mad = 0 -+ self.__stddev = 0 -+ return -+ -+ self._log(Log.INFO, f"reducing {self.__id}") -+ total = 0 # total number of samples -+ total_us = 0 -+ keys = list(self.__samples.keys()) -+ keys.sort() -+ -+ # if numsamples is odd, then + 1 gives us the actual mid -+ # if numsamples is even, we avg mid and mid + 1, so we actually -+ # want to know mid + 1 since we will combine it with mid and -+ # the lastkey if the last key is at the end of a previous bucket -+ mid = int(self.__numsamples / 2) + 1 -+ -+ # mean, mode and median -+ occurances = 0 -+ lastkey = -1 -+ for i in keys: -+ if mid > total and mid <= total + self.__samples[i]: -+ # Test if numsamples is even and if mid+1 is the next bucket -+ if self.__numsamples & 1 != 0 and mid == total+1: -+ self.__median = (lastkey + i) / 2 -+ else: -+ self.__median = i -+ lastkey = i -+ total += self.__samples[i] -+ total_us += i * self.__samples[i] -+ if self.__samples[i] > occurances: -+ occurances = self.__samples[i] -+ self.__mode = i -+ self.__mean = float(total_us) / float(self.__numsamples) -+ -+ # range -+ for i in keys: -+ if self.__samples[i]: -+ low = i -+ break -+ high = keys[-1] -+ while high and self.__samples.setdefault(high, 0) == 0: -+ high -= 1 -+ self.__range = high - low -+ -+ # Mean Absolute Deviation and Standard Deviation -+ madsum = 0 -+ varsum = 0 -+ for i in keys: -+ madsum += float(abs(float(i) - self.__mean) * self.__samples[i]) -+ varsum += float(((float(i) - self.__mean) ** 2) * self.__samples[i]) -+ self.__mad = madsum / self.__numsamples -+ self.__stddev = math.sqrt(varsum / (self.__numsamples - 1)) -+ -+ -+ def MakeReport(self): -+ rep_n = libxml2.newNode(self.__type) -+ -+ if self.__type == 'system': -+ rep_n.newProp('description', self.description) -+ else: -+ rep_n.newProp('id', str(self.__id)) -+ rep_n.newProp('priority', str(self.__priority)) -+ -+ stat_n = rep_n.newChild(None, 'statistics', None) -+ stat_n.newTextChild(None, 'samples', str(self.__numsamples)) -+ -+ if self.__numsamples > 0: -+ n = stat_n.newTextChild(None, 'minimum', str(self.min)) -+ n.newProp('unit', 'us') -+ -+ n = stat_n.newTextChild(None, 'maximum', str(self.max)) -+ n.newProp('unit', 'us') -+ -+ n = stat_n.newTextChild(None, 'median', str(self.__median)) -+ n.newProp('unit', 'us') -+ -+ n = stat_n.newTextChild(None, 'mode', str(self.__mode)) -+ n.newProp('unit', 'us') -+ -+ n = stat_n.newTextChild(None, 'range', str(self.__range)) -+ n.newProp('unit', 'us') -+ -+ n = stat_n.newTextChild(None, 'mean', str(self.__mean)) -+ n.newProp('unit', 'us') -+ -+ n = stat_n.newTextChild(None, 'mean_absolute_deviation', str(self.__mad)) -+ n.newProp('unit', 'us') -+ -+ n = stat_n.newTextChild(None, 'standard_deviation', str(self.__stddev)) -+ n.newProp('unit', 'us') -+ -+ hist_n = rep_n.newChild(None, 'histogram', None) -+ hist_n.newProp('nbuckets', str(len(self.__samples))) -+ -+ keys = list(self.__samples.keys()) -+ keys.sort() -+ for k in keys: -+ if self.__samples[k] == 0: -+ # Don't report buckets without any samples -+ continue -+ b_n = hist_n.newChild(None, 'bucket', None) -+ b_n.newProp('index', str(k)) -+ b_n.newProp('value', str(self.__samples[k])) -+ -+ return rep_n -+ - class Timerlat(rtevalModulePrototype): - """ measurement modules for rteval """ - def __init__(self, config, logger=None): -@@ -23,6 +181,8 @@ class Timerlat(rtevalModulePrototype): - - self.__numanodes = int(self.__cfg.setdefault('numanodes', 0)) - self.__priority = int(self.__cfg.setdefault('priority', 95)) -+ default_buckets = ModuleParameters()["buckets"]["default"] -+ self.__buckets = int(self.__cfg.setdefault('buckets', default_buckets)) - - self.__cpulist = self.__cfg.setdefault('cpulist', "") - self.__cpus = [str(c) for c in expand_cpulist(self.__cpulist)] -@@ -31,6 +191,20 @@ class Timerlat(rtevalModulePrototype): - self.__timerlat_out = None - self.__timerlat_err = None - self.__started = False -+ -+ # Create a TLRunData object for each core we'll measure -+ info = cpuinfo() -+ self.__timerlatdata = {} -+ for core in self.__cpus: -+ self.__timerlatdata[core] = TLRunData(core, 'core', self.__priority, -+ logfnc=self._log) -+ self.__timerlatdata[core].description = info[core]['model name'] -+ -+ # Create a TLRunData object for the overall system -+ self.__timerlatdata['system'] = TLRunData('system', 'system', -+ self.__priority, -+ logfnc=self._log) -+ self.__timerlatdata['system'].description = (f"({self.__numcores} cores) ") + info['0']['model name'] - self._log(Log.DEBUG, f"system using {self.__numcores} cpu cores") - - -@@ -43,6 +217,7 @@ class Timerlat(rtevalModulePrototype): - def _WorkloadPrepare(self): - self.__cmd = ['rtla', 'timerlat', 'hist', '-P', f'f:{int(self.__priority)}', '-u'] - self.__cmd.append(f'-c{self.__cpulist}') -+ self.__cmd.append(f'-E{self.__buckets}') - self._log(Log.DEBUG, f'self.__cmd = {self.__cmd}') - self.__timerlat_out = tempfile.SpooledTemporaryFile(mode='w+b') - self.__timerlat_err = tempfile.SpooledTemporaryFile(mode='w+b') -@@ -77,16 +252,82 @@ class Timerlat(rtevalModulePrototype): - os.kill(self.__timerlat_process.pid, signal.SIGINT) - time.sleep(2) - -- self._setFinished() -- self.__started = False -- -- def MakeReport(self): -+ # Parse histogram output - self.__timerlat_out.seek(0) - for line in self.__timerlat_out: - line = bytes.decode(line) -- print(line) -+ -+ # Skip any blank lines -+ if not line: -+ continue -+ -+ if line.startswith('#'): -+ if line.startswith('# Duration:'): -+ duration = line.split()[2] -+ duration += line.split()[3] -+ self.__timerlatdata['system'].duration = duration -+ continue -+ elif line.startswith('Index'): -+ #print(line) -+ continue -+ elif line.startswith('over:'): -+ #print(line) -+ continue -+ elif line.startswith('count:'): -+ #print(line) -+ continue -+ elif line.startswith('min:'): -+ #print(line) -+ continue -+ elif line.startswith('avg:'): -+ #print(line) -+ continue -+ elif line.startswith('max:'): -+ #print(line) -+ continue -+ else: -+ pass -+ #print(line) -+ -+ vals = line.split() -+ if not vals: -+ # If we don't have any values, don't try parsing -+ continue -+ try: -+ # The index corresponds to the bucket number -+ index = int(vals[0]) -+ except: -+ self._log(Log.DEBUG, f'timerlat: unexpected output: {line}') -+ continue -+ -+ for i, core in enumerate(self.__cpus): -+ self.__timerlatdata[core].bucket(index, int(vals[i*3+1]), -+ int(vals[i*3+2]), -+ int(vals[i*3+3])) -+ self.__timerlatdata['system'].bucket(index, int(vals[i*3+1]), -+ int(vals[i*3+2]), -+ int(vals[i*3+3])) -+ # Generate statistics for each RunData object -+ for n in list(self.__timerlatdata.keys()): -+ self.__timerlatdata[n].reduce() -+ - self.__timerlat_out.close() - -+ self._setFinished() -+ self.__started = False -+ -+ def MakeReport(self): -+ rep_n = libxml2.newNode('timerlat') -+ rep_n.newProp('command_line', ' '.join(self.__cmd)) -+ -+ rep_n.addChild(self.__timerlatdata['system'].MakeReport()) -+ for thr in self.__cpus: -+ if str(thr) not in self.__timerlatdata: -+ continue -+ rep_n.addChild(self.__timerlatdata[str(thr)].MakeReport()) -+ -+ return rep_n -+ - - def ModuleInfo(): - """ Required measurement module information """ -@@ -97,7 +338,10 @@ def ModuleParameters(): - """ default parameters """ - return {"priority": {"descr": "Run rtla timerlat with this priority", - "default": 95, -- "metavar": "PRIO" } -+ "metavar": "PRIO" }, -+ "buckets": {"descr": "Number of buckets", -+ "default": 3500, -+ "metavar": "NUM"} - } - - def create(params, logger): -@@ -107,6 +351,10 @@ def create(params, logger): - if __name__ == '__main__': - from rteval.rtevalConfig import rtevalConfig - -+ if os.getuid() != 0: -+ print("Must be root to run timerlat!") -+ sys.exit(1) -+ - l = Log() - l.SetLogVerbosity(Log.INFO|Log.DEBUG|Log.ERR|Log.WARN) - -@@ -128,4 +376,8 @@ if __name__ == '__main__': - tl._WorkloadTask() - time.sleep(RUNTIME) - tl._WorkloadCleanup() -- tl.MakeReport() -+ rep_n = tl.MakeReport() -+ -+ xml = libxml2.newDoc('1.0') -+ xml.setRootElement(rep_n) -+ xml.saveFormatFileEnc('-', 'UTF-8', 1) --- -2.44.0 - diff --git a/rteval.spec b/rteval.spec index b4e0c6f..bf40aa3 100644 --- a/rteval.spec +++ b/rteval.spec @@ -1,6 +1,6 @@ Name: rteval -Version: 3.7 -Release: 6%{?dist} +Version: 3.8 +Release: 1%{?dist} Summary: Utility to evaluate system suitability for RT Linux Group: Development/Tools @@ -32,27 +32,6 @@ Requires: dmidecode BuildArch: noarch # Patches -Patch1: rteval-Change-the-default-kernel-for-kcompile.patch -Patch2: rteval-Remove-upstream-spec-file.patch -Patch3: rteval-Makefile-More-rpm-cleanups.patch -Patch4: rteval-Disable-use-of-python-dmidecode.patch -Patch5: rteval-Refactor-collapse_cpulist-in-systopology.patch -Patch6: rteval-Minor-improvements-to-CpuList-class.patch -Patch7: rteval-Convert-CpuList-class-to-a-module.patch -Patch8: rteval-Add-relative-cpulists-for-measurements.patch -Patch9: rteval-Implement-initial-dmidecode-support.patch -Patch10: rteval-Add-relative-cpulists-for-loads.patch -Patch11: rteval-Cyclictest.py-Make-standalone-file-work-again.patch -Patch12: rteval-Add-rtla-timerlat-as-a-measurement-module.patch -Patch13: rteval-cyclictest.py-Fix-the-description-in-the-xml-.patch -Patch14: rteval-cyclictest.py-Remove-unused-method-sample.patch -Patch15: rteval-cyclictest.py-Fix-the-median-calculation.patch -Patch16: rteval-timerlat.py-Add-statistics-and-generate-xml.patch -Patch17: rteval-Add-summary-reporting-for-timerlat.patch -Patch18: rteval-Generate-raw-histogram-data-for-an-existing-t.patch -Patch19: rteval-Make-output-work-with-new-dmiinfo-format.patch -Patch20: rteval-cyclictest.py-reduce-Fix-exception-with-missi.patch -Patch21: rteval-Use-get-instead-of-setdefault-for-calculating.patch %description The rteval script is a utility for measuring various aspects of @@ -65,27 +44,6 @@ to the screen. %prep %setup -q -%patch 1 -p1 -%patch 2 -p1 -%patch 3 -p1 -%patch 4 -p1 -%patch 5 -p1 -%patch 6 -p1 -%patch 7 -p1 -%patch 8 -p1 -%patch 9 -p1 -%patch 10 -p1 -%patch 11 -p1 -%patch 12 -p1 -%patch 13 -p1 -%patch 14 -p1 -%patch 15 -p1 -%patch 16 -p1 -%patch 17 -p1 -%patch 18 -p1 -%patch 19 -p1 -%patch 20 -p1 -%patch 21 -p1 %build %{__python3} setup.py build @@ -106,6 +64,10 @@ to the screen. %{_bindir}/rteval %changelog +* Tue May 07 2024 John Kacur - 3.8-1 +- Rebase rteval to upstream rteval-3.8 +Resolves: RHEL-30165 + * Mon May 06 2024 John Kacur - 3.7-6 - Add timerlat as a measurement module Resolves: RHEL-28058 diff --git a/sources b/sources index 6f6d7cb..7f3bd84 100644 --- a/sources +++ b/sources @@ -1,3 +1 @@ -SHA512 (rteval-3.7.tar.xz) = f6010fae457144e21b122746808411a6071b0c1db7d763bcf9e409fb570fc542c7463647ded77118534574f8d476f68576902af191a4edf8c296233a63704660 -SHA512 (0001-rteval-Change-the-default-kernel-for-kcompile-to-lin.patch) = e127ed1a3a5c4596597f53eeacdb080dfaa000df86f4bc275dbc58867a3f3bf33c1f476ee56e892aa46039fc0b18d1410b26f5338e309314da8eb7f702088ff9 -SHA512 (0002-rteval-Remove-upstream-spec-file.patch) = f240e551028f751ada510e7c6e7e06ffd4a6176c2a815a3d5719e8c33d00e0499bbc7285b99a69eeeabfd51fc23889691b740d40877c07a36d4afc4b9a6beda3 +SHA512 (rteval-3.8.tar.xz) = 25426b8f05d517b8b418975ff50bbc18ed406d3945eb0167af8e61687c872044a4fcfe39817f103bd4e2f4490c4613e64bfe28006c360698b039fbb10f1784c0