From a90c6ef4208673939556eff8b577a01e11d7ea55 Mon Sep 17 00:00:00 2001 From: CentOS Sources Date: Mon, 8 Nov 2021 12:23:50 +0000 Subject: [PATCH] import rteval-3.2-3.el8 --- ...of-exclusive-load-module-and-make-st.patch | 79 ++++++++++++ ...otrun-work-correctly-in-load-modules.patch | 115 ++++++++++++++++++ ...rteval-Remove-mult-from-hackbench.py.patch | 49 ++++++++ SOURCES/rteval-Remove-self.__err_sleep.patch | 31 +++++ ...ble-running-on-a-system-with-low-mem.patch | 42 +++++++ SPECS/rteval.spec | 19 ++- 6 files changed, 334 insertions(+), 1 deletion(-) create mode 100644 SOURCES/rteval-Add-idea-of-exclusive-load-module-and-make-st.patch create mode 100644 SOURCES/rteval-Make-donotrun-work-correctly-in-load-modules.patch create mode 100644 SOURCES/rteval-Remove-mult-from-hackbench.py.patch create mode 100644 SOURCES/rteval-Remove-self.__err_sleep.patch create mode 100644 SOURCES/rteval-hackbench.py-Enable-running-on-a-system-with-low-mem.patch diff --git a/SOURCES/rteval-Add-idea-of-exclusive-load-module-and-make-st.patch b/SOURCES/rteval-Add-idea-of-exclusive-load-module-and-make-st.patch new file mode 100644 index 0000000..db22161 --- /dev/null +++ b/SOURCES/rteval-Add-idea-of-exclusive-load-module-and-make-st.patch @@ -0,0 +1,79 @@ +From 89622562925f432396a07cb8d7bb07da89151f2f Mon Sep 17 00:00:00 2001 +From: John Kacur +Date: Thu, 23 Sep 2021 14:12:40 -0400 +Subject: [PATCH 2/7] rteval: Add idea of exclusive load module and make + stress-ng one + +When running stress-ng as a load module in rteval, we don't want to run +kcompile or hackbench, so create the notion of an exclusive load module +and make stress-ng one. + +Signed-off-by: John Kacur +--- + rteval/modules/__init__.py | 21 +++++++++++++++++++++ + rteval/modules/loads/stressng.py | 2 ++ + 2 files changed, 23 insertions(+) + +diff --git a/rteval/modules/__init__.py b/rteval/modules/__init__.py +index 6ff82d10bf8c..d52dd597186a 100644 +--- a/rteval/modules/__init__.py ++++ b/rteval/modules/__init__.py +@@ -58,6 +58,7 @@ class rtevalModulePrototype(threading.Thread): + "stop": threading.Event(), + "finished": threading.Event()} + self._donotrun = False ++ self._exclusive = False + self.__timestamps = {} + self.__sleeptime = 2.0 + +@@ -75,6 +76,16 @@ class rtevalModulePrototype(threading.Thread): + return self.__ready + + ++ def is_exclusive(self): ++ """ Returns true if this workload should run alone """ ++ return self._exclusive ++ ++ ++ def set_donotrun(self): ++ """ set a module's donotrun field to True """ ++ self._donotrun = True ++ ++ + def _setReady(self, state=True): + """ Sets the ready flag for the module """ + self.__ready = state +@@ -459,7 +470,17 @@ class RtEvalModules: + raise rtevalRuntimeError("No %s modules configured" % self._module_type) + + self._logger.log(Log.INFO, "Preparing %s modules" % self._module_type) ++ exclusive = 0 ++ for (modname, mod) in self.__modules: ++ if mod.is_exclusive() and mod.WorkloadWillRun(): ++ exclusive += 1 + for (modname, mod) in self.__modules: ++ if exclusive >= 1: ++ if exclusive != 1: ++ msg = f"More than one exclusive load: {exclusive}" ++ raise RuntimeError(msg) ++ if not mod.is_exclusive() and mod.WorkloadWillRun(): ++ mod.set_donotrun() + mod.start() + if mod.WorkloadWillRun(): + self._logger.log(Log.DEBUG, "\t - Started %s preparations" % modname) +diff --git a/rteval/modules/loads/stressng.py b/rteval/modules/loads/stressng.py +index 926de38e3116..d084814142fb 100644 +--- a/rteval/modules/loads/stressng.py ++++ b/rteval/modules/loads/stressng.py +@@ -27,6 +27,8 @@ class Stressng(CommandLineLoad): + self._donotrun = False + else: + self._donotrun = True ++ """ When this module runs, other load modules should not """ ++ self._exclusive = True + + def _WorkloadSetup(self): + " Since there is nothing to build, we don't need to do anything here " +-- +2.31.1 + diff --git a/SOURCES/rteval-Make-donotrun-work-correctly-in-load-modules.patch b/SOURCES/rteval-Make-donotrun-work-correctly-in-load-modules.patch new file mode 100644 index 0000000..24fa794 --- /dev/null +++ b/SOURCES/rteval-Make-donotrun-work-correctly-in-load-modules.patch @@ -0,0 +1,115 @@ +From e94a774b8c14e01622b170517eedb9e90232c42c Mon Sep 17 00:00:00 2001 +From: John Kacur +Date: Thu, 23 Sep 2021 11:29:24 -0400 +Subject: [PATCH 1/7] rteval: Make donotrun work correctly in load modules + +hackbench and kcompile don't correctly adhere to the donotrun variable +which is inherited from class rtevalModulePrototype. If the variable is set +after the modules are loaded, although the modules do not run, some +thread accounting still occurs which causes problems on shutdown of +rteval. + +Fix in the methods in hackbench and kcompile where this is relevant, by +checking the variable before the method runs. + +While we are at it fix a few other things +- change len(threading.enumerate()) to threading.active_count() +- Fix a spelling typo + +This patch is in preparation for a patch to allow stress-ng to run as a +load module without running other modules. + +Signed-off-by: John Kacur +--- + rteval/__init__.py | 4 ++-- + rteval/modules/loads/__init__.py | 2 +- + rteval/modules/loads/hackbench.py | 3 +++ + rteval/modules/loads/kcompile.py | 9 +++++++++ + 4 files changed, 15 insertions(+), 3 deletions(-) + +diff --git a/rteval/__init__.py b/rteval/__init__.py +index 5faed23bc927..22af8e85d5aa 100644 +--- a/rteval/__init__.py ++++ b/rteval/__init__.py +@@ -208,7 +208,7 @@ class RtEval(rtevalReport): + report_interval = int(self.__rtevcfg.report_interval) + if with_loads: + self._loadmods.Unleash() +- nthreads = len(threading.enumerate()) ++ nthreads = threading.active_count() + else: + nthreads = None + self.__logger.log(Log.INFO, "Waiting 30 seconds to let load modules settle down") +@@ -233,7 +233,7 @@ class RtEval(rtevalReport): + "Measurement threads did not use the full time slot. Doing a controlled stop.") + + if with_loads: +- if len(threading.enumerate()) < nthreads: ++ if threading.active_count() < nthreads: + raise RuntimeError("load thread died!") + + if not load_avg_checked: +diff --git a/rteval/modules/loads/__init__.py b/rteval/modules/loads/__init__.py +index 9a942f454536..2c2105efa964 100644 +--- a/rteval/modules/loads/__init__.py ++++ b/rteval/modules/loads/__init__.py +@@ -106,7 +106,7 @@ class LoadModules(RtEvalModules): + "Loads and imports all the configured modules" + + for m in modcfg: +- # hope to eventually have different kinds but module is only on ++ # hope to eventually have different kinds but module is only one + # for now (jcw) + if m[1].lower() == 'module': + self._LoadModule(m[0]) +diff --git a/rteval/modules/loads/hackbench.py b/rteval/modules/loads/hackbench.py +index 2fb90c1946a5..21a803b8a3cd 100644 +--- a/rteval/modules/loads/hackbench.py ++++ b/rteval/modules/loads/hackbench.py +@@ -42,6 +42,9 @@ class Hackbench(CommandLineLoad): + CommandLineLoad.__init__(self, "hackbench", config, logger) + + def _WorkloadSetup(self): ++ if self._donotrun: ++ return ++ + 'calculate arguments based on input parameters' + (mem, units) = self.memsize + if units == 'KB': +diff --git a/rteval/modules/loads/kcompile.py b/rteval/modules/loads/kcompile.py +index 8d08a3d44302..ec85b75f38b5 100644 +--- a/rteval/modules/loads/kcompile.py ++++ b/rteval/modules/loads/kcompile.py +@@ -158,6 +158,9 @@ class Kcompile(CommandLineLoad): + "error removing builddir (%s) (ret=%d)" % (self.builddir, ret)) + + def _WorkloadSetup(self): ++ if self._donotrun: ++ return ++ + # find our source tarball + if 'tarball' in self._cfg: + tarfile = os.path.join(self.srcdir, self._cfg.tarfile) +@@ -219,6 +222,9 @@ class Kcompile(CommandLineLoad): + + + def _WorkloadBuild(self): ++ if self._donotrun: ++ return ++ + null = os.open("/dev/null", os.O_RDWR) + if self._logging: + out = self.open_logfile("kcompile-build.stdout") +@@ -293,6 +299,9 @@ class Kcompile(CommandLineLoad): + + + def _WorkloadCleanup(self): ++ if self._donotrun: ++ return ++ + self._log(Log.DEBUG, "out of stopevent loop") + for n in self.buildjobs: + if self.buildjobs[n].jobid.poll() is None: +-- +2.31.1 + diff --git a/SOURCES/rteval-Remove-mult-from-hackbench.py.patch b/SOURCES/rteval-Remove-mult-from-hackbench.py.patch new file mode 100644 index 0000000..abf65c8 --- /dev/null +++ b/SOURCES/rteval-Remove-mult-from-hackbench.py.patch @@ -0,0 +1,49 @@ +From 50e885286b535c4014e769791c25d172e7ee1a8d Mon Sep 17 00:00:00 2001 +From: John Kacur +Date: Tue, 14 Sep 2021 14:48:01 -0400 +Subject: [PATCH 02/11] rteval: Remove mult from hackbench.py + +The method for calculating the number of jobs to run for hackbench was +changed with the following commit. +commit 67629a1b69ffe72af6bfb3a0d4362ac1920005a7 + +However, the mult variable which was used to calculate the number of +jobs using the older method was not removed. + +Remove it now and simplify the logic. + +Signed-off-by: John Kacur +--- + rteval/modules/loads/hackbench.py | 7 ++----- + 1 file changed, 2 insertions(+), 5 deletions(-) + +diff --git a/rteval/modules/loads/hackbench.py b/rteval/modules/loads/hackbench.py +index ab028c495d8b..8119d50f626a 100644 +--- a/rteval/modules/loads/hackbench.py ++++ b/rteval/modules/loads/hackbench.py +@@ -41,7 +41,6 @@ class Hackbench(CommandLineLoad): + def __init__(self, config, logger): + CommandLineLoad.__init__(self, "hackbench", config, logger) + +- + def _WorkloadSetup(self): + 'calculate arguments based on input parameters' + (mem, units) = self.memsize +@@ -51,12 +50,10 @@ class Hackbench(CommandLineLoad): + mem = mem / 1024.0 + elif units == 'TB': + mem = mem * 1024 ++ + ratio = float(mem) / float(self.num_cpus) +- if ratio >= 0.75: +- mult = float(self._cfg.setdefault('jobspercore', 2)) +- else: ++ if ratio < 0.75: + self._log(Log.WARN, "Low memory system (%f GB/core)!" % ratio) +- mult = 0 + + sysTop = SysTopology() + # get the number of nodes +-- +2.31.1 + diff --git a/SOURCES/rteval-Remove-self.__err_sleep.patch b/SOURCES/rteval-Remove-self.__err_sleep.patch new file mode 100644 index 0000000..f92063c --- /dev/null +++ b/SOURCES/rteval-Remove-self.__err_sleep.patch @@ -0,0 +1,31 @@ +From 0d226e7032399e94f8bdeac84c55333209f0a558 Mon Sep 17 00:00:00 2001 +From: John Kacur +Date: Tue, 14 Sep 2021 14:56:42 -0400 +Subject: [PATCH 03/11] rteval: Remove self.__err_sleep + +commit 5ed68ae77ec05786aab99fbed35d0347a5d25997 +changed the code to exit gracefully if hackbench failed from memory +errors. No attempt is made anymore to wait to see if memory pressures +ease. The variable __err_sleep however was not removed at the time. +Remove it now, as it is now longer used. + +Signed-off-by: John Kacur +--- + rteval/modules/loads/hackbench.py | 1 - + 1 file changed, 1 deletion(-) + +diff --git a/rteval/modules/loads/hackbench.py b/rteval/modules/loads/hackbench.py +index 8119d50f626a..2fb90c1946a5 100644 +--- a/rteval/modules/loads/hackbench.py ++++ b/rteval/modules/loads/hackbench.py +@@ -97,7 +97,6 @@ class Hackbench(CommandLineLoad): + '-l', str(self._cfg.setdefault('loops', '1000')), + '-s', str(self._cfg.setdefault('datasize', '1000')) + ] +- self.__err_sleep = 5.0 + + def _WorkloadBuild(self): + # Nothing to build, so we're basically ready +-- +2.31.1 + diff --git a/SOURCES/rteval-hackbench.py-Enable-running-on-a-system-with-low-mem.patch b/SOURCES/rteval-hackbench.py-Enable-running-on-a-system-with-low-mem.patch new file mode 100644 index 0000000..42cd393 --- /dev/null +++ b/SOURCES/rteval-hackbench.py-Enable-running-on-a-system-with-low-mem.patch @@ -0,0 +1,42 @@ +From cb8263770e4f5834a43db6be8ffb55ffd7f876c9 Mon Sep 17 00:00:00 2001 +From: Punit Agrawal +Date: Wed, 1 Sep 2021 17:08:15 +0900 +Subject: [PATCH 01/11] rteval: hackbench.py: Enable running on a system with + low memory + +The hackbench workload refues to run on RockPro64, a hexacore 64bit +Arm board with 4GB memory, complaining about insufficient memory +per-core. + +On further investigation, it turns out that workload is using an +arbitrary limit of 0.75 GB/core but will quite happily run on much +lower lower memory systems. + +Instead of preventing execution, convert the info message to a warning +when the memory is lower than expected but continue execution. This +should enable the workload to be used on a wider range of systems. + +Signed-off-by: Punit Agrawal +Signed-off-by: John Kacur +--- + rteval/modules/loads/hackbench.py | 3 +-- + 1 file changed, 1 insertion(+), 2 deletions(-) + +diff --git a/rteval/modules/loads/hackbench.py b/rteval/modules/loads/hackbench.py +index 3b692070e9d9..ab028c495d8b 100644 +--- a/rteval/modules/loads/hackbench.py ++++ b/rteval/modules/loads/hackbench.py +@@ -55,9 +55,8 @@ class Hackbench(CommandLineLoad): + if ratio >= 0.75: + mult = float(self._cfg.setdefault('jobspercore', 2)) + else: +- self._log(Log.INFO, "Low memory system (%f GB/core)! Not running" % ratio) ++ self._log(Log.WARN, "Low memory system (%f GB/core)!" % ratio) + mult = 0 +- self._donotrun = True + + sysTop = SysTopology() + # get the number of nodes +-- +2.31.1 + diff --git a/SPECS/rteval.spec b/SPECS/rteval.spec index e0f0d07..e25c281 100644 --- a/SPECS/rteval.spec +++ b/SPECS/rteval.spec @@ -1,6 +1,6 @@ Name: rteval Version: 3.2 -Release: 2%{?dist} +Release: 3%{?dist} Summary: Utility to evaluate system suitability for RT Linux Group: Development/Tools @@ -32,6 +32,11 @@ Obsoletes: rteval-common <= 3.1 #Patches Patch1: rteval-Restrict-measurement-threads-to-the-cpus-in-cpumask.patch +Patch2: rteval-hackbench.py-Enable-running-on-a-system-with-low-mem.patch +Patch3: rteval-Remove-mult-from-hackbench.py.patch +Patch4: rteval-Remove-self.__err_sleep.patch +Patch5: rteval-Add-idea-of-exclusive-load-module-and-make-st.patch +Patch6: rteval-Make-donotrun-work-correctly-in-load-modules.patch %description The rteval script is a utility for measuring various aspects of @@ -45,6 +50,11 @@ to the screen. %prep %setup -q %patch1 -p1 +%patch2 -p1 +%patch3 -p1 +%patch4 -p1 +%patch5 -p1 +%patch6 -p1 %build %{__python3} setup.py build @@ -82,6 +92,13 @@ rm -rf $RPM_BUILD_ROOT %{python3_sitelib}/rteval/__pycache__/* %changelog +* Thu Nov 04 2021 John Kacur - 3.2-3 +- allow hackbench to run with warning on low mem +- clean-ups to hackbench.py +- make donotrun work correctly in load modules +- Add the idea of an exclusive load module and make stress-ng one +Resolves: rhbz#1872776 + * Thu Jul 15 2021 John Kacur - 3.2-2 - Restrict measurement threads to cpus in cpumask Resolves: rhbz#1942261