From b6c480a921d0f869690aebfe7351c97f107bde1d Mon Sep 17 00:00:00 2001 From: Pavel Moravec Date: Mon, 15 Nov 2021 22:35:41 +0100 Subject: [PATCH] sos updates to RHEL 9.0 GA Resolves: bz2011537 bz2012859 bz2020778 bz2023481 Signed-off-by: Pavel Moravec --- sos-bz2011537-estimate-only-option.patch | 108 ++++++++++++++++++ ...9-plugin-timeout-unhandled-exception.patch | 31 +++++ ...2020778-filter-namespace-per-pattern.patch | 54 +++++++++ ...3481-plugin-timeouts-proper-handling.patch | 99 ++++++++++++++++ sos.spec | 18 ++- 5 files changed, 309 insertions(+), 1 deletion(-) create mode 100644 sos-bz2012859-plugin-timeout-unhandled-exception.patch create mode 100644 sos-bz2020778-filter-namespace-per-pattern.patch create mode 100644 sos-bz2023481-plugin-timeouts-proper-handling.patch diff --git a/sos-bz2011537-estimate-only-option.patch b/sos-bz2011537-estimate-only-option.patch index d0e053a..968bdde 100644 --- a/sos-bz2011537-estimate-only-option.patch +++ b/sos-bz2011537-estimate-only-option.patch @@ -255,3 +255,111 @@ index 7feb31ee..1b5bc97d 100644 -- 2.31.1 +From 589d47c93257b55bc796ef6ac25b88c974ee3d72 Mon Sep 17 00:00:00 2001 +From: Pavel Moravec +Date: Mon, 8 Nov 2021 16:38:24 +0100 +Subject: [PATCH] [report] Calculate sizes of dirs, symlinks and manifest in + estimate mode + +Enhance --estimate-mode to calculate sizes of also: +- symlinks +- directories themselves +- manifest.json file + +Use os.lstat() method instead of os.stat() to properly calculate the +sizes (and not destinations of symlinks, e.g.). + +Print five biggest plugins instead of three as sos logs and reports do +stand as one "plugin" in the list, often. + +Resolves: #2752 + +Signed-off-by: Pavel Moravec +--- + sos/report/__init__.py | 56 +++++++++++++++++++++--------------------- + 1 file changed, 28 insertions(+), 28 deletions(-) + +diff --git a/sos/report/__init__.py b/sos/report/__init__.py +index 10952566..a4c92acc 100644 +--- a/sos/report/__init__.py ++++ b/sos/report/__init__.py +@@ -1050,8 +1050,7 @@ class SoSReport(SoSComponent): + from pathlib import Path + tmpdir_path = Path(self.archive.get_tmp_dir()) + self.estimated_plugsizes[plugin[1]] = sum( +- [f.stat().st_size for f in tmpdir_path.glob('**/*') +- if (os.path.isfile(f) and not os.path.islink(f))]) ++ [f.lstat().st_size for f in tmpdir_path.glob('**/*')]) + # remove whole tmp_dir content - including "sos_commands" and + # similar dirs that will be re-created on demand by next plugin + # if needed; it is less error-prone approach than skipping +@@ -1273,6 +1272,33 @@ class SoSReport(SoSComponent): + short_name='manifest.json' + ) + ++ # print results in estimate mode (to include also just added manifest) ++ if self.opts.estimate_only: ++ from sos.utilities import get_human_readable ++ from pathlib import Path ++ # add sos_logs, sos_reports dirs, etc., basically everything ++ # that remained in self.tmpdir after plugins' contents removal ++ # that still will be moved to the sos report final directory path ++ tmpdir_path = Path(self.tmpdir) ++ self.estimated_plugsizes['sos_logs_reports'] = sum( ++ [f.lstat().st_size for f in tmpdir_path.glob('**/*')]) ++ ++ _sum = get_human_readable(sum(self.estimated_plugsizes.values())) ++ self.ui_log.info("Estimated disk space requirement for whole " ++ "uncompressed sos report directory: %s" % _sum) ++ bigplugins = sorted(self.estimated_plugsizes.items(), ++ key=lambda x: x[1], reverse=True)[:5] ++ bp_out = ", ".join("%s: %s" % ++ (p, get_human_readable(v, precision=0)) ++ for p, v in bigplugins) ++ self.ui_log.info("Five biggest plugins: %s" % bp_out) ++ self.ui_log.info("") ++ self.ui_log.info("Please note the estimation is relevant to the " ++ "current options.") ++ self.ui_log.info("Be aware that the real disk space requirements " ++ "might be different.") ++ self.ui_log.info("") ++ + # package up and compress the results + if not self.opts.build: + old_umask = os.umask(0o077) +@@ -1377,32 +1403,6 @@ class SoSReport(SoSComponent): + self.policy.display_results(archive, directory, checksum, + map_file=map_file) + +- if self.opts.estimate_only: +- from sos.utilities import get_human_readable +- from pathlib import Path +- # add sos_logs, sos_reports dirs, etc., basically everything +- # that remained in self.tmpdir after plugins' contents removal +- # that still will be moved to the sos report final directory path +- tmpdir_path = Path(self.tmpdir) +- self.estimated_plugsizes['sos_logs_reports'] = sum( +- [f.stat().st_size for f in tmpdir_path.glob('**/*')]) +- +- _sum = get_human_readable(sum(self.estimated_plugsizes.values())) +- self.ui_log.info("Estimated disk space requirement for whole " +- "uncompressed sos report directory: %s" % _sum) +- bigplugins = sorted(self.estimated_plugsizes.items(), +- key=lambda x: x[1], reverse=True)[:3] +- bp_out = ", ".join("%s: %s" % +- (p, get_human_readable(v, precision=0)) +- for p, v in bigplugins) +- self.ui_log.info("Three biggest plugins: %s" % bp_out) +- self.ui_log.info("") +- self.ui_log.info("Please note the estimation is relevant to the " +- "current options.") +- self.ui_log.info("Be aware that the real disk space requirements " +- "might be different.") +- self.ui_log.info("") +- + if self.opts.upload or self.opts.upload_url: + if not self.opts.build: + try: +-- +2.31.1 + diff --git a/sos-bz2012859-plugin-timeout-unhandled-exception.patch b/sos-bz2012859-plugin-timeout-unhandled-exception.patch new file mode 100644 index 0000000..e977fb5 --- /dev/null +++ b/sos-bz2012859-plugin-timeout-unhandled-exception.patch @@ -0,0 +1,31 @@ +From a93e118a9c88df52fd2c701d2276185f877d565c Mon Sep 17 00:00:00 2001 +From: Pavel Moravec +Date: Wed, 3 Nov 2021 16:07:15 +0100 +Subject: [PATCH] [report] shutdown threads for timeouted plugins + +Wait for shutting down threads of timeouted plugins, to prevent +them in writing to moved auxiliary files like sos_logs/sos.log + +Resolves: #2722 +Closes: #2746 + +Signed-off-by: Pavel Moravec +--- + sos/report/__init__.py | 1 + + 1 file changed, 1 insertion(+) + +diff --git a/sos/report/__init__.py b/sos/report/__init__.py +index 1b5bc97d..ef86b28d 100644 +--- a/sos/report/__init__.py ++++ b/sos/report/__init__.py +@@ -1046,6 +1046,7 @@ class SoSReport(SoSComponent): + self.ui_log.error("\n Plugin %s timed out\n" % plugin[1]) + self.running_plugs.remove(plugin[1]) + self.loaded_plugins[plugin[0]-1][1].set_timeout_hit() ++ pool.shutdown(wait=True) + pool._threads.clear() + if self.opts.estimate_only: + from pathlib import Path +-- +2.31.1 + diff --git a/sos-bz2020778-filter-namespace-per-pattern.patch b/sos-bz2020778-filter-namespace-per-pattern.patch new file mode 100644 index 0000000..5b0afdb --- /dev/null +++ b/sos-bz2020778-filter-namespace-per-pattern.patch @@ -0,0 +1,54 @@ +From 568eb2fbcf74ecad00d5c06989f55f8a6a9e3516 Mon Sep 17 00:00:00 2001 +From: Pavel Moravec +Date: Thu, 4 Nov 2021 23:14:21 +0100 +Subject: [PATCH] [report] fix filter_namespace per pattern + +Curently, -k networking.namespace_pattern=.. is broken as the R.E. test +forgets to add the namespace in case of positive match. + +Also ensure both plugopts namespace_pattern and namespaces work +together. + +Resolves: #2748 + +Signed-off-by: Pavel Moravec +--- + sos/report/plugins/__init__.py | 15 +++++++-------- + 1 file changed, 7 insertions(+), 8 deletions(-) + +diff --git a/sos/report/plugins/__init__.py b/sos/report/plugins/__init__.py +index 3e717993..a0d4e95d 100644 +--- a/sos/report/plugins/__init__.py ++++ b/sos/report/plugins/__init__.py +@@ -2953,21 +2953,20 @@ class Plugin(): + ) + for ns in ns_list: + # if ns_pattern defined, skip namespaces not matching the pattern +- if ns_pattern: +- if not bool(re.match(pattern, ns)): +- continue ++ if ns_pattern and not bool(re.match(pattern, ns)): ++ continue ++ out_ns.append(ns) + +- # if ns_max is defined at all, limit returned list to that number ++ # if ns_max is defined at all, break the loop when the limit is ++ # reached + # this allows the use of both '0' and `None` to mean unlimited +- elif ns_max: +- out_ns.append(ns) ++ if ns_max: + if len(out_ns) == ns_max: + self._log_warn("Limiting namespace iteration " + "to first %s namespaces found" + % ns_max) + break +- else: +- out_ns.append(ns) ++ + return out_ns + + +-- +2.31.1 + diff --git a/sos-bz2023481-plugin-timeouts-proper-handling.patch b/sos-bz2023481-plugin-timeouts-proper-handling.patch new file mode 100644 index 0000000..5a4dd34 --- /dev/null +++ b/sos-bz2023481-plugin-timeouts-proper-handling.patch @@ -0,0 +1,99 @@ +From 3fea9a564c4112d04f6324df0d8b212e78feb5b3 Mon Sep 17 00:00:00 2001 +From: Jake Hunsaker +Date: Wed, 3 Nov 2021 11:02:54 -0400 +Subject: [PATCH] [Plugin] Ensure specific plugin timeouts are only set for + that plugin + +It was discovered that setting a specific plugin timeout via the `-k +$plugin.timeout` option could influence the timeout setting for other +plugins that are not also having their timeout explicitly set. Fix this +by moving the default plugin opts into `Plugin.__init__()` so that each +plugin is ensured a private copy of these default plugin options. + +Additionally, add more timeout data to plugin manifest entries to allow +for better tracking of this setting. + +Adds a test case for this scenario. + +Closes: #2744 + +Signed-off-by: Jake Hunsaker +--- + sos/report/__init__.py | 2 +- + sos/report/plugins/__init__.py | 28 +++++++++++++------ + tests/vendor_tests/redhat/rhbz2018033.py | 35 ++++++++++++++++++++++++ + 3 files changed, 55 insertions(+), 10 deletions(-) + create mode 100644 tests/vendor_tests/redhat/rhbz2018033.py + +diff --git a/sos/report/__init__.py b/sos/report/__init__.py +index ef86b28d..c95e6300 100644 +--- a/sos/report/__init__.py ++++ b/sos/report/__init__.py +@@ -766,7 +766,7 @@ class SoSReport(SoSComponent): + if self.all_options: + self.ui_log.info(_("The following options are available for ALL " + "plugins:")) +- for opt in self.all_options[0][0]._default_plug_opts: ++ for opt in self.all_options[0][0].get_default_plugin_opts(): + val = opt[3] + if val == -1: + val = TIMEOUT_DEFAULT +diff --git a/sos/report/plugins/__init__.py b/sos/report/plugins/__init__.py +index 49f1af27..3e717993 100644 +--- a/sos/report/plugins/__init__.py ++++ b/sos/report/plugins/__init__.py +@@ -474,12 +474,6 @@ class Plugin(object): + # Default predicates + predicate = None + cmd_predicate = None +- _default_plug_opts = [ +- ('timeout', 'Timeout in seconds for plugin to finish', 'fast', -1), +- ('cmd-timeout', 'Timeout in seconds for a command', 'fast', -1), +- ('postproc', 'Enable post-processing collected plugin data', 'fast', +- True) +- ] + + def __init__(self, commons): + +@@ -506,7 +500,7 @@ class Plugin(object): + else logging.getLogger('sos') + + # add the default plugin opts +- self.option_list.extend(self._default_plug_opts) ++ self.option_list.extend(self.get_default_plugin_opts()) + + # get the option list into a dictionary + for opt in self.option_list: +@@ -591,6 +583,22 @@ class Plugin(): + # Initialise the default --dry-run predicate + self.set_predicate(SoSPredicate(self)) + ++ def get_default_plugin_opts(self): ++ return { ++ 'timeout': PluginOpt( ++ 'timeout', default=-1, val_type=int, ++ desc='Timeout in seconds for plugin to finish all collections' ++ ), ++ 'cmd-timeout': PluginOpt( ++ 'cmd-timeout', default=-1, val_type=int, ++ desc='Timeout in seconds for individual commands to finish' ++ ), ++ 'postproc': PluginOpt( ++ 'postproc', default=True, val_type=bool, ++ desc='Enable post-processing of collected data' ++ ) ++ } ++ + def set_plugin_manifest(self, manifest): + """Pass in a manifest object to the plugin to write to + +@@ -547,7 +541,9 @@ class Plugin(object): + self.manifest.add_field('setup_start', '') + self.manifest.add_field('setup_end', '') + self.manifest.add_field('setup_time', '') ++ self.manifest.add_field('timeout', self.timeout) + self.manifest.add_field('timeout_hit', False) ++ self.manifest.add_field('command_timeout', self.cmdtimeout) + self.manifest.add_list('commands', []) + self.manifest.add_list('files', []) + diff --git a/sos.spec b/sos.spec index 151a7d0..3eeef66 100644 --- a/sos.spec +++ b/sos.spec @@ -5,7 +5,7 @@ Summary: A set of tools to gather troubleshooting information from a system Name: sos Version: 4.2 -Release: 2%{?dist} +Release: 3%{?dist} Group: Applications/System Source0: https://github.com/sosreport/sos/archive/%{version}/sos-%{version}.tar.gz Source1: sos-audit-%{auditversion}.tgz @@ -34,6 +34,9 @@ Patch7: sos-bz2011536-iptables-based-on-ntf.patch Patch8: sos-bz2011507-foreman-puma-status.patch Patch9: sos-bz2012858-dryrun-uncaught-exception.patch Patch10: sos-bz2019697-openvswitch-offline-analysis.patch +Patch11: sos-bz2012859-plugin-timeout-unhandled-exception.patch +Patch12: sos-bz2023481-plugin-timeouts-proper-handling.patch +Patch13: sos-bz2020778-filter-namespace-per-pattern.patch %description Sos is a set of tools that gathers information about system @@ -54,6 +57,9 @@ support technicians and developers. %patch8 -p1 %patch9 -p1 %patch10 -p1 +%patch11 -p1 +%patch12 -p1 +%patch13 -p1 %build %py3_build @@ -121,6 +127,16 @@ of the system. Currently storage and filesystem commands are audited. %changelog +* Mon Nov 15 2021 Pavel Moravec = 4.2-3 +- [report] Calculate sizes of dirs, symlinks and manifest in + Resolves: bz2011537 +- [report] shutdown threads for timeouted plugins + Resolves: bz2012859 +- [report] fix filter_namespace per pattern + Resolves: bz2020778 +- Ensure specific plugin timeouts are only set + Resolves: bz2023481 + * Wed Nov 03 2021 Pavel Moravec = 4.2-2 - [firewall_tables] call iptables -t based on nft Resolves: bz2011536