sos updates to RHEL 9.0 GA

Resolves: bz2011537 bz2012859 bz2020778 bz2023481

Signed-off-by: Pavel Moravec <pmoravec@redhat.com>

sos-bz2011537-estimate-only-option.patch

@@ -255,3 +255,111 @@ index 7feb31ee..1b5bc97d 100644
 -- 
 2.31.1
 
+From 589d47c93257b55bc796ef6ac25b88c974ee3d72 Mon Sep 17 00:00:00 2001
+From: Pavel Moravec <pmoravec@redhat.com>
+Date: Mon, 8 Nov 2021 16:38:24 +0100
+Subject: [PATCH] [report] Calculate sizes of dirs, symlinks and manifest in
+ estimate mode
+
+Enhance --estimate-mode to calculate sizes of also:
+- symlinks
+- directories themselves
+- manifest.json file
+
+Use os.lstat() method instead of os.stat() to properly calculate the
+sizes (and not destinations of symlinks, e.g.).
+
+Print five biggest plugins instead of three as sos logs and reports do
+stand as one "plugin" in the list, often.
+
+Resolves: #2752
+
+Signed-off-by: Pavel Moravec <pmoravec@redhat.com>
+---
+ sos/report/__init__.py | 56 +++++++++++++++++++++---------------------
+ 1 file changed, 28 insertions(+), 28 deletions(-)
+
+diff --git a/sos/report/__init__.py b/sos/report/__init__.py
+index 10952566..a4c92acc 100644
+--- a/sos/report/__init__.py
++++ b/sos/report/__init__.py
+@@ -1050,8 +1050,7 @@ class SoSReport(SoSComponent):
+             from pathlib import Path
+             tmpdir_path = Path(self.archive.get_tmp_dir())
+             self.estimated_plugsizes[plugin[1]] = sum(
+-                    [f.stat().st_size for f in tmpdir_path.glob('**/*')
+-                     if (os.path.isfile(f) and not os.path.islink(f))])
++                    [f.lstat().st_size for f in tmpdir_path.glob('**/*')])
+             # remove whole tmp_dir content - including "sos_commands" and
+             # similar dirs that will be re-created on demand by next plugin
+             # if needed; it is less error-prone approach than skipping
+@@ -1273,6 +1272,33 @@ class SoSReport(SoSComponent):
+                 short_name='manifest.json'
+             )
+ 
++        # print results in estimate mode (to include also just added manifest)
++        if self.opts.estimate_only:
++            from sos.utilities import get_human_readable
++            from pathlib import Path
++            # add sos_logs, sos_reports dirs, etc., basically everything
++            # that remained in self.tmpdir after plugins' contents removal
++            # that still will be moved to the sos report final directory path
++            tmpdir_path = Path(self.tmpdir)
++            self.estimated_plugsizes['sos_logs_reports'] = sum(
++                    [f.lstat().st_size for f in tmpdir_path.glob('**/*')])
++
++            _sum = get_human_readable(sum(self.estimated_plugsizes.values()))
++            self.ui_log.info("Estimated disk space requirement for whole "
++                             "uncompressed sos report directory: %s" % _sum)
++            bigplugins = sorted(self.estimated_plugsizes.items(),
++                                key=lambda x: x[1], reverse=True)[:5]
++            bp_out = ",  ".join("%s: %s" %
++                                (p, get_human_readable(v, precision=0))
++                                for p, v in bigplugins)
++            self.ui_log.info("Five biggest plugins:  %s" % bp_out)
++            self.ui_log.info("")
++            self.ui_log.info("Please note the estimation is relevant to the "
++                             "current options.")
++            self.ui_log.info("Be aware that the real disk space requirements "
++                             "might be different.")
++            self.ui_log.info("")
++
+         # package up and compress the results
+         if not self.opts.build:
+             old_umask = os.umask(0o077)
+@@ -1377,32 +1403,6 @@ class SoSReport(SoSComponent):
+             self.policy.display_results(archive, directory, checksum,
+                                         map_file=map_file)
+ 
+-        if self.opts.estimate_only:
+-            from sos.utilities import get_human_readable
+-            from pathlib import Path
+-            # add sos_logs, sos_reports dirs, etc., basically everything
+-            # that remained in self.tmpdir after plugins' contents removal
+-            # that still will be moved to the sos report final directory path
+-            tmpdir_path = Path(self.tmpdir)
+-            self.estimated_plugsizes['sos_logs_reports'] = sum(
+-                    [f.stat().st_size for f in tmpdir_path.glob('**/*')])
+-
+-            _sum = get_human_readable(sum(self.estimated_plugsizes.values()))
+-            self.ui_log.info("Estimated disk space requirement for whole "
+-                             "uncompressed sos report directory: %s" % _sum)
+-            bigplugins = sorted(self.estimated_plugsizes.items(),
+-                                key=lambda x: x[1], reverse=True)[:3]
+-            bp_out = ",  ".join("%s: %s" %
+-                                (p, get_human_readable(v, precision=0))
+-                                for p, v in bigplugins)
+-            self.ui_log.info("Three biggest plugins:  %s" % bp_out)
+-            self.ui_log.info("")
+-            self.ui_log.info("Please note the estimation is relevant to the "
+-                             "current options.")
+-            self.ui_log.info("Be aware that the real disk space requirements "
+-                             "might be different.")
+-            self.ui_log.info("")
+-
+         if self.opts.upload or self.opts.upload_url:
+             if not self.opts.build:
+                 try:
+-- 
+2.31.1
+

sos-bz2012859-plugin-timeout-unhandled-exception.patch

@@ -0,0 +1,31 @@
+From a93e118a9c88df52fd2c701d2276185f877d565c Mon Sep 17 00:00:00 2001
+From: Pavel Moravec <pmoravec@redhat.com>
+Date: Wed, 3 Nov 2021 16:07:15 +0100
+Subject: [PATCH] [report] shutdown threads for timeouted plugins
+
+Wait for shutting down threads of timeouted plugins, to prevent
+them in writing to moved auxiliary files like sos_logs/sos.log
+
+Resolves: #2722
+Closes: #2746
+
+Signed-off-by: Pavel Moravec <pmoravec@redhat.com>
+---
+ sos/report/__init__.py | 1 +
+ 1 file changed, 1 insertion(+)
+
+diff --git a/sos/report/__init__.py b/sos/report/__init__.py
+index 1b5bc97d..ef86b28d 100644
+--- a/sos/report/__init__.py
++++ b/sos/report/__init__.py
+@@ -1046,6 +1046,7 @@ class SoSReport(SoSComponent):
+                 self.ui_log.error("\n Plugin %s timed out\n" % plugin[1])
+                 self.running_plugs.remove(plugin[1])
+                 self.loaded_plugins[plugin[0]-1][1].set_timeout_hit()
++                pool.shutdown(wait=True)
+                 pool._threads.clear()
+         if self.opts.estimate_only:
+             from pathlib import Path
+-- 
+2.31.1
+

sos-bz2020778-filter-namespace-per-pattern.patch

@@ -0,0 +1,54 @@
+From 568eb2fbcf74ecad00d5c06989f55f8a6a9e3516 Mon Sep 17 00:00:00 2001
+From: Pavel Moravec <pmoravec@redhat.com>
+Date: Thu, 4 Nov 2021 23:14:21 +0100
+Subject: [PATCH] [report] fix filter_namespace per pattern
+
+Curently, -k networking.namespace_pattern=.. is broken as the R.E. test
+forgets to add the namespace in case of positive match.
+
+Also ensure both plugopts namespace_pattern and namespaces work
+together.
+
+Resolves: #2748
+
+Signed-off-by: Pavel Moravec <pmoravec@redhat.com>
+---
+ sos/report/plugins/__init__.py | 15 +++++++--------
+ 1 file changed, 7 insertions(+), 8 deletions(-)
+
+diff --git a/sos/report/plugins/__init__.py b/sos/report/plugins/__init__.py
+index 3e717993..a0d4e95d 100644
+--- a/sos/report/plugins/__init__.py
++++ b/sos/report/plugins/__init__.py
+@@ -2953,21 +2953,20 @@ class Plugin():
+                 )
+         for ns in ns_list:
+             # if ns_pattern defined, skip namespaces not matching the pattern
+-            if ns_pattern:
+-                if not bool(re.match(pattern, ns)):
+-                    continue
++            if ns_pattern and not bool(re.match(pattern, ns)):
++                continue
++            out_ns.append(ns)
+ 
+-            # if ns_max is defined at all, limit returned list to that number
++            # if ns_max is defined at all, break the loop when the limit is
++            # reached
+             # this allows the use of both '0' and `None` to mean unlimited
+-            elif ns_max:
+-                out_ns.append(ns)
++            if ns_max:
+                 if len(out_ns) == ns_max:
+                     self._log_warn("Limiting namespace iteration "
+                                    "to first %s namespaces found"
+                                    % ns_max)
+                     break
+-            else:
+-                out_ns.append(ns)
++
+         return out_ns
+ 
+ 
+-- 
+2.31.1
+

sos-bz2023481-plugin-timeouts-proper-handling.patch

@@ -0,0 +1,99 @@
+From 3fea9a564c4112d04f6324df0d8b212e78feb5b3 Mon Sep 17 00:00:00 2001
+From: Jake Hunsaker <jhunsake@redhat.com>
+Date: Wed, 3 Nov 2021 11:02:54 -0400
+Subject: [PATCH] [Plugin] Ensure specific plugin timeouts are only set for
+ that plugin
+
+It was discovered that setting a specific plugin timeout via the `-k
+$plugin.timeout` option could influence the timeout setting for other
+plugins that are not also having their timeout explicitly set. Fix this
+by moving the default plugin opts into `Plugin.__init__()` so that each
+plugin is ensured a private copy of these default plugin options.
+
+Additionally, add more timeout data to plugin manifest entries to allow
+for better tracking of this setting.
+
+Adds a test case for this scenario.
+
+Closes: #2744
+
+Signed-off-by: Jake Hunsaker <jhunsake@redhat.com>
+---
+ sos/report/__init__.py                   |  2 +-
+ sos/report/plugins/__init__.py           | 28 +++++++++++++------
+ tests/vendor_tests/redhat/rhbz2018033.py | 35 ++++++++++++++++++++++++
+ 3 files changed, 55 insertions(+), 10 deletions(-)
+ create mode 100644 tests/vendor_tests/redhat/rhbz2018033.py
+
+diff --git a/sos/report/__init__.py b/sos/report/__init__.py
+index ef86b28d..c95e6300 100644
+--- a/sos/report/__init__.py
++++ b/sos/report/__init__.py
+@@ -766,7 +766,7 @@ class SoSReport(SoSComponent):
+         if self.all_options:
+             self.ui_log.info(_("The following options are available for ALL "
+                                "plugins:"))
+-            for opt in self.all_options[0][0]._default_plug_opts:
++            for opt in self.all_options[0][0].get_default_plugin_opts():
+                 val = opt[3]
+                 if val == -1:
+                     val = TIMEOUT_DEFAULT
+diff --git a/sos/report/plugins/__init__.py b/sos/report/plugins/__init__.py
+index 49f1af27..3e717993 100644
+--- a/sos/report/plugins/__init__.py
++++ b/sos/report/plugins/__init__.py
+@@ -474,12 +474,6 @@ class Plugin(object):
+     # Default predicates
+     predicate = None
+     cmd_predicate = None
+-    _default_plug_opts = [
+-        ('timeout', 'Timeout in seconds for plugin to finish', 'fast', -1),
+-        ('cmd-timeout', 'Timeout in seconds for a command', 'fast', -1),
+-        ('postproc', 'Enable post-processing collected plugin data', 'fast',
+-         True)
+-    ]
+ 
+     def __init__(self, commons):
+ 
+@@ -506,7 +500,7 @@ class Plugin(object):
+             else logging.getLogger('sos')
+ 
+         # add the default plugin opts
+-        self.option_list.extend(self._default_plug_opts)
++        self.option_list.extend(self.get_default_plugin_opts())
+ 
+         # get the option list into a dictionary
+         for opt in self.option_list:
+@@ -591,6 +583,22 @@ class Plugin():
+         # Initialise the default --dry-run predicate
+         self.set_predicate(SoSPredicate(self))
+ 
++    def get_default_plugin_opts(self):
++        return {
++            'timeout': PluginOpt(
++                'timeout', default=-1, val_type=int,
++                desc='Timeout in seconds for plugin to finish all collections'
++            ),
++            'cmd-timeout': PluginOpt(
++                'cmd-timeout', default=-1, val_type=int,
++                desc='Timeout in seconds for individual commands to finish'
++            ),
++            'postproc': PluginOpt(
++                'postproc', default=True, val_type=bool,
++                desc='Enable post-processing of collected data'
++            )
++        }
++
+     def set_plugin_manifest(self, manifest):
+         """Pass in a manifest object to the plugin to write to
+ 
+@@ -547,7 +541,9 @@ class Plugin(object):
+         self.manifest.add_field('setup_start', '')
+         self.manifest.add_field('setup_end', '')
+         self.manifest.add_field('setup_time', '')
++        self.manifest.add_field('timeout', self.timeout)
+         self.manifest.add_field('timeout_hit', False)
++        self.manifest.add_field('command_timeout', self.cmdtimeout)
+         self.manifest.add_list('commands', [])
+         self.manifest.add_list('files', [])
+ 

sos.spec

@@ -5,7 +5,7 @@
 Summary: A set of tools to gather troubleshooting information from a system
 Name: sos
 Version: 4.2
-Release: 2%{?dist}
+Release: 3%{?dist}
 Group: Applications/System
 Source0: https://github.com/sosreport/sos/archive/%{version}/sos-%{version}.tar.gz
 Source1: sos-audit-%{auditversion}.tgz
@@ -34,6 +34,9 @@ Patch7: sos-bz2011536-iptables-based-on-ntf.patch
 Patch8: sos-bz2011507-foreman-puma-status.patch
 Patch9: sos-bz2012858-dryrun-uncaught-exception.patch
 Patch10: sos-bz2019697-openvswitch-offline-analysis.patch
+Patch11: sos-bz2012859-plugin-timeout-unhandled-exception.patch
+Patch12: sos-bz2023481-plugin-timeouts-proper-handling.patch
+Patch13: sos-bz2020778-filter-namespace-per-pattern.patch
 
 %description
 Sos is a set of tools that gathers information about system
@@ -54,6 +57,9 @@ support technicians and developers.
 %patch8 -p1
 %patch9 -p1
 %patch10 -p1
+%patch11 -p1
+%patch12 -p1
+%patch13 -p1
 
 %build
 %py3_build
@@ -121,6 +127,16 @@ of the system.  Currently storage and filesystem commands are audited.
 
 
 %changelog
+* Mon Nov 15 2021 Pavel Moravec <pmoravec@redhat.com> = 4.2-3
+- [report] Calculate sizes of dirs, symlinks and manifest in
+  Resolves: bz2011537
+- [report] shutdown threads for timeouted plugins
+  Resolves: bz2012859
+- [report] fix filter_namespace per pattern
+  Resolves: bz2020778
+- Ensure specific plugin timeouts are only set
+  Resolves: bz2023481
+
 * Wed Nov 03 2021 Pavel Moravec <pmoravec@redhat.com> = 4.2-2
 - [firewall_tables] call iptables -t <table> based on nft
   Resolves: bz2011536