sos updates to RHEL9.0

Resolves: bz2011537 bz2024893 bz2037350

Signed-off-by: Pavel Moravec <pmoravec@redhat.com>

sos-bz2011537-estimate-only-option.patch

@@ -363,3 +363,57 @@ index 10952566..a4c92acc 100644
 -- 
 2.31.1
 
+From c6a5bbb8d75aadd5c7f76d3f469929aba2cf8060 Mon Sep 17 00:00:00 2001
+From: Pavel Moravec <pmoravec@redhat.com>
+Date: Wed, 5 Jan 2022 10:33:58 +0100
+Subject: [PATCH] [report] Provide better warning about estimate-mode
+
+As --estimate-only calculates disk usage based on `stat` data that
+differs from outputs of other commands like `du`, enhance the warning
+about reliability of the calculated estimation.
+
+Also add a rule-of-thumb recommendation of real disk space requirements.
+
+Resolves: #2815
+
+Signed-off-by: Pavel Moravec <pmoravec@redhat.com>
+---
+ man/en/sos-report.1    | 10 +++++++---
+ sos/report/__init__.py |  3 ++-
+ 2 files changed, 9 insertions(+), 4 deletions(-)
+
+diff --git a/man/en/sos-report.1 b/man/en/sos-report.1
+index 464a77e54..e34773986 100644
+--- a/man/en/sos-report.1
++++ b/man/en/sos-report.1
+@@ -343,9 +343,13 @@ is available at the end.
+ 
+ Plugins will be collected sequentially, size of collected files and commands outputs
+ will be calculated and the plugin files will be immediatelly deleted prior execution
+-of the next plugin. This still can consume whole free disk space, though. Please note,
+-size estimations may not be accurate for highly utilized systems due to changes between
+-an estimate and a real execution.
++of the next plugin. This still can consume whole free disk space, though.
++
++Please note, size estimations may not be accurate for highly utilized systems due to
++changes between an estimate and a real execution. Also some difference between
++estimation (using `stat` command) and other commands used (i.e. `du`).
++
++A rule of thumb is to reserve at least double the estimation.
+ .TP
+ .B \--upload
+ If specified, attempt to upload the resulting archive to a vendor defined location.
+diff --git a/sos/report/__init__.py b/sos/report/__init__.py
+index ef61fb344..e0617b45e 100644
+--- a/sos/report/__init__.py
++++ b/sos/report/__init__.py
+@@ -1330,7 +1330,8 @@ def final_work(self):
+             self.ui_log.info("Please note the estimation is relevant to the "
+                              "current options.")
+             self.ui_log.info("Be aware that the real disk space requirements "
+-                             "might be different.")
++                             "might be different. A rule of thumb is to "
++                             "reserve at least double the estimation.")
+             self.ui_log.info("")
+ 
+         # package up and compress the results

sos-bz2024893-cleaner-hostnames-improvements.patch

@@ -1595,3 +1595,205 @@ index 5cd8e9857..33b0e6c80 100644
          ob_domain = self._new_obfuscated_domain(dname)
          ob_domain = '.'.join([ob_domain, top_domain])
          self.dataset['.'.join(domain)] = ob_domain
+From f5e1298162a9393ea2d9f5c4df40dfece50f5f88 Mon Sep 17 00:00:00 2001
+From: Jake Hunsaker <jhunsake@redhat.com>
+Date: Thu, 6 Jan 2022 13:15:15 -0500
+Subject: [PATCH 1/3] [hostname] Fix loading and detection of long base domains
+
+Our domain matching has up to now assumed that users would be providing
+'base' domains such as 'example.com' whereby something like
+'foo.bar.example.com' is a subdomain (or host) within that base domain.
+
+However, the use case exists to provide 'foo.bar.example.com' as the
+base domain, without wanting to obfuscate 'example.com' directly.
+
+This commit fixes our handling of both loading these longer domains and
+doing the 'domain is part of a domain we want to obfuscate' check.
+
+Signed-off-by: Jake Hunsaker <jhunsake@redhat.com>
+---
+ sos/cleaner/mappings/hostname_map.py | 9 ++++++++-
+ 1 file changed, 8 insertions(+), 1 deletion(-)
+
+diff --git a/sos/cleaner/mappings/hostname_map.py b/sos/cleaner/mappings/hostname_map.py
+index 33b0e6c8..7a7cf6b8 100644
+--- a/sos/cleaner/mappings/hostname_map.py
++++ b/sos/cleaner/mappings/hostname_map.py
+@@ -50,10 +50,14 @@ class SoSHostnameMap(SoSMap):
+         in this parser, we need to re-inject entries from the map_file into
+         these dicts and not just the underlying 'dataset' dict
+         """
+-        for domain in self.dataset:
++        for domain, ob_pair in self.dataset.items():
+             if len(domain.split('.')) == 1:
+                 self.hosts[domain.split('.')[0]] = self.dataset[domain]
+             else:
++                if ob_pair.startswith('obfuscateddomain'):
++                    # directly exact domain matches
++                    self._domains[domain] = ob_pair.split('.')[0]
++                    continue
+                 # strip the host name and trailing top-level domain so that
+                 # we in inject the domain properly for later string matching
+ 
+@@ -102,9 +106,12 @@ class SoSHostnameMap(SoSMap):
+         and should be obfuscated
+         """
+         host = domain.split('.')
++        no_tld = '.'.join(domain.split('.')[0:-1])
+         if len(host) == 1:
+             # don't block on host's shortname
+             return host[0] in self.hosts.keys()
++        elif any([no_tld.endswith(_d) for _d in self._domains]):
++            return True
+         else:
+             domain = host[0:-1]
+             for known_domain in self._domains:
+-- 
+2.31.1
+
+
+From e241cf33a14ecd4e848a5fd857c5d3d7d07fbd71 Mon Sep 17 00:00:00 2001
+From: Jake Hunsaker <jhunsake@redhat.com>
+Date: Thu, 6 Jan 2022 13:18:44 -0500
+Subject: [PATCH 2/3] [cleaner] Improve parser-specific file skipping
+
+This commit improves our handling of skipping files on a per-parser
+basis, by first filtering the list of parsers that `obfuscate_line()`
+will iterate over by the parser's `skip_file` class attr, rather than
+relying on higher-level checks.
+
+Signed-off-by: Jake Hunsaker <jhunsake@redhat.com>
+---
+ sos/cleaner/__init__.py | 17 ++++++++++++++---
+ 1 file changed, 14 insertions(+), 3 deletions(-)
+
+diff --git a/sos/cleaner/__init__.py b/sos/cleaner/__init__.py
+index 3f530d44..5686e213 100644
+--- a/sos/cleaner/__init__.py
++++ b/sos/cleaner/__init__.py
+@@ -12,6 +12,7 @@ import hashlib
+ import json
+ import logging
+ import os
++import re
+ import shutil
+ import tempfile
+ 
+@@ -640,10 +641,16 @@ third party.
+             self.log_debug("Obfuscating %s" % short_name or filename,
+                            caller=arc_name)
+             tfile = tempfile.NamedTemporaryFile(mode='w', dir=self.tmpdir)
++            _parsers = [
++                _p for _p in self.parsers if not
++                any([
++                    re.match(p, short_name) for p in _p.skip_files
++                ])
++            ]
+             with open(filename, 'r') as fname:
+                 for line in fname:
+                     try:
+-                        line, count = self.obfuscate_line(line)
++                        line, count = self.obfuscate_line(line, _parsers)
+                         subs += count
+                         tfile.write(line)
+                     except Exception as err:
+@@ -713,7 +720,7 @@ third party.
+                 pass
+         return string_data
+ 
+-    def obfuscate_line(self, line):
++    def obfuscate_line(self, line, parsers=None):
+         """Run a line through each of the obfuscation parsers, keeping a
+         cumulative total of substitutions done on that particular line.
+ 
+@@ -721,6 +728,8 @@ third party.
+ 
+             :param line str:        The raw line as read from the file being
+                                     processed
++            :param parsers:         A list of parser objects to obfuscate
++                                    with. If None, use all.
+ 
+         Returns the fully obfuscated line and the number of substitutions made
+         """
+@@ -729,7 +738,9 @@ third party.
+         count = 0
+         if not line.strip():
+             return line, count
+-        for parser in self.parsers:
++        if parsers is None:
++            parsers = self.parsers
++        for parser in parsers:
+             try:
+                 line, _count = parser.parse_line(line)
+                 count += _count
+-- 
+2.31.1
+
+
+From 96c9a833e77639a853b7d3d6f1df68bbbbe5e9cb Mon Sep 17 00:00:00 2001
+From: Jake Hunsaker <jhunsake@redhat.com>
+Date: Thu, 6 Jan 2022 13:20:32 -0500
+Subject: [PATCH 3/3] [cleaner] Add skips for known files and usernames
+
+Adds skips for `/proc/kallsyms` which should never be obfuscated, as
+well as any packaging-related log file for the IP parser. Further, do
+not obfuscate the `stack` users, as that is a well-known user for many
+configurations that, if obfuscated, could result in undesired string
+substitutions in normal logging.
+
+Signed-off-by: Jake Hunsaker <jhunsake@redhat.com>
+---
+ sos/cleaner/archives/__init__.py       | 2 ++
+ sos/cleaner/parsers/ip_parser.py       | 3 ++-
+ sos/cleaner/parsers/username_parser.py | 1 +
+ 3 files changed, 5 insertions(+), 1 deletion(-)
+
+diff --git a/sos/cleaner/archives/__init__.py b/sos/cleaner/archives/__init__.py
+index 795c5a78..cbf1f809 100644
+--- a/sos/cleaner/archives/__init__.py
++++ b/sos/cleaner/archives/__init__.py
+@@ -43,6 +43,7 @@ class SoSObfuscationArchive():
+     type_name = 'undetermined'
+     description = 'undetermined'
+     is_nested = False
++    skip_files = []
+     prep_files = {}
+ 
+     def __init__(self, archive_path, tmpdir):
+@@ -111,6 +112,7 @@ class SoSObfuscationArchive():
+         Returns: list of files and file regexes
+         """
+         return [
++            'proc/kallsyms',
+             'sosreport-',
+             'sys/firmware',
+             'sys/fs',
+diff --git a/sos/cleaner/parsers/ip_parser.py b/sos/cleaner/parsers/ip_parser.py
+index 71d38be8..b007368c 100644
+--- a/sos/cleaner/parsers/ip_parser.py
++++ b/sos/cleaner/parsers/ip_parser.py
+@@ -37,7 +37,8 @@ class SoSIPParser(SoSCleanerParser):
+         'sos_commands/snappy/snap_list_--all',
+         'sos_commands/snappy/snap_--version',
+         'sos_commands/vulkan/vulkaninfo',
+-        'var/log/.*dnf.*'
++        'var/log/.*dnf.*',
++        'var/log/.*packag.*'  # get 'packages' and 'packaging' logs
+     ]
+ 
+     map_file_key = 'ip_map'
+diff --git a/sos/cleaner/parsers/username_parser.py b/sos/cleaner/parsers/username_parser.py
+index 229c7de4..3208a655 100644
+--- a/sos/cleaner/parsers/username_parser.py
++++ b/sos/cleaner/parsers/username_parser.py
+@@ -32,6 +32,7 @@ class SoSUsernameParser(SoSCleanerParser):
+         'nobody',
+         'nfsnobody',
+         'shutdown',
++        'stack',
+         'reboot',
+         'root',
+         'ubuntu',
+-- 
+2.31.1
+

sos.spec

@@ -5,7 +5,7 @@
 Summary: A set of tools to gather troubleshooting information from a system
 Name: sos
 Version: 4.2
-Release: 8%{?dist}
+Release: 9%{?dist}
 Group: Applications/System
 Source0: https://github.com/sosreport/sos/archive/%{version}/sos-%{version}.tar.gz
 Source1: sos-audit-%{auditversion}.tgz
@@ -41,6 +41,7 @@ Patch14: sos-bz2024893-cleaner-hostnames-improvements.patch
 Patch15: sos-bz2025611-RHTS-api-change.patch
 Patch16: sos-bz2034001-nvidia-GPU-info.patch
 Patch17: sos-bz2031777-rhui-logs.patch
+Patch18: sos-bz2037350-ocp-backports.patch
 
 %description
 Sos is a set of tools that gathers information about system
@@ -68,6 +69,7 @@ support technicians and developers.
 %patch15 -p1
 %patch16 -p1
 %patch17 -p1
+%patch18 -p1
 
 %build
 %py3_build
@@ -135,6 +137,14 @@ of the system.  Currently storage and filesystem commands are audited.
 
 
 %changelog
+* Fri Jan 07 2022 Pavel Moravec <pmoravec@redhat.com> = 4.2-9
+- add oc transport, backport various PRs for OCP
+  Resolves: bz2037350
+- [report] Provide better warning about estimate-mode
+  Resolves: bz2011537
+- [hostname] Fix loading and detection of long base domains
+  Resolves: bz2024893
+
 * Sun Dec 19 2021 Pavel Moravec <pmoravec@redhat.com> = 4.2-8
 - [rhui] New log folder
   Resolves: bz2031777