From 1eaffef2d04db9dafa82215ff8bd8b4699e10301 Mon Sep 17 00:00:00 2001 From: Pavel Moravec Date: Fri, 8 Oct 2021 09:20:07 +0200 Subject: [PATCH] sos updates to RHEL 9.0 GA - rebase to sos-4.2 - add few fixups above Resolves: bz1869561 bz2011533 bz2011534 bz2011535 Resolves: bz2011537 bz2011538 Signed-off-by: Pavel Moravec --- ...bz1869561-cpuX-individual-sizelimits.patch | 48 + ...bz1925419-gluster-pubkeys-statusfile.patch | 39 - ...8874-potential-issues-static-analyse.patch | 65 - sos-bz1959779-conversions-and-upgrades.patch | 50 - ...65002-skip-selinux-of-proc-everytime.patch | 135 -- ...1967110-collect-cleaning-consistency.patch | 243 -- sos-bz1967111-manpages-see-also.patch | 99 - sos-bz1967112-add-cmd-timeout.patch | 315 --- sos-bz1967113-ds-mask-password-in-ldif.patch | 50 - sos-bz1967114-gather-cups-browsed-logs.patch | 30 - sos-bz1967115-sssd-memcache-and-logs.patch | 62 - sos-bz1967116-ibmvNIC-dynamic-debugs.patch | 29 - sos-bz1967117-pulpcore-plugin.patch | 147 -- sos-bz1967118-saphana-traceback.patch | 30 - sos-bz1967119-collect-nstat.patch | 36 - ...apper-plugin-and-allocation-failures.patch | 121 - sos-bz1967718-sssd-common.patch | 36 - sos-bz1985976-enhance-tc-hw-offload.patch | 32 - ...z1985982-obfuscate-fqdn-from-dnf-log.patch | 78 - sos-bz1985983-ocp-cluster-cleaner.patch | 2156 ----------------- sos-bz1985985-sos-log-effective-options.patch | 284 --- sos-bz1992859-rhui-plugin.patch | 387 --- ...1992861-cleaner-AD-users-obfuscation.patch | 142 -- ...2011533-unpackaged-recursive-symlink.patch | 42 + ...pacapture-under-allow-system-changes.patch | 49 + sos-bz2011535-kernel-psi.patch | 33 + sos-bz2011537-estimate-only-option.patch | 179 ++ ...8-iptables-save-under-nf_tables-kmod.patch | 73 + sos.spec | 66 +- sources | 2 +- 30 files changed, 451 insertions(+), 4607 deletions(-) create mode 100644 sos-bz1869561-cpuX-individual-sizelimits.patch delete mode 100644 sos-bz1925419-gluster-pubkeys-statusfile.patch delete mode 100644 sos-bz1938874-potential-issues-static-analyse.patch delete mode 100644 sos-bz1959779-conversions-and-upgrades.patch delete mode 100644 sos-bz1965002-skip-selinux-of-proc-everytime.patch delete mode 100644 sos-bz1967110-collect-cleaning-consistency.patch delete mode 100644 sos-bz1967111-manpages-see-also.patch delete mode 100644 sos-bz1967112-add-cmd-timeout.patch delete mode 100644 sos-bz1967113-ds-mask-password-in-ldif.patch delete mode 100644 sos-bz1967114-gather-cups-browsed-logs.patch delete mode 100644 sos-bz1967115-sssd-memcache-and-logs.patch delete mode 100644 sos-bz1967116-ibmvNIC-dynamic-debugs.patch delete mode 100644 sos-bz1967117-pulpcore-plugin.patch delete mode 100644 sos-bz1967118-saphana-traceback.patch delete mode 100644 sos-bz1967119-collect-nstat.patch delete mode 100644 sos-bz1967120-snapper-plugin-and-allocation-failures.patch delete mode 100644 sos-bz1967718-sssd-common.patch delete mode 100644 sos-bz1985976-enhance-tc-hw-offload.patch delete mode 100644 sos-bz1985982-obfuscate-fqdn-from-dnf-log.patch delete mode 100644 sos-bz1985983-ocp-cluster-cleaner.patch delete mode 100644 sos-bz1985985-sos-log-effective-options.patch delete mode 100644 sos-bz1992859-rhui-plugin.patch delete mode 100644 sos-bz1992861-cleaner-AD-users-obfuscation.patch create mode 100644 sos-bz2011533-unpackaged-recursive-symlink.patch create mode 100644 sos-bz2011534-opacapture-under-allow-system-changes.patch create mode 100644 sos-bz2011535-kernel-psi.patch create mode 100644 sos-bz2011537-estimate-only-option.patch create mode 100644 sos-bz2011538-iptables-save-under-nf_tables-kmod.patch diff --git a/sos-bz1869561-cpuX-individual-sizelimits.patch b/sos-bz1869561-cpuX-individual-sizelimits.patch new file mode 100644 index 0000000..4d579d7 --- /dev/null +++ b/sos-bz1869561-cpuX-individual-sizelimits.patch @@ -0,0 +1,48 @@ +From b09ed75b09075d86c184b0a63cce9260f2cee4ca Mon Sep 17 00:00:00 2001 +From: Pavel Moravec +Date: Mon, 30 Aug 2021 11:27:48 +0200 +Subject: [PATCH] [processor] Apply sizelimit to /sys/devices/system/cpu/cpuX + +Copy /sys/devices/system/cpu/cpuX with separately applied sizelimit. + +This is required for systems with tens/hundreds of CPUs where the +cumulative directory size exceeds 25MB or even 100MB. + +Resolves: #2639 +Closes: #2665 + +Signed-off-by: Pavel Moravec +--- + sos/report/plugins/processor.py | 9 ++++++++- + 1 file changed, 8 insertions(+), 1 deletion(-) + +diff --git a/sos/report/plugins/processor.py b/sos/report/plugins/processor.py +index 0ddfd126..2df2dc9a 100644 +--- a/sos/report/plugins/processor.py ++++ b/sos/report/plugins/processor.py +@@ -7,6 +7,7 @@ + # See the LICENSE file in the source distribution for further information. + + from sos.report.plugins import Plugin, IndependentPlugin ++import os + + + class Processor(Plugin, IndependentPlugin): +@@ -34,7 +35,13 @@ class Processor(Plugin, IndependentPlugin): + self.add_copy_spec([ + "/proc/cpuinfo", + "/sys/class/cpuid", +- "/sys/devices/system/cpu" ++ ]) ++ # copy /sys/devices/system/cpu/cpuX with separately applied sizelimit ++ # this is required for systems with tens/hundreds of CPUs where the ++ # cumulative directory size exceeds 25MB or even 100MB. ++ cdirs = self.listdir('/sys/devices/system/cpu') ++ self.add_copy_spec([ ++ os.path.join('/sys/devices/system/cpu', cdir) for cdir in cdirs + ]) + + self.add_cmd_output([ +-- +2.31.1 + diff --git a/sos-bz1925419-gluster-pubkeys-statusfile.patch b/sos-bz1925419-gluster-pubkeys-statusfile.patch deleted file mode 100644 index 7fd317c..0000000 --- a/sos-bz1925419-gluster-pubkeys-statusfile.patch +++ /dev/null @@ -1,39 +0,0 @@ -From 4fb834ec862228afb276ccbd45aa86c66044ea66 Mon Sep 17 00:00:00 2001 -From: Pavel Moravec -Date: Mon, 15 Mar 2021 09:09:51 +0100 -Subject: [PATCH] [gluster] collect public keys from the right dir - -Collection of glusterfind dir is achieved by /var/lib/gluster -so it doesn't be collected explicitly. - -/var/lib/glusterd/glusterfind/.keys/ subdir is required to be -explicitly collected, as add_copy_spec uses glob.glob() that skips -hidden files. - -Resolves: #2451 - -Signed-off-by: Pavel Moravec -Signed-off-by: Jake Hunsaker ---- - sos/report/plugins/gluster.py | 5 ++--- - 1 file changed, 2 insertions(+), 3 deletions(-) - -diff --git a/sos/report/plugins/gluster.py b/sos/report/plugins/gluster.py -index e1a89df2..952cab63 100644 ---- a/sos/report/plugins/gluster.py -+++ b/sos/report/plugins/gluster.py -@@ -76,9 +76,8 @@ class Gluster(Plugin, RedHatPlugin): - "/var/lib/glusterd/", - # collect nfs-ganesha related configuration - "/run/gluster/shared_storage/nfs-ganesha/", -- # collect status files and public ssh keys -- "/var/lib/glusterd/.keys/", -- "/var/lib/glusterd/glusterfind/" -+ # collect public ssh keys (a_s_c skips implicit hidden files) -+ "/var/lib/glusterd/glusterfind/.keys/", - ] + glob.glob('/run/gluster/*tier-dht/*')) - - if not self.get_option("all_logs"): --- -2.26.2 - diff --git a/sos-bz1938874-potential-issues-static-analyse.patch b/sos-bz1938874-potential-issues-static-analyse.patch deleted file mode 100644 index 0c359e6..0000000 --- a/sos-bz1938874-potential-issues-static-analyse.patch +++ /dev/null @@ -1,65 +0,0 @@ -From 6d5cbe90e17534d53d7fe42dff4d8ca734acf594 Mon Sep 17 00:00:00 2001 -From: Jake Hunsaker -Date: Tue, 29 Jun 2021 15:49:00 -0400 -Subject: [PATCH] [yum] Fix potential traceback when yum history is empty - -Like we did in #969 for `dnf`, fix a potential issue where we would -generate a traceback in the plugin when `yum history` is empty. - -Signed-off-by: Jake Hunsaker ---- - sos/report/plugins/yum.py | 2 +- - 1 file changed, 1 insertion(+), 1 deletion(-) - -diff --git a/sos/report/plugins/yum.py b/sos/report/plugins/yum.py -index 54e222df..aec805e6 100644 ---- a/sos/report/plugins/yum.py -+++ b/sos/report/plugins/yum.py -@@ -91,7 +91,7 @@ class Yum(Plugin, RedHatPlugin): - # packages installed/erased/updated per transaction - if self.get_option("yum-history-info"): - history = self.exec_cmd("yum history") -- transactions = None -+ transactions = -1 - if history['status'] == 0: - for line in history['output'].splitlines(): - try: --- -2.31.1 - -From a7a4ef73faee6cddba36bf670d4a20ab0521c36f Mon Sep 17 00:00:00 2001 -From: Pavel Moravec -Date: Wed, 30 Jun 2021 13:10:56 +0200 -Subject: [PATCH] [plugins] Set default predicate instead of None for - robustness - -Just making the code more robustness, it could be dangerous to -set pred = None and then potentially call log_skipped_cmd that -expects "pred" of SoSPredicate type. - -Currently such a call flow can not happen, but it is worth to -make the code more robust for potential future changes. - -Resolves: #2601 - -Signed-off-by: Pavel Moravec ---- - sos/report/plugins/__init__.py | 2 +- - 1 file changed, 1 insertion(+), 1 deletion(-) - -diff --git a/sos/report/plugins/__init__.py b/sos/report/plugins/__init__.py -index 6fd1a3b2..b9cd28ed 100644 ---- a/sos/report/plugins/__init__.py -+++ b/sos/report/plugins/__init__.py -@@ -1629,7 +1629,7 @@ class Plugin(object): - - def _add_cmd_output(self, **kwargs): - """Internal helper to add a single command to the collection list.""" -- pred = kwargs.pop('pred') if 'pred' in kwargs else None -+ pred = kwargs.pop('pred') if 'pred' in kwargs else SoSPredicate(self) - soscmd = SoSCommand(**kwargs) - self._log_debug("packed command: " + soscmd.__str__()) - for _skip_cmd in self.skip_commands: --- -2.31.1 - diff --git a/sos-bz1959779-conversions-and-upgrades.patch b/sos-bz1959779-conversions-and-upgrades.patch deleted file mode 100644 index a39f839..0000000 --- a/sos-bz1959779-conversions-and-upgrades.patch +++ /dev/null @@ -1,50 +0,0 @@ -From ee5d9d017b0a1bfeaebee9c21c17e89ef1f909a8 Mon Sep 17 00:00:00 2001 -From: Pavel Moravec -Date: Mon, 26 Jul 2021 13:30:09 +0200 -Subject: [PATCH] [MigrationResults] collect info about conversions and - upgrades - -A new tiny plugin independent on leapp and convert2rhel is proposed. - -It should collect /etc/migration-results with info about RHEL -conversions and upgrades, whenever the file is present. - -Resolves: #2627 -Relevant to: rhbz#1959598 - -Signed-off-by: Pavel Moravec ---- - sos/report/plugins/migration_results.py | 21 +++++++++++++++++++++ - 1 file changed, 21 insertions(+) - create mode 100644 sos/report/plugins/migration_results.py - -diff --git a/sos/report/plugins/migration_results.py b/sos/report/plugins/migration_results.py -new file mode 100644 -index 00000000..b67480ba ---- /dev/null -+++ b/sos/report/plugins/migration_results.py -@@ -0,0 +1,21 @@ -+# This file is part of the sos project: https://github.com/sosreport/sos -+# -+# This copyrighted material is made available to anyone wishing to use, -+# modify, copy, or redistribute it subject to the terms and conditions of -+# version 2 of the GNU General Public License. -+# -+# See the LICENSE file in the source distribution for further information. -+ -+from sos.report.plugins import Plugin, RedHatPlugin -+ -+ -+class MigrationResults(Plugin, RedHatPlugin): -+ -+ short_desc = 'Information about conversions and upgrades' -+ -+ plugin_name = 'migration_results' -+ profiles = ('system',) -+ -+ files = ('/etc/migration-results',) -+ -+# vim: et ts=4 sw=4 --- -2.31.1 - diff --git a/sos-bz1965002-skip-selinux-of-proc-everytime.patch b/sos-bz1965002-skip-selinux-of-proc-everytime.patch deleted file mode 100644 index 1b62a1a..0000000 --- a/sos-bz1965002-skip-selinux-of-proc-everytime.patch +++ /dev/null @@ -1,135 +0,0 @@ -From 206d65618f20995b168dcc63090d1e6871450e90 Mon Sep 17 00:00:00 2001 -From: Pavel Moravec -Date: Wed, 26 May 2021 15:45:26 +0200 -Subject: [PATCH] [archive] skip copying SELinux context for /proc and /sys - everytime - -A supplement of #1399 fix, now also for adding strings or special -device files. - -Also adding a (vendor) test case for it. - -Resolves: #2560 - -Signed-off-by: Pavel Moravec ---- - sos/archive.py | 35 +++++++++++---------- - tests/vendor_tests/redhat/rhbz1965001.py | 39 ++++++++++++++++++++++++ - 2 files changed, 56 insertions(+), 18 deletions(-) - create mode 100644 tests/vendor_tests/redhat/rhbz1965001.py - -diff --git a/sos/archive.py b/sos/archive.py -index 4dd31d75..b02b2475 100644 ---- a/sos/archive.py -+++ b/sos/archive.py -@@ -326,6 +326,20 @@ class FileCacheArchive(Archive): - return None - return dest - -+ def _copy_attributes(self, src, dest): -+ # copy file attributes, skip SELinux xattrs for /sys and /proc -+ try: -+ stat = os.stat(src) -+ if src.startswith("/sys/") or src.startswith("/proc/"): -+ shutil.copymode(src, dest) -+ os.utime(dest, ns=(stat.st_atime_ns, stat.st_mtime_ns)) -+ else: -+ shutil.copystat(src, dest) -+ os.chown(dest, stat.st_uid, stat.st_gid) -+ except Exception as e: -+ self.log_debug("caught '%s' setting attributes of '%s'" -+ % (e, dest)) -+ - def add_file(self, src, dest=None): - with self._path_lock: - if not dest: -@@ -348,18 +362,7 @@ class FileCacheArchive(Archive): - else: - self.log_info("File %s not collected: '%s'" % (src, e)) - -- # copy file attributes, skip SELinux xattrs for /sys and /proc -- try: -- stat = os.stat(src) -- if src.startswith("/sys/") or src.startswith("/proc/"): -- shutil.copymode(src, dest) -- os.utime(dest, ns=(stat.st_atime_ns, stat.st_mtime_ns)) -- else: -- shutil.copystat(src, dest) -- os.chown(dest, stat.st_uid, stat.st_gid) -- except Exception as e: -- self.log_debug("caught '%s' setting attributes of '%s'" -- % (e, dest)) -+ self._copy_attributes(src, dest) - file_name = "'%s'" % src - else: - # Open file case: first rewind the file to obtain -@@ -388,11 +391,7 @@ class FileCacheArchive(Archive): - content = content.decode('utf8', 'ignore') - f.write(content) - if os.path.exists(src): -- try: -- shutil.copystat(src, dest) -- except OSError as e: -- self.log_error("Unable to add '%s' to archive: %s" % -- (dest, e)) -+ self._copy_attributes(src, dest) - self.log_debug("added string at '%s' to FileCacheArchive '%s'" - % (src, self._archive_root)) - -@@ -501,7 +500,7 @@ class FileCacheArchive(Archive): - self.log_info("add_node: %s - mknod '%s'" % (msg, dest)) - return - raise e -- shutil.copystat(path, dest) -+ self._copy_attributes(path, dest) - - def name_max(self): - if 'PC_NAME_MAX' in os.pathconf_names: -diff --git a/tests/vendor_tests/redhat/rhbz1965001.py b/tests/vendor_tests/redhat/rhbz1965001.py -new file mode 100644 -index 00000000..aa16ba81 ---- /dev/null -+++ b/tests/vendor_tests/redhat/rhbz1965001.py -@@ -0,0 +1,39 @@ -+# This file is part of the sos project: https://github.com/sosreport/sos -+# -+# This copyrighted material is made available to anyone wishing to use, -+# modify, copy, or redistribute it subject to the terms and conditions of -+# version 2 of the GNU General Public License. -+# -+# See the LICENSE file in the source distribution for further information. -+ -+ -+import tempfile -+import shutil -+from sos_tests import StageOneReportTest -+ -+ -+class rhbz1965001(StageOneReportTest): -+ """ -+ Copying /proc/sys/vm/{compact_memory,drop_caches} must ignore SELinux -+ context, otherwise an attempt to set the context to files under some -+ directories like /tmp raises an AVC denial, and an ERROR -+ "Unable to add '...' to archive: [Errno 13] Permission denied: '...' -+ is raise. -+ -+ https://bugzilla.redhat.com/show_bug.cgi?id=1965001 -+ -+ :avocado: enable -+ :avocado: tags=stageone -+ """ -+ -+ sos_cmd = '-o system' -+ # it is crucial to run the test case with --tmp-dir=/tmp/... as that is -+ # (an example of) directory exhibiting the relabel permission deny. -+ # /var/tmp directory allows those relabels. -+ # -+ # the directory shouldn't exist at this moment, otherwise -+ # "check to prevent multiple setUp() runs" in sos_tests.py would fail -+ _tmpdir = '/tmp/rhbz1965001_avocado_test' -+ -+ def test_no_permission_denied(self): -+ self.assertSosLogNotContains("Permission denied") --- -2.26.3 - diff --git a/sos-bz1967110-collect-cleaning-consistency.patch b/sos-bz1967110-collect-cleaning-consistency.patch deleted file mode 100644 index 0ded10a..0000000 --- a/sos-bz1967110-collect-cleaning-consistency.patch +++ /dev/null @@ -1,243 +0,0 @@ -From fc0218638f3e865c4315823e72aef2f46d012d07 Mon Sep 17 00:00:00 2001 -From: Jake Hunsaker -Date: Wed, 14 Apr 2021 11:55:03 -0400 -Subject: [PATCH 1/2] [clean] Load maps from all archives before obfuscation - loop - -Previously, maps were being prepped via archives after extraction. This -reduced the amount of file IO being done, but made it so that necessary -obfuscations from later archives in a series would not be obfuscated in -the archives obfuscated before those later archives were extracted. - -Fix this by extracting the map prep files into memory for each archive -to prep the maps before we enter the obfuscation loop entirely. - -Closes: #2490 -Related: RHBZ#1930181 -Resolves: #2492 - -Signed-off-by: Jake Hunsaker ---- - sos/cleaner/__init__.py | 69 +++++++++++++++----------- - sos/cleaner/parsers/username_parser.py | 13 +++-- - 2 files changed, 45 insertions(+), 37 deletions(-) - -diff --git a/sos/cleaner/__init__.py b/sos/cleaner/__init__.py -index b9eb61ef..d10cdc55 100644 ---- a/sos/cleaner/__init__.py -+++ b/sos/cleaner/__init__.py -@@ -292,6 +292,7 @@ third party. - - # we have at least one valid target to obfuscate - self.completed_reports = [] -+ self.preload_all_archives_into_maps() - self.obfuscate_report_paths() - - if not self.completed_reports: -@@ -473,6 +474,44 @@ third party. - self.ui_log.info("Exiting on user cancel") - os._exit(130) - -+ def preload_all_archives_into_maps(self): -+ """Before doing the actual obfuscation, if we have multiple archives -+ to obfuscate then we need to preload each of them into the mappings -+ to ensure that node1 is obfuscated in node2 as well as node2 being -+ obfuscated in node1's archive. -+ """ -+ self.log_info("Pre-loading multiple archives into obfuscation maps") -+ for _arc in self.report_paths: -+ is_dir = os.path.isdir(_arc) -+ if is_dir: -+ _arc_name = _arc -+ else: -+ archive = tarfile.open(_arc) -+ _arc_name = _arc.split('/')[-1].split('.tar')[0] -+ # for each parser, load the map_prep_file into memory, and then -+ # send that for obfuscation. We don't actually obfuscate the file -+ # here, do that in the normal archive loop -+ for _parser in self.parsers: -+ if not _parser.prep_map_file: -+ continue -+ _arc_path = os.path.join(_arc_name, _parser.prep_map_file) -+ try: -+ if is_dir: -+ _pfile = open(_arc_path, 'r') -+ content = _pfile.read() -+ else: -+ _pfile = archive.extractfile(_arc_path) -+ content = _pfile.read().decode('utf-8') -+ _pfile.close() -+ if isinstance(_parser, SoSUsernameParser): -+ _parser.load_usernames_into_map(content) -+ for line in content.splitlines(): -+ if isinstance(_parser, SoSHostnameParser): -+ _parser.load_hostname_into_map(line) -+ self.obfuscate_line(line, _parser.prep_map_file) -+ except Exception as err: -+ self.log_debug("Could not prep %s: %s" % (_arc_path, err)) -+ - def obfuscate_report(self, report): - """Individually handle each archive or directory we've discovered by - running through each file therein. -@@ -493,7 +532,6 @@ third party. - start_time = datetime.now() - arc_md.add_field('start_time', start_time) - archive.extract() -- self.prep_maps_from_archive(archive) - archive.report_msg("Beginning obfuscation...") - - file_list = archive.get_file_list() -@@ -542,35 +580,6 @@ third party. - self.ui_log.info("Exception while processing %s: %s" - % (report, err)) - -- def prep_maps_from_archive(self, archive): -- """Open specific files from an archive and try to load those values -- into our mappings before iterating through the entire archive. -- -- Positional arguments: -- -- :param archive SoSObfuscationArchive: An open archive object -- """ -- for parser in self.parsers: -- if not parser.prep_map_file: -- continue -- prep_file = archive.get_file_path(parser.prep_map_file) -- if not prep_file: -- self.log_debug("Could not prepare %s: %s does not exist" -- % (parser.name, parser.prep_map_file), -- caller=archive.archive_name) -- continue -- # this is a bit clunky, but we need to load this particular -- # parser in a different way due to how hostnames are validated for -- # obfuscation -- if isinstance(parser, SoSHostnameParser): -- with open(prep_file, 'r') as host_file: -- hostname = host_file.readline().strip() -- parser.load_hostname_into_map(hostname) -- if isinstance(parser, SoSUsernameParser): -- parser.load_usernames_into_map(prep_file) -- self.obfuscate_file(prep_file, parser.prep_map_file, -- archive.archive_name) -- - def obfuscate_file(self, filename, short_name=None, arc_name=None): - """Obfuscate and individual file, line by line. - -diff --git a/sos/cleaner/parsers/username_parser.py b/sos/cleaner/parsers/username_parser.py -index 5223c018..2bb6c7f3 100644 ---- a/sos/cleaner/parsers/username_parser.py -+++ b/sos/cleaner/parsers/username_parser.py -@@ -39,16 +39,15 @@ class SoSUsernameParser(SoSCleanerParser): - super(SoSUsernameParser, self).__init__(conf_file) - self.mapping.load_names_from_options(opt_names) - -- def load_usernames_into_map(self, fname): -+ def load_usernames_into_map(self, content): - """Since we don't get the list of usernames from a straight regex for - this parser, we need to override the initial parser prepping here. - """ -- with open(fname, 'r') as lastfile: -- for line in lastfile.read().splitlines()[1:]: -- user = line.split()[0] -- if user in self.skip_list: -- continue -- self.mapping.get(user) -+ for line in content.splitlines()[1:]: -+ user = line.split()[0] -+ if user in self.skip_list: -+ continue -+ self.mapping.get(user) - - def parse_line(self, line): - count = 0 --- -2.26.3 - - -From b713f458bfa92427147de754ea36054bfde53d71 Mon Sep 17 00:00:00 2001 -From: Jake Hunsaker -Date: Wed, 14 Apr 2021 12:22:28 -0400 -Subject: [PATCH 2/2] [clean] Remove duplicate file skipping within - obfuscate_line() - -A redundant file skipping check was being executed within -`obfuscate_line()` that would cause subsequent archives being obfuscated -to skip line obfuscation within a file, despite iterating through the -entire file. - -Remove this redundant check, thus allowing proper obfuscation. - -Closes: #2490 -Related: RHBZ#1930181 -Resolves: #2492 - -Signed-off-by: Jake Hunsaker ---- - sos/cleaner/__init__.py | 11 +++-------- - sos/cleaner/obfuscation_archive.py | 2 -- - 2 files changed, 3 insertions(+), 10 deletions(-) - -diff --git a/sos/cleaner/__init__.py b/sos/cleaner/__init__.py -index d10cdc55..bdd24f95 100644 ---- a/sos/cleaner/__init__.py -+++ b/sos/cleaner/__init__.py -@@ -508,7 +508,7 @@ third party. - for line in content.splitlines(): - if isinstance(_parser, SoSHostnameParser): - _parser.load_hostname_into_map(line) -- self.obfuscate_line(line, _parser.prep_map_file) -+ self.obfuscate_line(line) - except Exception as err: - self.log_debug("Could not prep %s: %s" % (_arc_path, err)) - -@@ -606,7 +606,7 @@ third party. - if not line.strip(): - continue - try: -- line, count = self.obfuscate_line(line, short_name) -+ line, count = self.obfuscate_line(line) - subs += count - tfile.write(line) - except Exception as err: -@@ -631,7 +631,7 @@ third party. - pass - return string_data - -- def obfuscate_line(self, line, filename): -+ def obfuscate_line(self, line): - """Run a line through each of the obfuscation parsers, keeping a - cumulative total of substitutions done on that particular line. - -@@ -639,16 +639,11 @@ third party. - - :param line str: The raw line as read from the file being - processed -- :param filename str: Filename the line was read from - - Returns the fully obfuscated line and the number of substitutions made - """ - count = 0 - for parser in self.parsers: -- if filename and any([ -- re.match(_s, filename) for _s in parser.skip_files -- ]): -- continue - try: - line, _count = parser.parse_line(line) - count += _count -diff --git a/sos/cleaner/obfuscation_archive.py b/sos/cleaner/obfuscation_archive.py -index 84ca30cd..c64ab13b 100644 ---- a/sos/cleaner/obfuscation_archive.py -+++ b/sos/cleaner/obfuscation_archive.py -@@ -219,8 +219,6 @@ class SoSObfuscationArchive(): - :param filename str: Filename relative to the extracted - archive root - """ -- if filename in self.file_sub_list: -- return True - - if not os.path.isfile(self.get_file_path(filename)): - return True --- -2.26.3 - diff --git a/sos-bz1967111-manpages-see-also.patch b/sos-bz1967111-manpages-see-also.patch deleted file mode 100644 index 6486b48..0000000 --- a/sos-bz1967111-manpages-see-also.patch +++ /dev/null @@ -1,99 +0,0 @@ -From 3b439fb64d8d65b0c09aa8452bf0181ec20f8bcf Mon Sep 17 00:00:00 2001 -From: Jose Castillo -Date: Wed, 3 Mar 2021 13:03:16 +0100 -Subject: [PATCH] [man] Multiple fixes in man pages - -This patch fixes references to sosreport, to the -preferred 'sos report'. Also adds "SEE ALSO" consistently -for all man pages, and fixes a MAINTAINER line. - -Resolves: #2432 - -Signed-off-by: Jose Castillo -Signed-off-by: Jake Hunsaker ---- - man/en/sos-clean.1 | 5 +++++ - man/en/sos-collect.1 | 1 + - man/en/sos-report.1 | 22 ++++++++++++++-------- - 3 files changed, 20 insertions(+), 8 deletions(-) - -diff --git a/man/en/sos-clean.1 b/man/en/sos-clean.1 -index 0c62ed07..d64a0ec7 100644 ---- a/man/en/sos-clean.1 -+++ b/man/en/sos-clean.1 -@@ -77,6 +77,11 @@ Default: 4 - .TP - .B \-\-no-update - Do not write the mapping file contents to /etc/sos/cleaner/default_mapping -+.SH SEE ALSO -+.BR sos (1) -+.BR sos-report (1) -+.BR sos-collect (1) -+ - .SH MAINTAINER - .nf - Jake Hunsaker -diff --git a/man/en/sos-collect.1 b/man/en/sos-collect.1 -index d4e5e648..da36542d 100644 ---- a/man/en/sos-collect.1 -+++ b/man/en/sos-collect.1 -@@ -330,6 +330,7 @@ Sosreport option. Override the default compression type. - .SH SEE ALSO - .BR sos (1) - .BR sos-report (1) -+.BR sos-clean (1) - - .SH MAINTAINER - Jake Hunsaker -diff --git a/man/en/sos-report.1 b/man/en/sos-report.1 -index e7fae97b..81005959 100644 ---- a/man/en/sos-report.1 -+++ b/man/en/sos-report.1 -@@ -38,11 +38,12 @@ sosreport \- Collect and package diagnostic and support data - [-h|--help]\fR - - .SH DESCRIPTION --\fBsosreport\fR generates an archive of configuration and diagnostic --information from the running system. The archive may be stored locally --or centrally for recording or tracking purposes or may be sent to --technical support representatives, developers or system administrators --to assist with technical fault-finding and debugging. -+\fBreport\fR is an sos subcommand that generates an archive of -+configuration and diagnostic information from the running system. -+The archive may be stored locally or centrally for recording or -+tracking purposes or may be sent to technical support representatives, -+developers or system administrators to assist with technical -+fault-finding and debugging. - .LP - Sos is modular in design and is able to collect data from a wide - range of subsystems and packages that may be installed. An -@@ -110,8 +111,8 @@ User defined presets are saved under /var/lib/sos/presets as JSON-formatted file - .B \--add-preset ADD_PRESET [options] - Add a preset with name ADD_PRESET that enables [options] when called. - --For example, 'sosreport --add-preset mypreset --log-size=50 -n logs' will enable --a user to run 'sosreport --preset mypreset' that sets the maximum log size to -+For example, 'sos report --add-preset mypreset --log-size=50 -n logs' will enable -+a user to run 'sos report --preset mypreset' that sets the maximum log size to - 50 and disables the logs plugin. - - Note: to set a description for the preset that is displayed with \fB--list-presets\fR, -@@ -343,9 +344,14 @@ been tested for this port or may still be under active development. - .TP - .B \--help - Display usage message. -+.SH SEE ALSO -+.BR sos (1) -+.BR sos-clean (1) -+.BR sos-collect (1) -+ - .SH MAINTAINER - .nf --Bryn M. Reeves -+Jake Hunsaker - .fi - .SH AUTHORS & CONTRIBUTORS - See \fBAUTHORS\fR file in the package documentation. --- -2.26.3 - diff --git a/sos-bz1967112-add-cmd-timeout.patch b/sos-bz1967112-add-cmd-timeout.patch deleted file mode 100644 index db84839..0000000 --- a/sos-bz1967112-add-cmd-timeout.patch +++ /dev/null @@ -1,315 +0,0 @@ -From 90b6b709e9f4002376b656b155d00d85382f1828 Mon Sep 17 00:00:00 2001 -From: Pavel Moravec -Date: Mon, 29 Mar 2021 16:23:01 +0200 -Subject: [PATCH] [report] add --cmd-timeout option - -Add --cmd-timeout option to configure command timeout. Plugin-specific -option of the same name (i.e. -k logs.cmd-timeout=60) can control the -timeout per plugin. - -Option defaults and global/plugin-specific option preference follows the ---plugin-timeout rules. - -Resolves: #2466 - -Signed-off-by: Pavel Moravec -Signed-off-by: Jake Hunsaker ---- - man/en/sos-report.1 | 18 +++++++++- - sos/collector/__init__.py | 3 ++ - sos/collector/sosnode.py | 5 +++ - sos/options.py | 3 +- - sos/report/__init__.py | 5 ++- - sos/report/plugins/__init__.py | 63 ++++++++++++++++++++++++---------- - 6 files changed, 76 insertions(+), 21 deletions(-) - -diff --git a/man/en/sos-report.1 b/man/en/sos-report.1 -index 81005959..51cf3436 100644 ---- a/man/en/sos-report.1 -+++ b/man/en/sos-report.1 -@@ -17,6 +17,7 @@ sosreport \- Collect and package diagnostic and support data - [--label label] [--case-id id]\fR - [--threads threads]\fR - [--plugin-timeout TIMEOUT]\fR -+ [--cmd-timeout TIMEOUT]\fR - [-s|--sysroot SYSROOT]\fR - [-c|--chroot {auto|always|never}\fR - [--tmp-dir directory]\fR -@@ -247,7 +248,7 @@ Specify a timeout in seconds to allow each plugin to run for. A value of 0 - means no timeout will be set. A value of -1 is used to indicate the default - timeout of 300 seconds. - --Note that this options sets the timeout for all plugins. If you want to set -+Note that this option sets the timeout for all plugins. If you want to set - a timeout for a specific plugin, use the 'timeout' plugin option available to - all plugins - e.g. '-k logs.timeout=600'. - -@@ -255,6 +256,21 @@ The plugin-specific timeout option will override this option. For example, using - \'--plugin-timeout=60 -k logs.timeout=600\' will set a timeout of 600 seconds for - the logs plugin and 60 seconds for all other enabled plugins. - .TP -+.B \--cmd-timeout TIMEOUT -+Specify a timeout limit in seconds for a command execution. Same defaults logic -+from --plugin-timeout applies here. -+ -+This option sets the command timeout for all plugins. If you want to set a cmd -+timeout for a specific plugin, use the 'cmd-timeout' plugin option available to -+all plugins - e.g. '-k logs.cmd-timeout=600'. -+ -+Again, the same plugin/global precedence logic as for --plugin-timeout applies -+here. -+ -+Note that setting --cmd-timeout (or -k logs.cmd-timeout) high should be followed -+by increasing the --plugin-timeout equivalent, otherwise the plugin can easily -+timeout on slow commands execution. -+.TP - .B \--case-id NUMBER - Specify a case identifier to associate with the archive. - Identifiers may include alphanumeric characters, commas and periods ('.'). -diff --git a/sos/collector/__init__.py b/sos/collector/__init__.py -index 406c8f35..1ae73508 100644 ---- a/sos/collector/__init__.py -+++ b/sos/collector/__init__.py -@@ -82,6 +82,7 @@ class SoSCollector(SoSComponent): - 'password_per_node': False, - 'plugin_options': [], - 'plugin_timeout': None, -+ 'cmd_timeout': None, - 'preset': '', - 'save_group': '', - 'since': '', -@@ -276,6 +277,8 @@ class SoSCollector(SoSComponent): - help='Do not collect env vars in sosreports') - sos_grp.add_argument('--plugin-timeout', type=int, default=None, - help='Set the global plugin timeout value') -+ sos_grp.add_argument('--cmd-timeout', type=int, default=None, -+ help='Set the global command timeout value') - sos_grp.add_argument('--since', default=None, - help=('Escapes archived files older than date. ' - 'This will also affect --all-logs. ' -diff --git a/sos/collector/sosnode.py b/sos/collector/sosnode.py -index a1679655..dbbee12e 100644 ---- a/sos/collector/sosnode.py -+++ b/sos/collector/sosnode.py -@@ -664,6 +664,11 @@ class SosNode(): - '--skip-files=%s' % (quote(self.opts.skip_files)) - ) - -+ if self.check_sos_version('4.2'): -+ if self.opts.cmd_timeout: -+ sos_opts.append('--cmd-timeout=%s' -+ % quote(str(self.opts.cmd_timeout))) -+ - sos_cmd = sos_cmd.replace( - 'sosreport', - os.path.join(self.host.sos_bin_path, self.sos_bin) -diff --git a/sos/options.py b/sos/options.py -index b82a7d36..1eda55d6 100644 ---- a/sos/options.py -+++ b/sos/options.py -@@ -283,7 +283,8 @@ class SoSOptions(): - if name in ("add_preset", "del_preset", "desc", "note"): - return False - # Exception list for options that still need to be reported when 0 -- if name in ['log_size', 'plugin_timeout'] and value == 0: -+ if name in ['log_size', 'plugin_timeout', 'cmd_timeout'] \ -+ and value == 0: - return True - return has_value(name, value) - -diff --git a/sos/report/__init__.py b/sos/report/__init__.py -index 25478ba7..945d0fc1 100644 ---- a/sos/report/__init__.py -+++ b/sos/report/__init__.py -@@ -107,6 +107,7 @@ class SoSReport(SoSComponent): - 'only_plugins': [], - 'preset': 'auto', - 'plugin_timeout': 300, -+ 'cmd_timeout': 300, - 'profiles': [], - 'since': None, - 'verify': False, -@@ -266,6 +267,8 @@ class SoSReport(SoSComponent): - help="A preset identifier", default="auto") - report_grp.add_argument("--plugin-timeout", default=None, - help="set a timeout for all plugins") -+ report_grp.add_argument("--cmd-timeout", default=None, -+ help="set a command timeout for all plugins") - report_grp.add_argument("-p", "--profile", "--profiles", - action="extend", dest="profiles", type=str, - default=[], -@@ -709,7 +712,7 @@ class SoSReport(SoSComponent): - - self.ui_log.info(_("The following plugin options are available:")) - for (plug, plugname, optname, optparm) in self.all_options: -- if optname in ('timeout', 'postproc'): -+ if optname in ('timeout', 'postproc', 'cmd-timeout'): - continue - # format option value based on its type (int or bool) - if type(optparm["enabled"]) == bool: -diff --git a/sos/report/plugins/__init__.py b/sos/report/plugins/__init__.py -index 02625eb1..779119af 100644 ---- a/sos/report/plugins/__init__.py -+++ b/sos/report/plugins/__init__.py -@@ -472,6 +472,9 @@ class Plugin(object): - _default_plug_opts = [ - ('timeout', 'Timeout in seconds for plugin. The default value (-1) ' + - 'defers to the general plugin timeout, 300 seconds', 'fast', -1), -+ ('cmd-timeout', 'Timeout in seconds for a command execution. The ' + -+ 'default value (-1) defers to the general cmd timeout, 300 ' + -+ 'seconds', 'fast', -1), - ('postproc', 'Enable post-processing collected plugin data', 'fast', - True) - ] -@@ -532,16 +535,15 @@ class Plugin(object): - self.manifest.add_list('commands', []) - self.manifest.add_list('files', []) - -- @property -- def timeout(self): -- """Returns either the default plugin timeout value, the value as -- provided on the commandline via -k plugin.timeout=value, or the value -- of the global --plugin-timeout option. -+ def timeout_from_options(self, optname, plugoptname, default_timeout): -+ """Returns either the default [plugin|cmd] timeout value, the value as -+ provided on the commandline via -k plugin.[|cmd-]timeout=value, or the -+ value of the global --[plugin|cmd]-timeout option. - """ - _timeout = None - try: -- opt_timeout = self.get_option('plugin_timeout') -- own_timeout = int(self.get_option('timeout')) -+ opt_timeout = self.get_option(optname) -+ own_timeout = int(self.get_option(plugoptname)) - if opt_timeout is None: - _timeout = own_timeout - elif opt_timeout is not None and own_timeout == -1: -@@ -551,10 +553,30 @@ class Plugin(object): - else: - return None - except ValueError: -- return self.plugin_timeout # Default to known safe value -+ return default_timeout # Default to known safe value - if _timeout is not None and _timeout > -1: - return _timeout -- return self.plugin_timeout -+ return default_timeout -+ -+ @property -+ def timeout(self): -+ """Returns either the default plugin timeout value, the value as -+ provided on the commandline via -k plugin.timeout=value, or the value -+ of the global --plugin-timeout option. -+ """ -+ _timeout = self.timeout_from_options('plugin_timeout', 'timeout', -+ self.plugin_timeout) -+ return _timeout -+ -+ @property -+ def cmdtimeout(self): -+ """Returns either the default command timeout value, the value as -+ provided on the commandline via -k plugin.cmd-timeout=value, or the -+ value of the global --cmd-timeout option. -+ """ -+ _cmdtimeout = self.timeout_from_options('cmd_timeout', 'cmd-timeout', -+ self.cmd_timeout) -+ return _cmdtimeout - - def set_timeout_hit(self): - self._timeout_hit = True -@@ -1235,8 +1257,8 @@ class Plugin(object): - """ - - global_options = ( -- 'all_logs', 'allow_system_changes', 'log_size', 'plugin_timeout', -- 'since', 'verify' -+ 'all_logs', 'allow_system_changes', 'cmd_timeout', 'log_size', -+ 'plugin_timeout', 'since', 'verify' - ) - - if optionname in global_options: -@@ -1505,7 +1527,7 @@ class Plugin(object): - 'tags': _spec_tags - }) - -- def add_blockdev_cmd(self, cmds, devices='block', timeout=300, -+ def add_blockdev_cmd(self, cmds, devices='block', timeout=None, - sizelimit=None, chroot=True, runat=None, env=None, - binary=False, prepend_path=None, whitelist=[], - blacklist=[], tags=[]): -@@ -1569,7 +1591,7 @@ class Plugin(object): - whitelist=whitelist, blacklist=blacklist, - tags=_dev_tags) - -- def _add_device_cmd(self, cmds, devices, timeout=300, sizelimit=None, -+ def _add_device_cmd(self, cmds, devices, timeout=None, sizelimit=None, - chroot=True, runat=None, env=None, binary=False, - prepend_path=None, whitelist=[], blacklist=[], - tags=[]): -@@ -1627,7 +1649,7 @@ class Plugin(object): - changes=soscmd.changes) - - def add_cmd_output(self, cmds, suggest_filename=None, -- root_symlink=None, timeout=cmd_timeout, stderr=True, -+ root_symlink=None, timeout=None, stderr=True, - chroot=True, runat=None, env=None, binary=False, - sizelimit=None, pred=None, subdir=None, - changes=False, foreground=False, tags=[]): -@@ -1849,7 +1871,7 @@ class Plugin(object): - self._log_debug("added string ...'%s' as '%s'" % (summary, filename)) - - def _collect_cmd_output(self, cmd, suggest_filename=None, -- root_symlink=False, timeout=cmd_timeout, -+ root_symlink=False, timeout=None, - stderr=True, chroot=True, runat=None, env=None, - binary=False, sizelimit=None, subdir=None, - changes=False, foreground=False, tags=[]): -@@ -1883,6 +1905,8 @@ class Plugin(object): - if self._timeout_hit: - return - -+ if timeout is None: -+ timeout = self.cmdtimeout - _tags = [] - - if isinstance(tags, str): -@@ -1975,7 +1999,7 @@ class Plugin(object): - return result - - def collect_cmd_output(self, cmd, suggest_filename=None, -- root_symlink=False, timeout=cmd_timeout, -+ root_symlink=False, timeout=None, - stderr=True, chroot=True, runat=None, env=None, - binary=False, sizelimit=None, pred=None, - subdir=None, tags=[]): -@@ -2044,7 +2068,7 @@ class Plugin(object): - tags=tags - ) - -- def exec_cmd(self, cmd, timeout=cmd_timeout, stderr=True, chroot=True, -+ def exec_cmd(self, cmd, timeout=None, stderr=True, chroot=True, - runat=None, env=None, binary=False, pred=None, - foreground=False, container=False, quotecmd=False): - """Execute a command right now and return the output and status, but -@@ -2095,6 +2119,9 @@ class Plugin(object): - if not self.test_predicate(cmd=True, pred=pred): - return _default - -+ if timeout is None: -+ timeout = self.cmdtimeout -+ - if chroot or self.commons['cmdlineopts'].chroot == 'always': - root = self.sysroot - else: -@@ -2331,7 +2358,7 @@ class Plugin(object): - - def add_journal(self, units=None, boot=None, since=None, until=None, - lines=None, allfields=False, output=None, -- timeout=cmd_timeout, identifier=None, catalog=None, -+ timeout=None, identifier=None, catalog=None, - sizelimit=None, pred=None, tags=[]): - """Collect journald logs from one of more units. - --- -2.26.3 - diff --git a/sos-bz1967113-ds-mask-password-in-ldif.patch b/sos-bz1967113-ds-mask-password-in-ldif.patch deleted file mode 100644 index 48aa77a..0000000 --- a/sos-bz1967113-ds-mask-password-in-ldif.patch +++ /dev/null @@ -1,50 +0,0 @@ -From 153c0154050a111fd7e5bcf4a685f906a1dea737 Mon Sep 17 00:00:00 2001 -From: Jose Castillo -Date: Wed, 10 Mar 2021 15:33:50 +0100 -Subject: [PATCH] [ds] Mask password and encription keys in ldif files - -Both /etc/dirsrv/slapd*/dse.ldif{,.startOK} files contain -sensitive information : -- all the nsSymmetricKey entries : symmetric encryption key -- nsslapd-rootpw : the admin password's hash - -This patch masks these entries in the files we collect. - -Resolves: #2442 - -Signed-off-by: Jose Castillo -Signed-off-by: Jake Hunsaker ---- - sos/report/plugins/ds.py | 18 ++++++++++++++++++ - 1 file changed, 18 insertions(+) - -diff --git a/sos/report/plugins/ds.py b/sos/report/plugins/ds.py -index f4d68d6e..d467dc89 100644 ---- a/sos/report/plugins/ds.py -+++ b/sos/report/plugins/ds.py -@@ -74,4 +74,22 @@ class DirectoryServer(Plugin, RedHatPlugin): - - self.add_cmd_output("ls -l /var/lib/dirsrv/slapd-*/db/*") - -+ def postproc(self): -+ # Example for scrubbing rootpw hash -+ # -+ # nsslapd-rootpw: AAAAB3NzaC1yc2EAAAADAQABAAABAQDeXYA3juyPqaUuyfWV2HuIM -+ # v3gebb/5cvx9ehEAFF2yIKvsQN2EJGTV+hBM1DEOB4eyy/H11NqcNwm/2QsagDB3PVwYp -+ # 9VKN3BdhQjlhuoYKhLwgtYUMiGL8AX5g1qxjirIkTRJwjbXkSNuQaXig7wVjmvXnB2o7B -+ # zLtu99DiL1AizfVeZTYA+OVowYKYaXYljVmVKS+g3t29Obaom54ZLpfuoGMmyO64AJrWs -+ # -+ # to -+ # -+ # nsslapd-rootpw:******** -+ -+ regexppass = r"(nsslapd-rootpw(\s)*:(\s)*)(\S+)([\r\n]\s.*)*\n" -+ regexpkey = r"(nsSymmetricKey(\s)*::(\s)*)(\S+)([\r\n]\s.*)*\n" -+ repl = r"\1********\n" -+ self.do_path_regex_sub('/etc/dirsrv/*', regexppass, repl) -+ self.do_path_regex_sub('/etc/dirsrv/*', regexpkey, repl) -+ - # vim: set et ts=4 sw=4 : --- -2.26.3 - diff --git a/sos-bz1967114-gather-cups-browsed-logs.patch b/sos-bz1967114-gather-cups-browsed-logs.patch deleted file mode 100644 index 3e6c393..0000000 --- a/sos-bz1967114-gather-cups-browsed-logs.patch +++ /dev/null @@ -1,30 +0,0 @@ -From 0d56e43299009ffa91f665d85b5a08ba76da9c1f Mon Sep 17 00:00:00 2001 -From: Jose Castillo -Date: Wed, 17 Mar 2021 13:10:36 +0100 -Subject: [PATCH] [cups] Add gathering cups-browsed logs - -Gather logs from the service cups-browsed sent -to the journal. - -Resolves: #2452 - -Signed-off-by: Jose Castillo -Signed-off-by: Bryan Quigley ---- - sos/report/plugins/cups.py | 1 + - 1 file changed, 1 insertion(+) - -diff --git a/sos/report/plugins/cups.py b/sos/report/plugins/cups.py -index 29a903e8..ab7b6b70 100644 ---- a/sos/report/plugins/cups.py -+++ b/sos/report/plugins/cups.py -@@ -40,5 +40,6 @@ class Cups(Plugin, IndependentPlugin): - ]) - - self.add_journal(units="cups") -+ self.add_journal(units="cups-browsed") - - # vim: set et ts=4 sw=4 : --- -2.26.3 - diff --git a/sos-bz1967115-sssd-memcache-and-logs.patch b/sos-bz1967115-sssd-memcache-and-logs.patch deleted file mode 100644 index ebc7578..0000000 --- a/sos-bz1967115-sssd-memcache-and-logs.patch +++ /dev/null @@ -1,62 +0,0 @@ -From d03c2fa4439c87783293c922b2825cf86e8818bd Mon Sep 17 00:00:00 2001 -From: =?UTF-8?q?Pawe=C5=82=20Po=C5=82awski?= -Date: Fri, 12 Mar 2021 12:42:30 +0100 -Subject: [PATCH] [sssd] Enable collecting SSSD memory cache -MIME-Version: 1.0 -Content-Type: text/plain; charset=UTF-8 -Content-Transfer-Encoding: 8bit - -SSSD plugin by default collects only logs and configuration. -This patch enables collecting memory cache maintained -by SSSD daemon. Cache does not contain any client sensible -data so can be safely included in the sos-report. - -Resolves: #2444 - -Signed-off-by: Paweł Poławski -Signed-off-by: Jake Hunsaker ---- - sos/report/plugins/sssd.py | 8 ++++++-- - 1 file changed, 6 insertions(+), 2 deletions(-) - -diff --git a/sos/report/plugins/sssd.py b/sos/report/plugins/sssd.py -index 9469c41c..aeb68c4f 100644 ---- a/sos/report/plugins/sssd.py -+++ b/sos/report/plugins/sssd.py -@@ -10,6 +10,7 @@ - - from sos.report.plugins import (Plugin, RedHatPlugin, DebianPlugin, - UbuntuPlugin, SoSPredicate) -+from glob import glob - - - class Sssd(Plugin): -@@ -22,11 +23,22 @@ class Sssd(Plugin): - - def setup(self): - self.add_copy_spec([ -+ # main config file - "/etc/sssd/sssd.conf", -- "/var/log/sssd/*", -- "/var/lib/sss/pubconf/krb5.include.d/*", - # SSSD 1.14 -- "/etc/sssd/conf.d/*.conf" -+ "/etc/sssd/conf.d/*.conf", -+ # dynamic Kerberos configuration -+ "/var/lib/sss/pubconf/krb5.include.d/*" -+ ]) -+ -+ # add individual log files -+ self.add_copy_spec(glob("/var/log/sssd/*log*")) -+ -+ # add memory cache -+ self.add_copy_spec([ -+ "/var/lib/sss/mc/passwd", -+ "/var/lib/sss/mc/group", -+ "/var/lib/sss/mc/initgroups" - ]) - - # call sssctl commands only when sssd service is running, --- -2.26.3 - diff --git a/sos-bz1967116-ibmvNIC-dynamic-debugs.patch b/sos-bz1967116-ibmvNIC-dynamic-debugs.patch deleted file mode 100644 index 7bb7fd7..0000000 --- a/sos-bz1967116-ibmvNIC-dynamic-debugs.patch +++ /dev/null @@ -1,29 +0,0 @@ -From dddabb07a88d398ed7b8a878e95acfd968af6698 Mon Sep 17 00:00:00 2001 -From: Mamatha Inamdar -Date: Tue, 23 Mar 2021 17:58:30 +0530 -Subject: [PATCH] This patch is to update kernel plugin to collect - dynamic_debug log files for ibmvNIC - -Resolves: #2458 - -Signed-off-by: Mamatha Inamdar -Signed-off-by: Bryan Quigley ---- - sos/report/plugins/kernel.py | 1 + - 1 file changed, 1 insertion(+) - -diff --git a/sos/report/plugins/kernel.py b/sos/report/plugins/kernel.py -index febe2ad0..dd7b6939 100644 ---- a/sos/report/plugins/kernel.py -+++ b/sos/report/plugins/kernel.py -@@ -106,6 +106,7 @@ class Kernel(Plugin, IndependentPlugin): - "/proc/misc", - "/var/log/dmesg", - "/sys/fs/pstore", -+ "/sys/kernel/debug/dynamic_debug/control", - clocksource_path + "available_clocksource", - clocksource_path + "current_clocksource" - ]) --- -2.26.3 - diff --git a/sos-bz1967117-pulpcore-plugin.patch b/sos-bz1967117-pulpcore-plugin.patch deleted file mode 100644 index e60a494..0000000 --- a/sos-bz1967117-pulpcore-plugin.patch +++ /dev/null @@ -1,147 +0,0 @@ -From 808d9f35ac504a58c337ffed14b39119a591808f Mon Sep 17 00:00:00 2001 -From: Pavel Moravec -Date: Tue, 27 Apr 2021 22:16:08 +0200 -Subject: [PATCH] [pulpcore] add plugin for pulp-3 - -Pulp-3 / pulpcore as a revolution from pulp-2 needs a separate -plugin, since both plugins have nothing in common and there might -be deployments where is active both pulp-2 and pulp-3. - -Resolves: #2278 - -Signed-off-by: Pavel Moravec -Signed-off-by: Jake Hunsaker ---- - sos/report/plugins/pulpcore.py | 120 +++++++++++++++++++++++++++++++++ - 1 file changed, 120 insertions(+) - create mode 100644 sos/report/plugins/pulpcore.py - -diff --git a/sos/report/plugins/pulpcore.py b/sos/report/plugins/pulpcore.py -new file mode 100644 -index 00000000..20403814 ---- /dev/null -+++ b/sos/report/plugins/pulpcore.py -@@ -0,0 +1,120 @@ -+# Copyright (C) 2021 Red Hat, Inc., Pavel Moravec -+ -+# This file is part of the sos project: https://github.com/sosreport/sos -+# -+# This copyrighted material is made available to anyone wishing to use, -+# modify, copy, or redistribute it subject to the terms and conditions of -+# version 2 of the GNU General Public License. -+# -+# See the LICENSE file in the source distribution for further information. -+ -+from sos.report.plugins import Plugin, IndependentPlugin -+from pipes import quote -+from re import match -+ -+ -+class PulpCore(Plugin, IndependentPlugin): -+ -+ short_desc = 'Pulp-3 aka pulpcore' -+ -+ plugin_name = "pulpcore" -+ commands = ("pulpcore-manager",) -+ files = ("/etc/pulp/settings.py",) -+ option_list = [ -+ ('task-days', 'days of tasks history', 'fast', 7) -+ ] -+ -+ def parse_settings_config(self): -+ databases_scope = False -+ self.dbhost = "localhost" -+ self.dbport = 5432 -+ self.dbpasswd = "" -+ # TODO: read also redis config (we dont expect much customisations) -+ # TODO: read also db user (pulp) and database name (pulpcore) -+ self.staticroot = "/var/lib/pulp/assets" -+ self.uploaddir = "/var/lib/pulp/media/upload" -+ -+ def separate_value(line, sep=':'): -+ # an auxiliary method to parse values from lines like: -+ # 'HOST': 'localhost', -+ val = line.split(sep)[1].lstrip().rstrip(',') -+ if (val.startswith('"') and val.endswith('"')) or \ -+ (val.startswith('\'') and val.endswith('\'')): -+ val = val[1:-1] -+ return val -+ -+ try: -+ for line in open("/etc/pulp/settings.py").read().splitlines(): -+ # skip empty lines and lines with comments -+ if not line or line[0] == '#': -+ continue -+ if line.startswith("DATABASES"): -+ databases_scope = True -+ continue -+ # example HOST line to parse: -+ # 'HOST': 'localhost', -+ if databases_scope and match(r"\s+'HOST'\s*:\s+\S+", line): -+ self.dbhost = separate_value(line) -+ if databases_scope and match(r"\s+'PORT'\s*:\s+\S+", line): -+ self.dbport = separate_value(line) -+ if databases_scope and match(r"\s+'PASSWORD'\s*:\s+\S+", line): -+ self.dbpasswd = separate_value(line) -+ # if line contains closing '}' database_scope end -+ if databases_scope and '}' in line: -+ databases_scope = False -+ if line.startswith("STATIC_ROOT = "): -+ self.staticroot = separate_value(line, sep='=') -+ if line.startswith("CHUNKED_UPLOAD_DIR = "): -+ self.uploaddir = separate_value(line, sep='=') -+ except IOError: -+ # fallback when the cfg file is not accessible -+ pass -+ # set the password to os.environ when calling psql commands to prevent -+ # printing it in sos logs -+ # we can't set os.environ directly now: other plugins can overwrite it -+ self.env = {"PGPASSWORD": self.dbpasswd} -+ -+ def setup(self): -+ self.parse_settings_config() -+ -+ self.add_copy_spec("/etc/pulp/settings.py") -+ -+ self.add_cmd_output("rq info -u redis://localhost:6379/8", -+ env={"LC_ALL": "en_US.UTF-8"}, -+ suggest_filename="rq_info") -+ self.add_cmd_output("curl -ks https://localhost/pulp/api/v3/status/", -+ suggest_filename="pulp_status") -+ dynaconf_env = {"LC_ALL": "en_US.UTF-8", -+ "PULP_SETTINGS": "/etc/pulp/settings.py", -+ "DJANGO_SETTINGS_MODULE": "pulpcore.app.settings"} -+ self.add_cmd_output("dynaconf list", env=dynaconf_env) -+ for _dir in [self.staticroot, self.uploaddir]: -+ self.add_cmd_output("ls -l %s" % _dir) -+ -+ task_days = self.get_option('task-days') -+ for table in ['core_task', 'core_taskgroup', -+ 'core_reservedresourcerecord', -+ 'core_taskreservedresourcerecord', -+ 'core_groupprogressreport', 'core_progressreport']: -+ _query = "select * from %s where pulp_last_updated > NOW() - " \ -+ "interval '%s days' order by pulp_last_updated" % \ -+ (table, task_days) -+ _cmd = "psql -h %s -p %s -U pulp -d pulpcore -c %s" % \ -+ (self.dbhost, self.dbport, quote(_query)) -+ self.add_cmd_output(_cmd, env=self.env, suggest_filename=table) -+ -+ def postproc(self): -+ # TODO obfuscate from /etc/pulp/settings.py : -+ # SECRET_KEY = "eKfeDkTnvss7p5WFqYdGPWxXfHnsbDBx" -+ # 'PASSWORD': 'tGrag2DmtLqKLTWTQ6U68f6MAhbqZVQj', -+ self.do_path_regex_sub( -+ "/etc/pulp/settings.py", -+ r"(SECRET_KEY\s*=\s*)(.*)", -+ r"\1********") -+ self.do_path_regex_sub( -+ "/etc/pulp/settings.py", -+ r"(PASSWORD\S*\s*:\s*)(.*)", -+ r"\1********") -+ -+ -+# vim: set et ts=4 sw=4 : --- -2.26.3 - diff --git a/sos-bz1967118-saphana-traceback.patch b/sos-bz1967118-saphana-traceback.patch deleted file mode 100644 index 4b784dc..0000000 --- a/sos-bz1967118-saphana-traceback.patch +++ /dev/null @@ -1,30 +0,0 @@ -From c998ea8c1c950586f91fc9728ee66590740968a5 Mon Sep 17 00:00:00 2001 -From: Pavel Moravec -Date: Tue, 11 May 2021 15:59:40 +0200 -Subject: [PATCH] [saphana] remove redundant unused argument of get_inst_info - -get_inst_info does not use and isnt called with 'prefix' argument - -Resolves: #2535 - -Signed-off-by: Pavel Moravec ---- - sos/report/plugins/saphana.py | 2 +- - 1 file changed, 1 insertion(+), 1 deletion(-) - -diff --git a/sos/report/plugins/saphana.py b/sos/report/plugins/saphana.py -index 82c497b4..00e84b59 100644 ---- a/sos/report/plugins/saphana.py -+++ b/sos/report/plugins/saphana.py -@@ -51,7 +51,7 @@ class saphana(Plugin, RedHatPlugin): - inst = inst.strip()[-2:] - self.get_inst_info(sid, sidadm, inst) - -- def get_inst_info(self, prefix, sid, sidadm, inst): -+ def get_inst_info(self, sid, sidadm, inst): - proc_cmd = 'su - %s -c "sapcontrol -nr %s -function GetProcessList"' - status_fname = "%s_%s_status" % (sid, inst) - self.add_cmd_output( --- -2.26.3 - diff --git a/sos-bz1967119-collect-nstat.patch b/sos-bz1967119-collect-nstat.patch deleted file mode 100644 index 75b7d29..0000000 --- a/sos-bz1967119-collect-nstat.patch +++ /dev/null @@ -1,36 +0,0 @@ -From 575ddeddf2f6e1d6a639922f9ccc51c7e46fbe12 Mon Sep 17 00:00:00 2001 -From: Seiichi Ikarashi -Date: Fri, 14 May 2021 09:49:33 +0900 -Subject: [PATCH] [networking] Add nstat command support - -As netstat command is being deprecated, -we need nstat as an alternative to "netstat -s". - -Signed-off-by: Seiichi Ikarashi ---- - sos/report/plugins/networking.py | 2 ++ - 1 file changed, 2 insertions(+) - -diff --git a/sos/report/plugins/networking.py b/sos/report/plugins/networking.py -index 8b4614bb..acfa027f 100644 ---- a/sos/report/plugins/networking.py -+++ b/sos/report/plugins/networking.py -@@ -87,6 +87,7 @@ class Networking(Plugin): - root_symlink="netstat") - - self.add_cmd_output([ -+ "nstat -zas", - "netstat -s", - "netstat %s -agn" % self.ns_wide, - "ip route show table all", -@@ -198,6 +199,7 @@ class Networking(Plugin): - ns_cmd_prefix + "netstat %s -neopa" % self.ns_wide, - ns_cmd_prefix + "netstat -s", - ns_cmd_prefix + "netstat %s -agn" % self.ns_wide, -+ ns_cmd_prefix + "nstat -zas", - ]) - - ss_cmd = ns_cmd_prefix + "ss -peaonmi" --- -2.26.3 - diff --git a/sos-bz1967120-snapper-plugin-and-allocation-failures.patch b/sos-bz1967120-snapper-plugin-and-allocation-failures.patch deleted file mode 100644 index e33a89e..0000000 --- a/sos-bz1967120-snapper-plugin-and-allocation-failures.patch +++ /dev/null @@ -1,121 +0,0 @@ -From 60105e0705f3483b9a3e8e98dafd6f0e1e277ab7 Mon Sep 17 00:00:00 2001 -From: Mamatha Inamdar -Date: Mon, 19 Apr 2021 16:55:52 +0530 -Subject: [PATCH 1/3] [block]:Patch to update block pluging to collect disk - info - -This patch is to update block plugin to collect -state of sda - -Resolves: #2504 - -Signed-off-by: Mamatha Inamdar ---- - sos/report/plugins/block.py | 1 + - 1 file changed, 1 insertion(+) - -diff --git a/sos/report/plugins/block.py b/sos/report/plugins/block.py -index f93b3231..c959d667 100644 ---- a/sos/report/plugins/block.py -+++ b/sos/report/plugins/block.py -@@ -38,7 +38,8 @@ class Block(Plugin, IndependentPlugin): - "/run/blkid/blkid.tab", - "/proc/partitions", - "/proc/diskstats", -- "/sys/block/*/queue/" -+ "/sys/block/*/queue/", -+ "/sys/block/sd*/device/state", - ]) - - cmds = [ --- -2.26.3 - - -From c6e0fe5cebd0d9581950db75fa2d234713b7e15a Mon Sep 17 00:00:00 2001 -From: Mamatha Inamdar -Date: Mon, 26 Apr 2021 23:09:19 +0530 -Subject: [PATCH 2/3] [snapper]:Ptach to update snapper plugin to collect - snapper info - -This patch is to Introduce snapper plugin to collect -/usr/lib/snapper/ information to check executable -permission for installation-helper command - -Resolves: #2504 - -Signed-off-by: Mamatha Inamdar ---- - sos/report/plugins/snapper.py | 27 +++++++++++++++++++++++++++ - 1 file changed, 27 insertions(+) - create mode 100644 sos/report/plugins/snapper.py - -diff --git a/sos/report/plugins/snapper.py b/sos/report/plugins/snapper.py -new file mode 100644 -index 00000000..9ef5fec2 ---- /dev/null -+++ b/sos/report/plugins/snapper.py -@@ -0,0 +1,27 @@ -+# This file is part of the sos project: https://github.com/sosreport/sos -+# -+# This copyrighted material is made available to anyone wishing to use, -+# modify, copy, or redistribute it subject to the terms and conditions of -+# version 2 of the GNU General Public License. -+# -+# See the LICENSE file in the source distribution for further information. -+ -+from sos.report.plugins import Plugin, IndependentPlugin -+ -+ -+class Snapper(Plugin, IndependentPlugin): -+ -+ short_desc = 'System snapper' -+ -+ plugin_name = 'snapper' -+ commands = ("snapper",) -+ -+ def setup(self): -+ -+ self.add_cmd_output([ -+ "ls -la /usr/lib/snapper/", -+ "snapper --version", -+ "snapper list" -+ ]) -+ -+# vim: set et ts=4 sw=4 : --- -2.26.3 - - -From 61ff5ce165e654a02fe80b9de5ec8e49ed808ec9 Mon Sep 17 00:00:00 2001 -From: Mamatha Inamdar -Date: Mon, 19 Apr 2021 17:49:08 +0530 -Subject: [PATCH 3/3] [kernel]:Patch to update kernel plugin to collect debug - info - -This patch is to update kernel plugin to collect -page allocation failure info - -Resolves: #2504 - -Signed-off-by: Mamatha Inamdar ---- - sos/report/plugins/kernel.py | 2 ++ - 1 file changed, 2 insertions(+) - -diff --git a/sos/report/plugins/kernel.py b/sos/report/plugins/kernel.py -index dd7b6939..9d53ca03 100644 ---- a/sos/report/plugins/kernel.py -+++ b/sos/report/plugins/kernel.py -@@ -107,6 +107,8 @@ class Kernel(Plugin, IndependentPlugin): - "/var/log/dmesg", - "/sys/fs/pstore", - "/sys/kernel/debug/dynamic_debug/control", -+ "/sys/kernel/debug/extfrag/unusable_index", -+ "/sys/kernel/debug/extfrag/extfrag_index", - clocksource_path + "available_clocksource", - clocksource_path + "current_clocksource" - ]) --- -2.26.3 - diff --git a/sos-bz1967718-sssd-common.patch b/sos-bz1967718-sssd-common.patch deleted file mode 100644 index 9937972..0000000 --- a/sos-bz1967718-sssd-common.patch +++ /dev/null @@ -1,36 +0,0 @@ -From 630dfbee936050698d33b59abd1e243c44e50af8 Mon Sep 17 00:00:00 2001 -From: Jan Jansky -Date: Thu, 3 Jun 2021 15:04:57 +0200 -Subject: [PATCH] [sssd] sssd plugin when sssd-common - -We have reports that sssd logs are not -collected, when we investigated -we found associate wants to collect -sssd related logs also when only -sssd-common package is installed. - -We got this confirmed by sbr-idm. - -Resolves: #2571 - -Signed-off-by: Jan Jansky ---- - sos/report/plugins/sssd.py | 2 +- - 1 file changed, 1 insertion(+), 1 deletion(-) - -diff --git a/sos/report/plugins/sssd.py b/sos/report/plugins/sssd.py -index 17933935..6f98e90c 100644 ---- a/sos/report/plugins/sssd.py -+++ b/sos/report/plugins/sssd.py -@@ -19,7 +19,7 @@ class Sssd(Plugin): - - plugin_name = "sssd" - profiles = ('services', 'security', 'identity') -- packages = ('sssd',) -+ packages = ('sssd', 'sssd-common') - - def setup(self): - self.add_copy_spec([ --- -2.26.3 - diff --git a/sos-bz1985976-enhance-tc-hw-offload.patch b/sos-bz1985976-enhance-tc-hw-offload.patch deleted file mode 100644 index ed1088b..0000000 --- a/sos-bz1985976-enhance-tc-hw-offload.patch +++ /dev/null @@ -1,32 +0,0 @@ -From bbb7f8bf522960a8ca7625f539e9e5d109abb704 Mon Sep 17 00:00:00 2001 -From: Pavel Moravec -Date: Wed, 19 May 2021 08:31:45 +0200 -Subject: [PATCH] [networking] collect also tc filter show ingress - -Both "tc -s filter show dev %eth [|ingress]" commands required as -they provide different output. - -Resolves: #2550 - -Signed-off-by: Pavel Moravec ---- - sos/report/plugins/networking.py | 3 ++- - 1 file changed, 2 insertions(+), 1 deletion(-) - -diff --git a/sos/report/plugins/networking.py b/sos/report/plugins/networking.py -index acfa027f..35646268 100644 ---- a/sos/report/plugins/networking.py -+++ b/sos/report/plugins/networking.py -@@ -156,7 +156,8 @@ class Networking(Plugin): - "ethtool --phy-statistics " + eth, - "ethtool --show-priv-flags " + eth, - "ethtool --show-eee " + eth, -- "tc -s filter show dev " + eth -+ "tc -s filter show dev " + eth, -+ "tc -s filter show dev " + eth + " ingress", - ], tags=eth) - - # skip EEPROM collection by default, as it might hang or --- -2.26.3 - diff --git a/sos-bz1985982-obfuscate-fqdn-from-dnf-log.patch b/sos-bz1985982-obfuscate-fqdn-from-dnf-log.patch deleted file mode 100644 index 07e005b..0000000 --- a/sos-bz1985982-obfuscate-fqdn-from-dnf-log.patch +++ /dev/null @@ -1,78 +0,0 @@ -From b27140a9126ea82efb517d60bf1b8455aaf4f5a6 Mon Sep 17 00:00:00 2001 -From: Jake Hunsaker -Date: Fri, 26 Mar 2021 11:12:33 -0400 -Subject: [PATCH] [cleaner] Only skip packaging-based files for the IP parser - -Files primarily containing package information, e.g. `installed-rpms` or -`installed-debs`, were previously being skipped by all parsers. In -reality, we only need to skip these for the IP parser due to the fact -that version numbers often generate a match for IP address regexes. - -This will also fix a problem where if a system was the build host for -certain packages, the hostname would remain in these files as the -hostname parser was previously not checking these files. - -Closes: #2400 -Resolves: #2464 - -Signed-off-by: Jake Hunsaker ---- - sos/cleaner/obfuscation_archive.py | 10 ---------- - sos/cleaner/parsers/ip_parser.py | 16 ++++++++++++++++ - 2 files changed, 16 insertions(+), 10 deletions(-) - -diff --git a/sos/cleaner/obfuscation_archive.py b/sos/cleaner/obfuscation_archive.py -index 981cc05f..84ca30cd 100644 ---- a/sos/cleaner/obfuscation_archive.py -+++ b/sos/cleaner/obfuscation_archive.py -@@ -59,20 +59,10 @@ class SoSObfuscationArchive(): - Returns: list of files and file regexes - """ - return [ -- 'installed-debs', -- 'installed-rpms', -- 'sos_commands/dpkg', -- 'sos_commands/python/pip_list', -- 'sos_commands/rpm', -- 'sos_commands/yum/.*list.*', -- 'sos_commands/snappy/snap_list_--all', -- 'sos_commands/snappy/snap_--version', -- 'sos_commands/vulkan/vulkaninfo', - 'sys/firmware', - 'sys/fs', - 'sys/kernel/debug', - 'sys/module', -- 'var/log/.*dnf.*', - r'.*\.tar$', # TODO: support archive unpacking - # Be explicit with these tar matches to avoid matching commands - r'.*\.tar\.xz', -diff --git a/sos/cleaner/parsers/ip_parser.py b/sos/cleaner/parsers/ip_parser.py -index 3ea7f865..08d1cd05 100644 ---- a/sos/cleaner/parsers/ip_parser.py -+++ b/sos/cleaner/parsers/ip_parser.py -@@ -24,6 +24,22 @@ class SoSIPParser(SoSCleanerParser): - # don't match package versions recorded in journals - r'.*dnf\[.*\]:' - ] -+ -+ skip_files = [ -+ # skip these as version numbers will frequently look like IP addresses -+ # when using regex matching -+ 'installed-debs', -+ 'installed-rpms', -+ 'sos_commands/dpkg', -+ 'sos_commands/python/pip_list', -+ 'sos_commands/rpm', -+ 'sos_commands/yum/.*list.*', -+ 'sos_commands/snappy/snap_list_--all', -+ 'sos_commands/snappy/snap_--version', -+ 'sos_commands/vulkan/vulkaninfo', -+ 'var/log/.*dnf.*' -+ ] -+ - map_file_key = 'ip_map' - prep_map_file = 'sos_commands/networking/ip_-o_addr' - --- -2.26.3 - diff --git a/sos-bz1985983-ocp-cluster-cleaner.patch b/sos-bz1985983-ocp-cluster-cleaner.patch deleted file mode 100644 index 205152f..0000000 --- a/sos-bz1985983-ocp-cluster-cleaner.patch +++ /dev/null @@ -1,2156 +0,0 @@ -From 29afda6e4ff90385d34bc61315542e7cb4baaf8d Mon Sep 17 00:00:00 2001 -From: Jake Hunsaker -Date: Fri, 9 Apr 2021 11:32:14 -0400 -Subject: [PATCH] [cleaner] Do not break iteration of parse_string_for_keys on - first match - -Previously, `parse_string_for_keys()`, called by `obfuscate_string()` -for non-regex based obfuscations, would return on the first match in the -string found for each parser. - -Instead, continue iterating over all items in each parser's dataset -before returning the (now fully) obfuscated string. - -Resolves: #2480 - -Signed-off-by: Jake Hunsaker ---- - sos/cleaner/parsers/__init__.py | 2 +- - 1 file changed, 1 insertion(+), 1 deletion(-) - -diff --git a/sos/cleaner/parsers/__init__.py b/sos/cleaner/parsers/__init__.py -index dd0451df..c77300aa 100644 ---- a/sos/cleaner/parsers/__init__.py -+++ b/sos/cleaner/parsers/__init__.py -@@ -104,7 +104,7 @@ class SoSCleanerParser(): - """ - for key, val in self.mapping.dataset.items(): - if key in string_data: -- return string_data.replace(key, val) -+ string_data = string_data.replace(key, val) - return string_data - - def get_map_contents(self): --- -2.26.3 - -From 52e6b2ae17e128f17a84ee83b7718c2901bcd5bd Mon Sep 17 00:00:00 2001 -From: Jake Hunsaker -Date: Wed, 12 May 2021 12:39:48 -0400 -Subject: [PATCH] [collect] Add options to provide registry auth for pulling - images - -Adds options that allow a user to specify registry authentication, -either via username/password or an authfile, to allow pulling an image -that exists on a non-public registry. - -If a username/password is provided, that will be used. If not, we will -attempt to use an authfile - either provided by the user or by a cluster -profile. - -Also adds an option to forcibly pull a new(er) version of the specified -image, to alleviate conditions where a too-old version of the image -already exists on the host. - -Closes: #2534 - -Signed-off-by: Jake Hunsaker ---- - man/en/sos-collect.1 | 30 +++++++++++++++++++++++ - sos/collector/__init__.py | 17 +++++++++++++ - sos/collector/sosnode.py | 40 +++++++++++++++++++++++++++---- - sos/policies/distros/__init__.py | 16 ++++++++++++- - sos/policies/distros/redhat.py | 25 ++++++++++++------- - sos/policies/runtimes/__init__.py | 25 +++++++++++++++++++ - 6 files changed, 140 insertions(+), 13 deletions(-) - -diff --git a/man/en/sos-collect.1 b/man/en/sos-collect.1 -index 286bfe71..cdbc3257 100644 ---- a/man/en/sos-collect.1 -+++ b/man/en/sos-collect.1 -@@ -26,6 +26,11 @@ sos collect \- Collect sosreports from multiple (cluster) nodes - [\-\-no\-pkg\-check] - [\-\-no\-local] - [\-\-master MASTER] -+ [\-\-image IMAGE] -+ [\-\-force-pull-image] -+ [\-\-registry-user USER] -+ [\-\-registry-password PASSWORD] -+ [\-\-registry-authfile FILE] - [\-o ONLY_PLUGINS] - [\-p SSH_PORT] - [\-\-password] -@@ -245,6 +250,31 @@ Specify a master node for the cluster. - If provided, then sos collect will check the master node, not localhost, for determining - the type of cluster in use. - .TP -+\fB\-\-image IMAGE\fR -+Specify an image to use for the temporary container created for collections on -+containerized host, if you do not want to use the default image specifed by the -+host's policy. Note that this should include the registry. -+.TP -+\fB\-\-force-pull-image\fR -+Use this option to force the container runtime to pull the specified image (even -+if it is the policy default image) even if the image already exists on the host. -+This may be useful to update an older container image on containerized hosts. -+.TP -+\fB\-\-registry-user USER\fR -+Specify the username to authenticate to the registry with in order to pull the container -+image -+.TP -+\fB\-\-registry-password PASSWORD\fR -+Specify the password to authenticate to the registry with in order to pull the container -+image. If no password is required, leave this blank. -+.TP -+\fB\-\-registry-authfile FILE\fR -+Specify the filename to use for providing authentication credentials to the registry -+to pull the container image. -+ -+Note that this file must exist on the node(s) performing the pull operations, not the -+node from which \fBsos collect\fR was run. -+.TP - \fB\-o\fR ONLY_PLUGINS, \fB\-\-only\-plugins\fR ONLY_PLUGINS - Sosreport option. Run ONLY the plugins listed. - -diff --git a/sos/collector/__init__.py b/sos/collector/__init__.py -index 1c742cf5..0624caad 100644 ---- a/sos/collector/__init__.py -+++ b/sos/collector/__init__.py -@@ -63,6 +63,7 @@ class SoSCollector(SoSComponent): - 'encrypt_pass': '', - 'group': None, - 'image': '', -+ 'force_pull_image': False, - 'jobs': 4, - 'keywords': [], - 'keyword_file': None, -@@ -84,6 +85,9 @@ class SoSCollector(SoSComponent): - 'plugin_timeout': None, - 'cmd_timeout': None, - 'preset': '', -+ 'registry_user': None, -+ 'registry_password': None, -+ 'registry_authfile': None, - 'save_group': '', - 'since': '', - 'skip_commands': [], -@@ -319,6 +323,19 @@ class SoSCollector(SoSComponent): - collect_grp.add_argument('--image', - help=('Specify the container image to use for' - ' containerized hosts.')) -+ collect_grp.add_argument('--force-pull-image', '--pull', default=False, -+ action='store_true', -+ help='Force pull the container image even if ' -+ 'it already exists on the host') -+ collect_grp.add_argument('--registry-user', default=None, -+ help='Username to authenticate to the ' -+ 'registry with for pulling an image') -+ collect_grp.add_argument('--registry-password', default=None, -+ help='Password to authenticate to the ' -+ 'registry with for pulling an image') -+ collect_grp.add_argument('--registry-authfile', default=None, -+ help='Use this authfile to provide registry ' -+ 'authentication when pulling an image') - collect_grp.add_argument('-i', '--ssh-key', help='Specify an ssh key') - collect_grp.add_argument('-j', '--jobs', default=4, type=int, - help='Number of concurrent nodes to collect') -diff --git a/sos/collector/sosnode.py b/sos/collector/sosnode.py -index 48693342..d1c11824 100644 ---- a/sos/collector/sosnode.py -+++ b/sos/collector/sosnode.py -@@ -134,9 +134,27 @@ class SosNode(): - """If the host is containerized, create the container we'll be using - """ - if self.host.containerized: -- res = self.run_command(self.host.create_sos_container(), -- need_root=True) -- if res['status'] in [0, 125]: # 125 means container exists -+ cmd = self.host.create_sos_container( -+ image=self.opts.image, -+ auth=self.get_container_auth(), -+ force_pull=self.opts.force_pull_image -+ ) -+ res = self.run_command(cmd, need_root=True) -+ if res['status'] in [0, 125]: -+ if res['status'] == 125: -+ if 'unable to retrieve auth token' in res['stdout']: -+ self.log_error( -+ "Could not pull image. Provide either a username " -+ "and password or authfile" -+ ) -+ raise Exception -+ elif 'unknown: Not found' in res['stdout']: -+ self.log_error('Specified image not found on registry') -+ raise Exception -+ # 'name exists' with code 125 means the container was -+ # created successfully, so ignore it. -+ # initial creations leads to an exited container, restarting it -+ # here will keep it alive for us to exec through - ret = self.run_command(self.host.restart_sos_container(), - need_root=True) - if ret['status'] == 0: -@@ -152,6 +170,20 @@ class SosNode(): - % res['stdout']) - raise Exception - -+ def get_container_auth(self): -+ """Determine what the auth string should be to pull the image used to -+ deploy our temporary container -+ """ -+ if self.opts.registry_user: -+ return self.host.runtimes['default'].fmt_registry_credentials( -+ self.opts.registry_user, -+ self.opts.registry_password -+ ) -+ else: -+ return self.host.runtimes['default'].fmt_registry_authfile( -+ self.opts.registry_authfile or self.host.container_authfile -+ ) -+ - def file_exists(self, fname): - """Checks for the presence of fname on the remote node""" - if not self.local: -@@ -343,7 +375,7 @@ class SosNode(): - % self.commons['policy'].distro) - return self.commons['policy'] - host = load(cache={}, sysroot=self.opts.sysroot, init=InitSystem(), -- probe_runtime=False, remote_exec=self.ssh_cmd, -+ probe_runtime=True, remote_exec=self.ssh_cmd, - remote_check=self.read_file('/etc/os-release')) - if host: - self.log_info("loaded policy %s for host" % host.distro) -diff --git a/sos/policies/distros/__init__.py b/sos/policies/distros/__init__.py -index 9fe31513..f5b9fd5b 100644 ---- a/sos/policies/distros/__init__.py -+++ b/sos/policies/distros/__init__.py -@@ -62,6 +62,7 @@ class LinuxPolicy(Policy): - sos_bin_path = '/usr/bin' - sos_container_name = 'sos-collector-tmp' - container_version_command = None -+ container_authfile = None - - def __init__(self, sysroot=None, init=None, probe_runtime=True): - super(LinuxPolicy, self).__init__(sysroot=sysroot, -@@ -626,13 +627,26 @@ class LinuxPolicy(Policy): - """ - return '' - -- def create_sos_container(self): -+ def create_sos_container(self, image=None, auth=None, force_pull=False): - """Returns the command that will create the container that will be - used for running commands inside a container on hosts that require it. - - This will use the container runtime defined for the host type to - launch a container. From there, we use the defined runtime to exec into - the container's namespace. -+ -+ :param image: The name of the image if not using the policy default -+ :type image: ``str`` or ``None`` -+ -+ :param auth: The auth string required by the runtime to pull an -+ image from the registry -+ :type auth: ``str`` or ``None`` -+ -+ :param force_pull: Should the runtime forcibly pull the image -+ :type force_pull: ``bool`` -+ -+ :returns: The command to execute to launch the temp container -+ :rtype: ``str`` - """ - return '' - -diff --git a/sos/policies/distros/redhat.py b/sos/policies/distros/redhat.py -index 241d3f13..20afbcc4 100644 ---- a/sos/policies/distros/redhat.py -+++ b/sos/policies/distros/redhat.py -@@ -452,15 +452,19 @@ support representative. - - return self.find_preset(ATOMIC) - -- def create_sos_container(self): -+ def create_sos_container(self, image=None, auth=None, force_pull=False): - _cmd = ("{runtime} run -di --name {name} --privileged --ipc=host" - " --net=host --pid=host -e HOST=/host -e NAME={name} -e " -- "IMAGE={image} -v /run:/run -v /var/log:/var/log -v " -+ "IMAGE={image} {pull} -v /run:/run -v /var/log:/var/log -v " - "/etc/machine-id:/etc/machine-id -v " -- "/etc/localtime:/etc/localtime -v /:/host {image}") -+ "/etc/localtime:/etc/localtime -v /:/host {auth} {image}") -+ _image = image or self.container_image -+ _pull = '--pull=always' if force_pull else '' - return _cmd.format(runtime=self.container_runtime, - name=self.sos_container_name, -- image=self.container_image) -+ image=_image, -+ pull=_pull, -+ auth=auth or '') - - def set_cleanup_cmd(self): - return 'docker rm --force sos-collector-tmp' -@@ -482,6 +486,7 @@ support representative. - container_image = 'registry.redhat.io/rhel8/support-tools' - sos_path_strip = '/host' - container_version_command = 'rpm -q sos' -+ container_authfile = '/var/lib/kubelet/config.json' - - def __init__(self, sysroot=None, init=None, probe_runtime=True, - remote_exec=None): -@@ -511,15 +516,19 @@ support representative. - # RH OCP environments. - return self.find_preset(RHOCP) - -- def create_sos_container(self): -+ def create_sos_container(self, image=None, auth=None, force_pull=False): - _cmd = ("{runtime} run -di --name {name} --privileged --ipc=host" - " --net=host --pid=host -e HOST=/host -e NAME={name} -e " -- "IMAGE={image} -v /run:/run -v /var/log:/var/log -v " -+ "IMAGE={image} {pull} -v /run:/run -v /var/log:/var/log -v " - "/etc/machine-id:/etc/machine-id -v " -- "/etc/localtime:/etc/localtime -v /:/host {image}") -+ "/etc/localtime:/etc/localtime -v /:/host {auth} {image}") -+ _image = image or self.container_image -+ _pull = '--pull=always' if force_pull else '' - return _cmd.format(runtime=self.container_runtime, - name=self.sos_container_name, -- image=self.container_image) -+ image=_image, -+ pull=_pull, -+ auth=auth or '') - - def set_cleanup_cmd(self): - return 'podman rm --force %s' % self.sos_container_name -diff --git a/sos/policies/runtimes/__init__.py b/sos/policies/runtimes/__init__.py -index 1a61b644..f28d6a1d 100644 ---- a/sos/policies/runtimes/__init__.py -+++ b/sos/policies/runtimes/__init__.py -@@ -157,6 +157,31 @@ class ContainerRuntime(): - quoted_cmd = cmd - return "%s %s %s" % (self.run_cmd, container, quoted_cmd) - -+ def fmt_registry_credentials(self, username, password): -+ """Format a string to pass to the 'run' command of the runtime to -+ enable authorization for pulling the image during `sos collect`, if -+ needed using username and optional password creds -+ -+ :param username: The name of the registry user -+ :type username: ``str`` -+ -+ :param password: The password of the registry user -+ :type password: ``str`` or ``None`` -+ -+ :returns: The string to use to enable a run command to pull the image -+ :rtype: ``str`` -+ """ -+ return "--creds=%s%s" % (username, ':' + password if password else '') -+ -+ def fmt_registry_authfile(self, authfile): -+ """Format a string to pass to the 'run' command of the runtime to -+ enable authorization for pulling the image during `sos collect`, if -+ needed using an authfile. -+ """ -+ if authfile: -+ return "--authfile %s" % authfile -+ return '' -+ - def get_logs_command(self, container): - """Get the command string used to dump container logs from the - runtime --- -2.26.3 - -From 3cbbd7df6f0700609eeef3210d7388298b9e0c21 Mon Sep 17 00:00:00 2001 -From: Jake Hunsaker -Date: Wed, 12 May 2021 13:26:45 -0400 -Subject: [PATCH] [sosnode] Allow clusters to set options only for master nodes - -Adds a method the `Cluster` that allows a profile to set sos options -specifically for master nodes. - -Signed-off-by: Jake Hunsaker ---- - sos/collector/clusters/__init__.py | 21 +++++++++++++++++++++ - sos/collector/sosnode.py | 6 ++++++ - 2 files changed, 27 insertions(+) - -diff --git a/sos/collector/clusters/__init__.py b/sos/collector/clusters/__init__.py -index 5c002bae..bfa3aad3 100644 ---- a/sos/collector/clusters/__init__.py -+++ b/sos/collector/clusters/__init__.py -@@ -137,6 +137,27 @@ class Cluster(): - """ - self.cluster_ssh_key = key - -+ def set_master_options(self, node): -+ """If there is a need to set specific options in the sos command being -+ run on the cluster's master nodes, override this method in the cluster -+ profile and do that here. -+ -+ :param node: The master node -+ :type node: ``SoSNode`` -+ """ -+ pass -+ -+ def check_node_is_master(self, node): -+ """In the event there are multiple masters, or if the collect command -+ is being run from a system that is technically capable of enumerating -+ nodes but the cluster profiles needs to specify master-specific options -+ for other nodes, override this method in the cluster profile -+ -+ :param node: The node for the cluster to check -+ :type node: ``SoSNode`` -+ """ -+ return node.address == self.master.address -+ - def exec_master_cmd(self, cmd, need_root=False): - """Used to retrieve command output from a (master) node in a cluster - -diff --git a/sos/collector/sosnode.py b/sos/collector/sosnode.py -index d1c11824..62666635 100644 ---- a/sos/collector/sosnode.py -+++ b/sos/collector/sosnode.py -@@ -647,6 +647,10 @@ class SosNode(): - self.cluster.sos_plugin_options[opt]) - self.opts.plugin_options.append(option) - -+ # set master-only options -+ if self.cluster.check_node_is_master(self): -+ self.cluster.set_master_options(self) -+ - def finalize_sos_cmd(self): - """Use host facts and compare to the cluster type to modify the sos - command if needed""" -@@ -707,6 +711,8 @@ class SosNode(): - os.path.join(self.host.sos_bin_path, self.sos_bin) - ) - -+ self.update_cmd_from_cluster() -+ - if self.opts.only_plugins: - plugs = [o for o in self.opts.only_plugins - if self._plugin_exists(o)] --- -2.26.3 - -From cae9dd79a59107aa92db5f90aed356e093985bd9 Mon Sep 17 00:00:00 2001 -From: Jake Hunsaker -Date: Wed, 12 May 2021 16:06:29 -0400 -Subject: [PATCH] [sosnode] Don't fail on sos-less bastion nodes used for node - lists - -If the master node is determined to not have sos installed, that is not -necessarily a fatal error for scenarios where the 'master' node is only -being used to enumerate node lists and is not actually part of the -cluster. This can happen when a user is using a bastion node to -enumerate and connect to the cluster environment, or if the local host -is being used to enumerate nodes via cluster client tooling. - -Signed-off-by: Jake Hunsaker ---- - sos/collector/sosnode.py | 17 ++++++++++++----- - 1 file changed, 12 insertions(+), 5 deletions(-) - -diff --git a/sos/collector/sosnode.py b/sos/collector/sosnode.py -index 62666635..7e56483d 100644 ---- a/sos/collector/sosnode.py -+++ b/sos/collector/sosnode.py -@@ -287,13 +287,20 @@ class SosNode(): - # use the containerized policy's command - pkgs = self.run_command(self.host.container_version_command, - use_container=True, need_root=True) -- ver = pkgs['stdout'].strip().split('-')[1] -- if ver: -- self.sos_info['version'] = ver -- if 'version' in self.sos_info: -+ if pkgs['status'] == 0: -+ ver = pkgs['stdout'].strip().split('-')[1] -+ if ver: -+ self.sos_info['version'] = ver -+ else: -+ self.sos_info['version'] = None -+ if self.sos_info['version']: - self.log_info('sos version is %s' % self.sos_info['version']) - else: -- self.log_error('sos is not installed on this node') -+ if not self.address == self.opts.master: -+ # in the case where the 'master' enumerates nodes but is not -+ # intended for collection (bastions), don't worry about sos not -+ # being present -+ self.log_error('sos is not installed on this node') - self.connected = False - return False - cmd = 'sosreport -l' --- -2.26.3 - -From cc5abe563d855dea9ac25f56de2e493228b48bf7 Mon Sep 17 00:00:00 2001 -From: Jake Hunsaker -Date: Wed, 12 May 2021 18:26:09 -0400 -Subject: [PATCH] [sosnode] Mark sos commands as explicitly needing root for - containers - -Fixes an issue where the sos inspection commands were not properly -marked as needing to be run as root (either directly or via sudo) for -containerized hosts, which would lead to incorrect sos command -formatting. - -Mark those commands, and the final container removal command, as -explicitly needing root permissions. - -Signed-off-by: Jake Hunsaker ---- - sos/collector/sosnode.py | 6 +++--- - 1 file changed, 3 insertions(+), 3 deletions(-) - -diff --git a/sos/collector/sosnode.py b/sos/collector/sosnode.py -index 7e56483d..1fc03076 100644 ---- a/sos/collector/sosnode.py -+++ b/sos/collector/sosnode.py -@@ -304,7 +304,7 @@ class SosNode(): - self.connected = False - return False - cmd = 'sosreport -l' -- sosinfo = self.run_command(cmd, use_container=True) -+ sosinfo = self.run_command(cmd, use_container=True, need_root=True) - if sosinfo['status'] == 0: - self._load_sos_plugins(sosinfo['stdout']) - if self.check_sos_version('3.6'): -@@ -312,7 +312,7 @@ class SosNode(): - - def _load_sos_presets(self): - cmd = 'sosreport --list-presets' -- res = self.run_command(cmd, use_container=True) -+ res = self.run_command(cmd, use_container=True, need_root=True) - if res['status'] == 0: - for line in res['stdout'].splitlines(): - if line.strip().startswith('name:'): -@@ -996,7 +996,7 @@ class SosNode(): - self.remove_file(self.sos_path + '.md5') - cleanup = self.host.set_cleanup_cmd() - if cleanup: -- self.run_command(cleanup) -+ self.run_command(cleanup, need_root=True) - - def collect_extra_cmd(self, filenames): - """Collect the file created by a cluster outside of sos""" --- -2.26.3 - -From 55e77ad4c7e90ba14b10c5fdf18b65aa5d6b9cf8 Mon Sep 17 00:00:00 2001 -From: Jake Hunsaker -Date: Wed, 12 May 2021 18:55:31 -0400 -Subject: [PATCH] [ocp] Add cluster profile for OCP4 - -Removes the previous OCP cluster profile and replaces it with an updated -one for OCP4 which is entirely separated from the kubernetes profile. - -Resolves: #2544 - -Signed-off-by: Jake Hunsaker ---- - sos/collector/clusters/kubernetes.py | 8 -- - sos/collector/clusters/ocp.py | 109 +++++++++++++++++++++++++++ - 2 files changed, 109 insertions(+), 8 deletions(-) - create mode 100644 sos/collector/clusters/ocp.py - -diff --git a/sos/collector/clusters/kubernetes.py b/sos/collector/clusters/kubernetes.py -index 6a867e31..08fd9554 100644 ---- a/sos/collector/clusters/kubernetes.py -+++ b/sos/collector/clusters/kubernetes.py -@@ -44,11 +44,3 @@ class kubernetes(Cluster): - return nodes - else: - raise Exception('Node enumeration did not return usable output') -- -- --class openshift(kubernetes): -- -- cluster_name = 'OpenShift Container Platform' -- packages = ('atomic-openshift',) -- sos_preset = 'ocp' -- cmd = 'oc' -diff --git a/sos/collector/clusters/ocp.py b/sos/collector/clusters/ocp.py -new file mode 100644 -index 00000000..283fcfd1 ---- /dev/null -+++ b/sos/collector/clusters/ocp.py -@@ -0,0 +1,109 @@ -+# Copyright Red Hat 2021, Jake Hunsaker -+ -+# This file is part of the sos project: https://github.com/sosreport/sos -+# -+# This copyrighted material is made available to anyone wishing to use, -+# modify, copy, or redistribute it subject to the terms and conditions of -+# version 2 of the GNU General Public License. -+# -+# See the LICENSE file in the source distribution for further information. -+ -+from pipes import quote -+from sos.collector.clusters import Cluster -+ -+ -+class ocp(Cluster): -+ """OpenShift Container Platform v4""" -+ -+ cluster_name = 'OpenShift Container Platform v4' -+ packages = ('openshift-hyperkube', 'openshift-clients') -+ -+ option_list = [ -+ ('label', '', 'Colon delimited list of labels to select nodes with'), -+ ('role', '', 'Colon delimited list of roles to select nodes with'), -+ ('kubeconfig', '', 'Path to the kubeconfig file') -+ ] -+ -+ def fmt_oc_cmd(self, cmd): -+ """Format the oc command to optionall include the kubeconfig file if -+ one is specified -+ """ -+ if self.get_option('kubeconfig'): -+ return "oc --config %s %s" % (self.get_option('kubeconfig'), cmd) -+ return "oc %s" % cmd -+ -+ def check_enabled(self): -+ if super(ocp, self).check_enabled(): -+ return True -+ _who = self.fmt_oc_cmd('whoami') -+ return self.exec_master_cmd(_who)['status'] == 0 -+ -+ def _build_dict(self, nodelist): -+ """From the output of get_nodes(), construct an easier-to-reference -+ dict of nodes that will be used in determining labels, master status, -+ etc... -+ -+ :param nodelist: The split output of `oc get nodes` -+ :type nodelist: ``list`` -+ -+ :returns: A dict of nodes with `get nodes` columns as keys -+ :rtype: ``dict`` -+ """ -+ nodes = {} -+ if 'NAME' in nodelist[0]: -+ # get the index of the fields -+ statline = nodelist.pop(0).split() -+ idx = {} -+ for state in ['status', 'roles', 'version', 'os-image']: -+ try: -+ idx[state] = statline.index(state.upper()) -+ except Exception: -+ pass -+ for node in nodelist: -+ _node = node.split() -+ nodes[_node[0]] = {} -+ for column in idx: -+ nodes[_node[0]][column] = _node[idx[column]] -+ return nodes -+ -+ def get_nodes(self): -+ nodes = [] -+ self.node_dict = {} -+ cmd = 'get nodes -o wide' -+ if self.get_option('label'): -+ labels = ','.join(self.get_option('label').split(':')) -+ cmd += " -l %s" % quote(labels) -+ res = self.exec_master_cmd(self.fmt_oc_cmd(cmd)) -+ if res['status'] == 0: -+ roles = [r for r in self.get_option('role').split(':')] -+ self.node_dict = self._build_dict(res['stdout'].splitlines()) -+ for node in self.node_dict: -+ if roles: -+ for role in roles: -+ if role in node: -+ nodes.append(node) -+ else: -+ nodes.append(node) -+ else: -+ msg = "'oc' command failed" -+ if 'Missing or incomplete' in res['stdout']: -+ msg = ("'oc' failed due to missing kubeconfig on master node." -+ " Specify one via '-c ocp.kubeconfig='") -+ raise Exception(msg) -+ return nodes -+ -+ def set_node_label(self, node): -+ if node.address not in self.node_dict: -+ return '' -+ for label in ['master', 'worker']: -+ if label in self.node_dict[node.address]['roles']: -+ return label -+ return '' -+ -+ def check_node_is_master(self, sosnode): -+ if sosnode.address not in self.node_dict: -+ return False -+ return 'master' in self.node_dict[sosnode.address]['roles'] -+ -+ def set_master_options(self, node): -+ node.opts.enable_plugins.append('openshift') --- -2.26.3 - -From a3c1caad21160545eda87ea1fde93e972a6fbf88 Mon Sep 17 00:00:00 2001 -From: Jake Hunsaker -Date: Wed, 26 May 2021 11:55:24 -0400 -Subject: [PATCH] [cleaner] Don't strip empty lines from substituted files - -Fixes an issue where empty lines would be stripped from files that have -other obfuscations in them. Those empty lines may be important for file -structure and/or readability, so we should instead simply not pass empty -lines to the parsers rather than skipping them wholesale in the flow of -writing obfuscations to a temp file before replacing the source file -with a potentially changed temp file. - -Resolves: #2562 - -Signed-off-by: Jake Hunsaker ---- - sos/cleaner/__init__.py | 6 ++++-- - 1 file changed, 4 insertions(+), 2 deletions(-) - -diff --git a/sos/cleaner/__init__.py b/sos/cleaner/__init__.py -index bdd24f95..55465b85 100644 ---- a/sos/cleaner/__init__.py -+++ b/sos/cleaner/__init__.py -@@ -603,8 +603,6 @@ third party. - tfile = tempfile.NamedTemporaryFile(mode='w', dir=self.tmpdir) - with open(filename, 'r') as fname: - for line in fname: -- if not line.strip(): -- continue - try: - line, count = self.obfuscate_line(line) - subs += count -@@ -642,7 +640,11 @@ third party. - - Returns the fully obfuscated line and the number of substitutions made - """ -+ # don't iterate over blank lines, but still write them to the tempfile -+ # to maintain the same structure when we write a scrubbed file back - count = 0 -+ if not line.strip(): -+ return line, count - for parser in self.parsers: - try: - line, _count = parser.parse_line(line) --- -2.26.3 - -From 892bbd8114703f5a4d23aa77ba5829b7ba59446f Mon Sep 17 00:00:00 2001 -From: Jake Hunsaker -Date: Wed, 5 May 2021 17:02:04 -0400 -Subject: [PATCH] [cleaner] Remove binary files by default - -Binary files generally speaking cannot be obfuscated, and as such we -should remove them from archives being obfuscated by default so that -sensitive data is not mistakenly included in an obfuscated archive. - -This commits adds a new `--keep-binary-files` option that if used will -keep any encountered binary files in the final archive. The default -option of `false` will ensure that encountered binary files are removed. - -The number of removed binary files per archive is reported when -obfuscation is completed for that archive. - -Closes: #2478 -Resolves: #2524 - -Signed-off-by: Jake Hunsaker ---- - man/en/sos-clean.1 | 12 ++++ - sos/cleaner/__init__.py | 21 +++++- - sos/cleaner/obfuscation_archive.py | 67 ++++++++++++++++++-- - sos/collector/__init__.py | 5 ++ - sos/report/__init__.py | 6 ++ - 8 files changed, 167 insertions(+), 7 deletions(-) - -diff --git a/man/en/sos-clean.1 b/man/en/sos-clean.1 -index 4856b43b..b77bc63c 100644 ---- a/man/en/sos-clean.1 -+++ b/man/en/sos-clean.1 -@@ -9,6 +9,7 @@ sos clean - Obfuscate sensitive data from one or more sosreports - [\-\-map-file] - [\-\-jobs] - [\-\-no-update] -+ [\-\-keep-binary-files] - - .SH DESCRIPTION - \fBsos clean\fR or \fBsos mask\fR is an sos subcommand used to obfuscate sensitive information from -@@ -77,6 +78,17 @@ Default: 4 - .TP - .B \-\-no-update - Do not write the mapping file contents to /etc/sos/cleaner/default_mapping -+.TP -+.B \-\-keep-binary-files -+Keep unprocessable binary files in the archive, rather than removing them. -+ -+Note that binary files cannot be obfuscated, and thus keeping them in the archive -+may result in otherwise sensitive information being included in the final archive. -+Users should review any archive that keeps binary files in place before sending to -+a third party. -+ -+Default: False (remove encountered binary files) -+ - .SH SEE ALSO - .BR sos (1) - .BR sos-report (1) -diff --git a/sos/cleaner/__init__.py b/sos/cleaner/__init__.py -index 55465b85..f88ff8a0 100644 ---- a/sos/cleaner/__init__.py -+++ b/sos/cleaner/__init__.py -@@ -47,6 +47,7 @@ class SoSCleaner(SoSComponent): - 'keyword_file': None, - 'map_file': '/etc/sos/cleaner/default_mapping', - 'no_update': False, -+ 'keep_binary_files': False, - 'target': '', - 'usernames': [] - } -@@ -183,6 +184,11 @@ third party. - action='store_true', - help='Do not update the --map file with new ' - 'mappings from this run') -+ clean_grp.add_argument('--keep-binary-files', default=False, -+ action='store_true', -+ dest='keep_binary_files', -+ help='Keep unprocessable binary files in the ' -+ 'archive instead of removing them') - clean_grp.add_argument('--usernames', dest='usernames', default=[], - action='extend', - help='List of usernames to obfuscate') -@@ -467,6 +473,11 @@ third party. - "%s concurrently\n" - % (len(self.report_paths), self.opts.jobs)) - self.ui_log.info(msg) -+ if self.opts.keep_binary_files: -+ self.ui_log.warning( -+ "WARNING: binary files that potentially contain sensitive " -+ "information will NOT be removed from the final archive\n" -+ ) - pool = ThreadPoolExecutor(self.opts.jobs) - pool.map(self.obfuscate_report, self.report_paths, chunksize=1) - pool.shutdown(wait=True) -@@ -539,6 +550,10 @@ third party. - short_name = fname.split(archive.archive_name + '/')[1] - if archive.should_skip_file(short_name): - continue -+ if (not self.opts.keep_binary_files and -+ archive.should_remove_file(short_name)): -+ archive.remove_file(short_name) -+ continue - try: - count = self.obfuscate_file(fname, short_name, - archive.archive_name) -@@ -574,7 +589,11 @@ third party. - arc_md.add_field('files_obfuscated', len(archive.file_sub_list)) - arc_md.add_field('total_substitutions', archive.total_sub_count) - self.completed_reports.append(archive) -- archive.report_msg("Obfuscation completed") -+ rmsg = '' -+ if archive.removed_file_count: -+ rmsg = " [removed %s unprocessable files]" -+ rmsg = rmsg % archive.removed_file_count -+ archive.report_msg("Obfuscation completed%s" % rmsg) - - except Exception as err: - self.ui_log.info("Exception while processing %s: %s" -diff --git a/sos/cleaner/obfuscation_archive.py b/sos/cleaner/obfuscation_archive.py -index c64ab13b..76841b51 100644 ---- a/sos/cleaner/obfuscation_archive.py -+++ b/sos/cleaner/obfuscation_archive.py -@@ -28,6 +28,7 @@ class SoSObfuscationArchive(): - - file_sub_list = [] - total_sub_count = 0 -+ removed_file_count = 0 - - def __init__(self, archive_path, tmpdir): - self.archive_path = archive_path -@@ -62,11 +63,7 @@ class SoSObfuscationArchive(): - 'sys/firmware', - 'sys/fs', - 'sys/kernel/debug', -- 'sys/module', -- r'.*\.tar$', # TODO: support archive unpacking -- # Be explicit with these tar matches to avoid matching commands -- r'.*\.tar\.xz', -- '.*.gz' -+ 'sys/module' - ] - - @property -@@ -76,6 +73,17 @@ class SoSObfuscationArchive(): - except Exception: - return False - -+ def remove_file(self, fname): -+ """Remove a file from the archive. This is used when cleaner encounters -+ a binary file, which we cannot reliably obfuscate. -+ """ -+ full_fname = self.get_file_path(fname) -+ # don't call a blank remove() here -+ if full_fname: -+ self.log_info("Removing binary file '%s' from archive" % fname) -+ os.remove(full_fname) -+ self.removed_file_count += 1 -+ - def extract(self): - if self.is_tarfile: - self.report_msg("Extracting...") -@@ -227,3 +235,52 @@ class SoSObfuscationArchive(): - if filename.startswith(_skip) or re.match(_skip, filename): - return True - return False -+ -+ def should_remove_file(self, fname): -+ """Determine if the file should be removed or not, due to an inability -+ to reliably obfuscate that file based on the filename. -+ -+ :param fname: Filename relative to the extracted archive root -+ :type fname: ``str`` -+ -+ :returns: ``True`` if the file cannot be reliably obfuscated -+ :rtype: ``bool`` -+ """ -+ obvious_removes = [ -+ r'.*\.gz', # TODO: support flat gz/xz extraction -+ r'.*\.xz', -+ r'.*\.bzip2', -+ r'.*\.tar\..*', # TODO: support archive unpacking -+ r'.*\.txz$', -+ r'.*\.tgz$', -+ r'.*\.bin', -+ r'.*\.journal', -+ r'.*\~$' -+ ] -+ -+ # if the filename matches, it is obvious we can remove them without -+ # doing the read test -+ for _arc_reg in obvious_removes: -+ if re.match(_arc_reg, fname): -+ return True -+ -+ return self.file_is_binary(fname) -+ -+ def file_is_binary(self, fname): -+ """Determine if the file is a binary file or not. -+ -+ -+ :param fname: Filename relative to the extracted archive root -+ :type fname: ``str`` -+ -+ :returns: ``True`` if file is binary, else ``False`` -+ :rtype: ``bool`` -+ """ -+ with open(self.get_file_path(fname), 'tr') as tfile: -+ try: -+ # when opened as above (tr), reading binary content will raise -+ # an exception -+ tfile.read(1) -+ return False -+ except UnicodeDecodeError: -+ return True -diff --git a/sos/collector/__init__.py b/sos/collector/__init__.py -index 9884836c..469db60d 100644 ---- a/sos/collector/__init__.py -+++ b/sos/collector/__init__.py -@@ -67,6 +67,7 @@ class SoSCollector(SoSComponent): - 'jobs': 4, - 'keywords': [], - 'keyword_file': None, -+ 'keep_binary_files': False, - 'label': '', - 'list_options': False, - 'log_size': 0, -@@ -410,6 +411,10 @@ class SoSCollector(SoSComponent): - dest='clean', - default=False, action='store_true', - help='Obfuscate sensistive information') -+ cleaner_grp.add_argument('--keep-binary-files', default=False, -+ action='store_true', dest='keep_binary_files', -+ help='Keep unprocessable binary files in the ' -+ 'archive instead of removing them') - cleaner_grp.add_argument('--domains', dest='domains', default=[], - action='extend', - help='Additional domain names to obfuscate') -diff --git a/sos/report/__init__.py b/sos/report/__init__.py -index d4345409..2cedc76e 100644 ---- a/sos/report/__init__.py -+++ b/sos/report/__init__.py -@@ -82,6 +82,7 @@ class SoSReport(SoSComponent): - 'case_id': '', - 'chroot': 'auto', - 'clean': False, -+ 'keep_binary_files': False, - 'desc': '', - 'domains': [], - 'dry_run': False, -@@ -344,6 +345,11 @@ class SoSReport(SoSComponent): - default='/etc/sos/cleaner/default_mapping', - help=('Provide a previously generated mapping' - ' file for obfuscation')) -+ cleaner_grp.add_argument('--keep-binary-files', default=False, -+ action='store_true', -+ dest='keep_binary_files', -+ help='Keep unprocessable binary files in the ' -+ 'archive instead of removing them') - cleaner_grp.add_argument('--usernames', dest='usernames', default=[], - action='extend', - help='List of usernames to obfuscate') - -From aed0102a1d6ef9a030c9e5349f092b51b9d1f22d Mon Sep 17 00:00:00 2001 -From: Jake Hunsaker -Date: Fri, 11 Jun 2021 23:20:59 -0400 -Subject: [PATCH 01/10] [SoSNode] Allow individually setting node options - -Like we now do for primary nodes, add the ability to individually set -node options via a new `set_node_options()` method for when blanket -setting options across all nodes via the options class attrs is not -sufficient. - -Signed-off-by: Jake Hunsaker ---- - sos/collector/clusters/__init__.py | 10 ++++++++++ - sos/collector/sosnode.py | 6 ++++-- - 2 files changed, 14 insertions(+), 2 deletions(-) - -diff --git a/sos/collector/clusters/__init__.py b/sos/collector/clusters/__init__.py -index 90e62d79..c4da1ab8 100644 ---- a/sos/collector/clusters/__init__.py -+++ b/sos/collector/clusters/__init__.py -@@ -137,6 +137,16 @@ class Cluster(): - """ - self.cluster_ssh_key = key - -+ def set_node_options(self, node): -+ """If there is a need to set specific options on ONLY the non-primary -+ nodes in a collection, override this method in the cluster profile -+ and do that here. -+ -+ :param node: The non-primary node -+ :type node: ``SoSNode`` -+ """ -+ pass -+ - def set_master_options(self, node): - """If there is a need to set specific options in the sos command being - run on the cluster's master nodes, override this method in the cluster -diff --git a/sos/collector/sosnode.py b/sos/collector/sosnode.py -index 1fc03076..7e784aa1 100644 ---- a/sos/collector/sosnode.py -+++ b/sos/collector/sosnode.py -@@ -657,6 +657,8 @@ class SosNode(): - # set master-only options - if self.cluster.check_node_is_master(self): - self.cluster.set_master_options(self) -+ else: -+ self.cluster.set_node_options(self) - - def finalize_sos_cmd(self): - """Use host facts and compare to the cluster type to modify the sos -@@ -713,13 +715,13 @@ class SosNode(): - sos_opts.append('--cmd-timeout=%s' - % quote(str(self.opts.cmd_timeout))) - -+ self.update_cmd_from_cluster() -+ - sos_cmd = sos_cmd.replace( - 'sosreport', - os.path.join(self.host.sos_bin_path, self.sos_bin) - ) - -- self.update_cmd_from_cluster() -- - if self.opts.only_plugins: - plugs = [o for o in self.opts.only_plugins - if self._plugin_exists(o)] --- -2.26.3 - - -From 96f166699d12704cc7cf73cb8b13278675f68730 Mon Sep 17 00:00:00 2001 -From: Jake Hunsaker -Date: Sat, 12 Jun 2021 00:02:36 -0400 -Subject: [PATCH 02/10] [sosnode] Support passing env vars to `run_command()` - -Updates `run_command()` to support passing new environment variables to -the command being run, for that command alone. This parameter takes a -dict, and if set we will first copy the existing set of env vars on the -node and then update that set of variables using the passed dict. - -Additionally, `execute_sos_command()` will now try to pass a new -`sos_env_vars` dict (default empty) so that clusters may set environment -variables specifically for the sos command being run, without having to -modify the actual sos command being executed. - -Signed-off-by: Jake Hunsaker ---- - sos/collector/sosnode.py | 27 ++++++++++++++++++++++++--- - 1 file changed, 24 insertions(+), 3 deletions(-) - -diff --git a/sos/collector/sosnode.py b/sos/collector/sosnode.py -index 7e784aa1..40472a4e 100644 ---- a/sos/collector/sosnode.py -+++ b/sos/collector/sosnode.py -@@ -45,6 +45,8 @@ class SosNode(): - self.host = None - self.cluster = None - self.hostname = None -+ self.sos_env_vars = {} -+ self._env_vars = {} - self._password = password or self.opts.password - if not self.opts.nopasswd_sudo and not self.opts.sudo_pw: - self.opts.sudo_pw = self._password -@@ -109,6 +111,21 @@ class SosNode(): - def _fmt_msg(self, msg): - return '{:<{}} : {}'.format(self._hostname, self.hostlen + 1, msg) - -+ @property -+ def env_vars(self): -+ if not self._env_vars: -+ if self.local: -+ self._env_vars = os.environ.copy() -+ else: -+ ret = self.run_command("env --null") -+ if ret['status'] == 0: -+ for ln in ret['output'].split('\x00'): -+ if not ln: -+ continue -+ _val = ln.split('=') -+ self._env_vars[_val[0]] = _val[1] -+ return self._env_vars -+ - def set_node_manifest(self, manifest): - """Set the manifest section that this node will write to - """ -@@ -404,7 +421,7 @@ class SosNode(): - return self.host.package_manager.pkg_by_name(pkg) is not None - - def run_command(self, cmd, timeout=180, get_pty=False, need_root=False, -- force_local=False, use_container=False): -+ force_local=False, use_container=False, env=None): - """Runs a given cmd, either via the SSH session or locally - - Arguments: -@@ -446,7 +463,10 @@ class SosNode(): - else: - if get_pty: - cmd = "/bin/bash -c %s" % quote(cmd) -- res = pexpect.spawn(cmd, encoding='utf-8') -+ if env: -+ _cmd_env = self.env_vars -+ _cmd_env.update(env) -+ res = pexpect.spawn(cmd, encoding='utf-8', env=_cmd_env) - if need_root: - if self.need_sudo: - res.sendline(self.opts.sudo_pw) -@@ -830,7 +850,8 @@ class SosNode(): - res = self.run_command(self.sos_cmd, - timeout=self.opts.timeout, - get_pty=True, need_root=True, -- use_container=True) -+ use_container=True, -+ env=self.sos_env_vars) - if res['status'] == 0: - for line in res['stdout'].splitlines(): - if fnmatch.fnmatch(line, '*sosreport-*tar*'): --- -2.26.3 - - -From a9e1632113406a646bdd7525982b699cf790aedb Mon Sep 17 00:00:00 2001 -From: Jake Hunsaker -Date: Tue, 15 Jun 2021 12:43:27 -0400 -Subject: [PATCH 03/10] [collect|sosnode] Avoiding clobbering sos options - between nodes - -This commit overhauls the function of `finalize_sos_cmd()` in several -ways. - -First, assign the sos report plugin related options directly to private -copies of those values for each node, so that the shared cluster profile -does not clober options between nodes. - -Second, provide a default Lock mechanism for clusters that need to -perform some node-comparison logic when assigning options based on node -role. - -Finally, finalize the sos command for each node _prior_ to the call to -`SoSNode.sosreport()` so that we can be sure that clusters are able to -appropriately compare and assign sos options across nodes before some -nodes have already started and/or finished their own sos report -collections. - -Signed-off-by: Jake Hunsaker ---- - sos/collector/__init__.py | 14 +++++ - sos/collector/clusters/__init__.py | 2 + - sos/collector/sosnode.py | 89 +++++++++++++++++------------- - 3 files changed, 67 insertions(+), 38 deletions(-) - -diff --git a/sos/collector/__init__.py b/sos/collector/__init__.py -index 469db60d..7b8cfcf7 100644 ---- a/sos/collector/__init__.py -+++ b/sos/collector/__init__.py -@@ -1186,6 +1186,10 @@ this utility or remote systems that it connects to. - "concurrently\n" - % (self.report_num, self.opts.jobs)) - -+ npool = ThreadPoolExecutor(self.opts.jobs) -+ npool.map(self._finalize_sos_cmd, self.client_list, chunksize=1) -+ npool.shutdown(wait=True) -+ - pool = ThreadPoolExecutor(self.opts.jobs) - pool.map(self._collect, self.client_list, chunksize=1) - pool.shutdown(wait=True) -@@ -1217,6 +1221,16 @@ this utility or remote systems that it connects to. - except Exception as err: - self.ui_log.error("Upload attempt failed: %s" % err) - -+ def _finalize_sos_cmd(self, client): -+ """Calls finalize_sos_cmd() on each node so that we have the final -+ command before we thread out the actual execution of sos -+ """ -+ try: -+ client.finalize_sos_cmd() -+ except Exception as err: -+ self.log_error("Could not finalize sos command for %s: %s" -+ % (client.address, err)) -+ - def _collect(self, client): - """Runs sosreport on each node""" - try: -diff --git a/sos/collector/clusters/__init__.py b/sos/collector/clusters/__init__.py -index c4da1ab8..bb728bc0 100644 ---- a/sos/collector/clusters/__init__.py -+++ b/sos/collector/clusters/__init__.py -@@ -11,6 +11,7 @@ - import logging - - from sos.options import ClusterOption -+from threading import Lock - - - class Cluster(): -@@ -66,6 +67,7 @@ class Cluster(): - if cls.__name__ != 'Cluster': - self.cluster_type.append(cls.__name__) - self.node_list = None -+ self.lock = Lock() - self.soslog = logging.getLogger('sos') - self.ui_log = logging.getLogger('sos_ui') - self.options = [] -diff --git a/sos/collector/sosnode.py b/sos/collector/sosnode.py -index 40472a4e..1c25cc34 100644 ---- a/sos/collector/sosnode.py -+++ b/sos/collector/sosnode.py -@@ -38,6 +38,7 @@ class SosNode(): - self.address = address.strip() - self.commons = commons - self.opts = commons['cmdlineopts'] -+ self._assign_config_opts() - self.tmpdir = commons['tmpdir'] - self.hostlen = commons['hostlen'] - self.need_sudo = commons['need_sudo'] -@@ -465,8 +466,8 @@ class SosNode(): - cmd = "/bin/bash -c %s" % quote(cmd) - if env: - _cmd_env = self.env_vars -- _cmd_env.update(env) -- res = pexpect.spawn(cmd, encoding='utf-8', env=_cmd_env) -+ env = _cmd_env.update(env) -+ res = pexpect.spawn(cmd, encoding='utf-8', env=env) - if need_root: - if self.need_sudo: - res.sendline(self.opts.sudo_pw) -@@ -484,9 +485,6 @@ class SosNode(): - - def sosreport(self): - """Run a sosreport on the node, then collect it""" -- self.sos_cmd = self.finalize_sos_cmd() -- self.log_info('Final sos command set to %s' % self.sos_cmd) -- self.manifest.add_field('final_sos_command', self.sos_cmd) - try: - path = self.execute_sos_command() - if path: -@@ -656,29 +654,42 @@ class SosNode(): - This will NOT override user supplied options. - """ - if self.cluster.sos_preset: -- if not self.opts.preset: -- self.opts.preset = self.cluster.sos_preset -+ if not self.preset: -+ self.preset = self.cluster.sos_preset - else: - self.log_info('Cluster specified preset %s but user has also ' - 'defined a preset. Using user specification.' - % self.cluster.sos_preset) - if self.cluster.sos_plugins: - for plug in self.cluster.sos_plugins: -- if plug not in self.opts.enable_plugins: -- self.opts.enable_plugins.append(plug) -+ if plug not in self.enable_plugins: -+ self.enable_plugins.append(plug) - - if self.cluster.sos_plugin_options: - for opt in self.cluster.sos_plugin_options: -- if not any(opt in o for o in self.opts.plugin_options): -+ if not any(opt in o for o in self.plugin_options): - option = '%s=%s' % (opt, - self.cluster.sos_plugin_options[opt]) -- self.opts.plugin_options.append(option) -+ self.plugin_options.append(option) - - # set master-only options - if self.cluster.check_node_is_master(self): -- self.cluster.set_master_options(self) -+ with self.cluster.lock: -+ self.cluster.set_master_options(self) - else: -- self.cluster.set_node_options(self) -+ with self.cluster.lock: -+ self.cluster.set_node_options(self) -+ -+ def _assign_config_opts(self): -+ """From the global opts configuration, assign those values locally -+ to this node so that they may be acted on individually. -+ """ -+ # assign these to new, private copies -+ self.only_plugins = list(self.opts.only_plugins) -+ self.skip_plugins = list(self.opts.skip_plugins) -+ self.enable_plugins = list(self.opts.enable_plugins) -+ self.plugin_options = list(self.opts.plugin_options) -+ self.preset = list(self.opts.preset) - - def finalize_sos_cmd(self): - """Use host facts and compare to the cluster type to modify the sos -@@ -742,59 +753,61 @@ class SosNode(): - os.path.join(self.host.sos_bin_path, self.sos_bin) - ) - -- if self.opts.only_plugins: -- plugs = [o for o in self.opts.only_plugins -- if self._plugin_exists(o)] -- if len(plugs) != len(self.opts.only_plugins): -- not_only = list(set(self.opts.only_plugins) - set(plugs)) -+ if self.only_plugins: -+ plugs = [o for o in self.only_plugins if self._plugin_exists(o)] -+ if len(plugs) != len(self.only_plugins): -+ not_only = list(set(self.only_plugins) - set(plugs)) - self.log_debug('Requested plugins %s were requested to be ' - 'enabled but do not exist' % not_only) -- only = self._fmt_sos_opt_list(self.opts.only_plugins) -+ only = self._fmt_sos_opt_list(self.only_plugins) - if only: - sos_opts.append('--only-plugins=%s' % quote(only)) -- return "%s %s" % (sos_cmd, ' '.join(sos_opts)) -+ self.sos_cmd = "%s %s" % (sos_cmd, ' '.join(sos_opts)) -+ self.log_info('Final sos command set to %s' % self.sos_cmd) -+ self.manifest.add_field('final_sos_command', self.sos_cmd) -+ return - -- if self.opts.skip_plugins: -+ if self.skip_plugins: - # only run skip-plugins for plugins that are enabled -- skip = [o for o in self.opts.skip_plugins -- if self._check_enabled(o)] -- if len(skip) != len(self.opts.skip_plugins): -- not_skip = list(set(self.opts.skip_plugins) - set(skip)) -+ skip = [o for o in self.skip_plugins if self._check_enabled(o)] -+ if len(skip) != len(self.skip_plugins): -+ not_skip = list(set(self.skip_plugins) - set(skip)) - self.log_debug('Requested to skip plugins %s, but plugins are ' - 'already not enabled' % not_skip) - skipln = self._fmt_sos_opt_list(skip) - if skipln: - sos_opts.append('--skip-plugins=%s' % quote(skipln)) - -- if self.opts.enable_plugins: -+ if self.enable_plugins: - # only run enable for plugins that are disabled -- opts = [o for o in self.opts.enable_plugins -- if o not in self.opts.skip_plugins -+ opts = [o for o in self.enable_plugins -+ if o not in self.skip_plugins - and self._check_disabled(o) and self._plugin_exists(o)] -- if len(opts) != len(self.opts.enable_plugins): -- not_on = list(set(self.opts.enable_plugins) - set(opts)) -+ if len(opts) != len(self.enable_plugins): -+ not_on = list(set(self.enable_plugins) - set(opts)) - self.log_debug('Requested to enable plugins %s, but plugins ' - 'are already enabled or do not exist' % not_on) - enable = self._fmt_sos_opt_list(opts) - if enable: - sos_opts.append('--enable-plugins=%s' % quote(enable)) - -- if self.opts.plugin_options: -- opts = [o for o in self.opts.plugin_options -+ if self.plugin_options: -+ opts = [o for o in self.plugin_options - if self._plugin_exists(o.split('.')[0]) - and self._plugin_option_exists(o.split('=')[0])] - if opts: - sos_opts.append('-k %s' % quote(','.join(o for o in opts))) - -- if self.opts.preset: -- if self._preset_exists(self.opts.preset): -- sos_opts.append('--preset=%s' % quote(self.opts.preset)) -+ if self.preset: -+ if self._preset_exists(self.preset): -+ sos_opts.append('--preset=%s' % quote(self.preset)) - else: - self.log_debug('Requested to enable preset %s but preset does ' -- 'not exist on node' % self.opts.preset) -+ 'not exist on node' % self.preset) - -- _sos_cmd = "%s %s" % (sos_cmd, ' '.join(sos_opts)) -- return _sos_cmd -+ self.sos_cmd = "%s %s" % (sos_cmd, ' '.join(sos_opts)) -+ self.log_info('Final sos command set to %s' % self.sos_cmd) -+ self.manifest.add_field('final_sos_command', self.sos_cmd) - - def determine_sos_label(self): - """Determine what, if any, label should be added to the sosreport""" --- -2.26.3 - - -From 7e6c078e51143f7064190b316a251ddd8d431495 Mon Sep 17 00:00:00 2001 -From: Jake Hunsaker -Date: Tue, 15 Jun 2021 18:38:34 -0400 -Subject: [PATCH 04/10] [cleaner] Improve handling of symlink obfuscation - -Improves handling of symlink obfuscation by only performing the -obfuscaiton on the ultimate target of any symlinks encountered. Now, -when a symlink is encountered, clean will obfuscate the link name and -re-write it in the archive, pointing to the (potentially obfuscated) -target name. - -Signed-off-by: Jake Hunsaker ---- - sos/cleaner/__init__.py | 65 +++++++++++++++++++++++++++++------------ - 1 file changed, 46 insertions(+), 19 deletions(-) - -diff --git a/sos/cleaner/__init__.py b/sos/cleaner/__init__.py -index abfb684b..b38c8dfc 100644 ---- a/sos/cleaner/__init__.py -+++ b/sos/cleaner/__init__.py -@@ -612,28 +612,55 @@ third party. - if not filename: - # the requested file doesn't exist in the archive - return -- self.log_debug("Obfuscating %s" % short_name or filename, -- caller=arc_name) - subs = 0 -- tfile = tempfile.NamedTemporaryFile(mode='w', dir=self.tmpdir) -- with open(filename, 'r') as fname: -- for line in fname: -- try: -- line, count = self.obfuscate_line(line) -- subs += count -- tfile.write(line) -- except Exception as err: -- self.log_debug("Unable to obfuscate %s: %s" -- % (short_name, err), caller=arc_name) -- tfile.seek(0) -- if subs: -- shutil.copy(tfile.name, filename) -- tfile.close() -- _ob_filename = self.obfuscate_string(short_name) -- if _ob_filename != short_name: -+ if not os.path.islink(filename): -+ # don't run the obfuscation on the link, but on the actual file -+ # at some other point. -+ self.log_debug("Obfuscating %s" % short_name or filename, -+ caller=arc_name) -+ tfile = tempfile.NamedTemporaryFile(mode='w', dir=self.tmpdir) -+ with open(filename, 'r') as fname: -+ for line in fname: -+ try: -+ line, count = self.obfuscate_line(line) -+ subs += count -+ tfile.write(line) -+ except Exception as err: -+ self.log_debug("Unable to obfuscate %s: %s" -+ % (short_name, err), caller=arc_name) -+ tfile.seek(0) -+ if subs: -+ shutil.copy(tfile.name, filename) -+ tfile.close() -+ -+ _ob_short_name = self.obfuscate_string(short_name.split('/')[-1]) -+ _ob_filename = short_name.replace(short_name.split('/')[-1], -+ _ob_short_name) -+ _sym_changed = False -+ if os.path.islink(filename): -+ _link = os.readlink(filename) -+ _ob_link = self.obfuscate_string(_link) -+ if _ob_link != _link: -+ _sym_changed = True -+ -+ if (_ob_filename != short_name) or _sym_changed: - arc_path = filename.split(short_name)[0] - _ob_path = os.path.join(arc_path, _ob_filename) -- os.rename(filename, _ob_path) -+ # ensure that any plugin subdirs that contain obfuscated strings -+ # get created with obfuscated counterparts -+ if not os.path.islink(filename): -+ os.rename(filename, _ob_path) -+ else: -+ # generate the obfuscated name of the link target -+ _target_ob = self.obfuscate_string(os.readlink(filename)) -+ # remove the unobfuscated original symlink first, in case the -+ # symlink name hasn't changed but the target has -+ os.remove(filename) -+ # create the newly obfuscated symlink, pointing to the -+ # obfuscated target name, which may not exist just yet, but -+ # when the actual file is obfuscated, will be created -+ os.symlink(_target_ob, _ob_path) -+ - return subs - - def obfuscate_string(self, string_data): --- -2.26.3 - - -From b5d166ac9ff79bc3740c5e66f16d60762f9a0ac0 Mon Sep 17 00:00:00 2001 -From: Jake Hunsaker -Date: Tue, 15 Jun 2021 22:56:19 -0400 -Subject: [PATCH 05/10] [cleaner] Iterate over matches with most precise match - first - -When matching strings in parsers to do obfuscation, we should be using -the most precise matches found first, rather than matching in the order -a match is hit. This ensures that we correctly obfuscate an entire -string, rather than potentially only partial substring(s) that exist -within the entire match. - -Signed-off-by: Jake Hunsaker ---- - sos/cleaner/parsers/__init__.py | 10 +++++++--- - sos/cleaner/parsers/keyword_parser.py | 2 +- - sos/cleaner/parsers/username_parser.py | 2 +- - 3 files changed, 9 insertions(+), 5 deletions(-) - -diff --git a/sos/cleaner/parsers/__init__.py b/sos/cleaner/parsers/__init__.py -index c77300aa..cfa20b95 100644 ---- a/sos/cleaner/parsers/__init__.py -+++ b/sos/cleaner/parsers/__init__.py -@@ -82,10 +82,12 @@ class SoSCleanerParser(): - for pattern in self.regex_patterns: - matches = [m[0] for m in re.findall(pattern, line, re.I)] - if matches: -+ matches.sort(reverse=True, key=lambda x: len(x)) - count += len(matches) - for match in matches: -- new_match = self.mapping.get(match.strip()) -- line = line.replace(match.strip(), new_match) -+ match = match.strip() -+ new_match = self.mapping.get(match) -+ line = line.replace(match, new_match) - return line, count - - def parse_string_for_keys(self, string_data): -@@ -102,7 +104,9 @@ class SoSCleanerParser(): - :returns: The obfuscated line - :rtype: ``str`` - """ -- for key, val in self.mapping.dataset.items(): -+ for pair in sorted(self.mapping.dataset.items(), reverse=True, -+ key=lambda x: len(x[0])): -+ key, val = pair - if key in string_data: - string_data = string_data.replace(key, val) - return string_data -diff --git a/sos/cleaner/parsers/keyword_parser.py b/sos/cleaner/parsers/keyword_parser.py -index 3dc2b7f0..9134f82d 100644 ---- a/sos/cleaner/parsers/keyword_parser.py -+++ b/sos/cleaner/parsers/keyword_parser.py -@@ -42,7 +42,7 @@ class SoSKeywordParser(SoSCleanerParser): - - def parse_line(self, line): - count = 0 -- for keyword in self.user_keywords: -+ for keyword in sorted(self.user_keywords, reverse=True): - if keyword in line: - line = line.replace(keyword, self.mapping.get(keyword)) - count += 1 -diff --git a/sos/cleaner/parsers/username_parser.py b/sos/cleaner/parsers/username_parser.py -index 2bb6c7f3..0c3bbac4 100644 ---- a/sos/cleaner/parsers/username_parser.py -+++ b/sos/cleaner/parsers/username_parser.py -@@ -51,7 +51,7 @@ class SoSUsernameParser(SoSCleanerParser): - - def parse_line(self, line): - count = 0 -- for username in self.mapping.dataset.keys(): -+ for username in sorted(self.mapping.dataset.keys(), reverse=True): - if username in line: - count = line.count(username) - line = line.replace(username, self.mapping.get(username)) --- -2.26.3 - - -From 7ed138fcd2ee6ece3e7fbd9e48293b212e0b4e41 Mon Sep 17 00:00:00 2001 -From: Jake Hunsaker -Date: Wed, 16 Jun 2021 01:15:45 -0400 -Subject: [PATCH 06/10] [cleaner] Explicitly obfuscate directory names within - archives - -This commits adds a step to `obfuscate_report()` that explicitly walks -through all directories in the archive, and obfuscates the directory -names if necessary. - -Since this uses `obfuscate_string()` for the directory names, a -`skip_keys` list has been added to maps to allow parsers/maps to -specify matched keys (such as short names for the hostname parser) that -should not be considered when obfuscating directory names (e.g. 'www'). - -Closes: #2465 - -Signed-off-by: Jake Hunsaker ---- - sos/cleaner/__init__.py | 26 ++++++++++++++++++++++++++ - sos/cleaner/mappings/__init__.py | 4 +++- - sos/cleaner/mappings/hostname_map.py | 5 +++++ - sos/cleaner/obfuscation_archive.py | 20 ++++++++++++++++++-- - sos/cleaner/parsers/__init__.py | 2 ++ - 5 files changed, 54 insertions(+), 3 deletions(-) - -diff --git a/sos/cleaner/__init__.py b/sos/cleaner/__init__.py -index b38c8dfc..88d4d0ea 100644 ---- a/sos/cleaner/__init__.py -+++ b/sos/cleaner/__init__.py -@@ -562,6 +562,11 @@ third party. - except Exception as err: - self.log_debug("Unable to parse file %s: %s" - % (short_name, err)) -+ try: -+ self.obfuscate_directory_names(archive) -+ except Exception as err: -+ self.log_info("Failed to obfuscate directories: %s" % err, -+ caller=archive.archive_name) - - # if the archive was already a tarball, repack it - method = archive.get_compression() -@@ -663,6 +668,27 @@ third party. - - return subs - -+ def obfuscate_directory_names(self, archive): -+ """For all directories that exist within the archive, obfuscate the -+ directory name if it contains sensitive strings found during execution -+ """ -+ self.log_info("Obfuscating directory names in archive %s" -+ % archive.archive_name) -+ for dirpath in sorted(archive.get_directory_list(), reverse=True): -+ for _name in os.listdir(dirpath): -+ _dirname = os.path.join(dirpath, _name) -+ _arc_dir = _dirname.split(archive.extracted_path)[-1] -+ if os.path.isdir(_dirname): -+ _ob_dirname = self.obfuscate_string(_name) -+ if _ob_dirname != _name: -+ _ob_arc_dir = _arc_dir.rstrip(_name) -+ _ob_arc_dir = os.path.join( -+ archive.extracted_path, -+ _ob_arc_dir.lstrip('/'), -+ _ob_dirname -+ ) -+ os.rename(_dirname, _ob_arc_dir) -+ - def obfuscate_string(self, string_data): - for parser in self.parsers: - try: -diff --git a/sos/cleaner/mappings/__init__.py b/sos/cleaner/mappings/__init__.py -index dd464e5a..5cf5c8b2 100644 ---- a/sos/cleaner/mappings/__init__.py -+++ b/sos/cleaner/mappings/__init__.py -@@ -20,8 +20,10 @@ class SoSMap(): - corresponding SoSMap() object, to allow for easy retrieval of obfuscated - items. - """ -- -+ # used for regex skips in parser.parse_line() - ignore_matches = [] -+ # used for filename obfuscations in parser.parse_string_for_keys() -+ skip_keys = [] - - def __init__(self): - self.dataset = {} -diff --git a/sos/cleaner/mappings/hostname_map.py b/sos/cleaner/mappings/hostname_map.py -index e0b7bf1d..c9a44d8d 100644 ---- a/sos/cleaner/mappings/hostname_map.py -+++ b/sos/cleaner/mappings/hostname_map.py -@@ -35,6 +35,11 @@ class SoSHostnameMap(SoSMap): - '^com..*' - ] - -+ skip_keys = [ -+ 'www', -+ 'api' -+ ] -+ - host_count = 0 - domain_count = 0 - _domains = {} -diff --git a/sos/cleaner/obfuscation_archive.py b/sos/cleaner/obfuscation_archive.py -index 88f978d9..90188358 100644 ---- a/sos/cleaner/obfuscation_archive.py -+++ b/sos/cleaner/obfuscation_archive.py -@@ -202,10 +202,22 @@ class SoSObfuscationArchive(): - """Return a list of all files within the archive""" - self.file_list = [] - for dirname, dirs, files in os.walk(self.extracted_path): -+ for _dir in dirs: -+ _dirpath = os.path.join(dirname, _dir) -+ # catch dir-level symlinks -+ if os.path.islink(_dirpath) and os.path.isdir(_dirpath): -+ self.file_list.append(_dirpath) - for filename in files: - self.file_list.append(os.path.join(dirname, filename)) - return self.file_list - -+ def get_directory_list(self): -+ """Return a list of all directories within the archive""" -+ dir_list = [] -+ for dirname, dirs, files in os.walk(self.extracted_path): -+ dir_list.append(dirname) -+ return dir_list -+ - def update_sub_count(self, fname, count): - """Called when a file has finished being parsed and used to track - total substitutions made and number of files that had changes made -@@ -230,7 +242,8 @@ class SoSObfuscationArchive(): - archive root - """ - -- if not os.path.isfile(self.get_file_path(filename)): -+ if (not os.path.isfile(self.get_file_path(filename)) and not -+ os.path.islink(self.get_file_path(filename))): - return True - - for _skip in self.skip_list: -@@ -266,7 +279,10 @@ class SoSObfuscationArchive(): - if re.match(_arc_reg, fname): - return True - -- return self.file_is_binary(fname) -+ if os.path.isfile(self.get_file_path(fname)): -+ return self.file_is_binary(fname) -+ # don't fail on dir-level symlinks -+ return False - - def file_is_binary(self, fname): - """Determine if the file is a binary file or not. -diff --git a/sos/cleaner/parsers/__init__.py b/sos/cleaner/parsers/__init__.py -index cfa20b95..84874475 100644 ---- a/sos/cleaner/parsers/__init__.py -+++ b/sos/cleaner/parsers/__init__.py -@@ -107,6 +107,8 @@ class SoSCleanerParser(): - for pair in sorted(self.mapping.dataset.items(), reverse=True, - key=lambda x: len(x[0])): - key, val = pair -+ if key in self.mapping.skip_keys: -+ continue - if key in string_data: - string_data = string_data.replace(key, val) - return string_data --- -2.26.3 - - -From f180150277b706e72f2445287f3d0b6943efa252 Mon Sep 17 00:00:00 2001 -From: Jake Hunsaker -Date: Wed, 16 Jun 2021 02:24:51 -0400 -Subject: [PATCH 07/10] [hostname parser,map] Attempt to detect strings with - FQDN substrings - -This commit updates the hostname parser and associated map to be able to -better detect and obfuscate FQDN substrings within file content and file -names, particularly when the regex patterns failed to match a hostname -that is formatted with '_' characters rather than '.' characters. - -The `get()` method has been updated to alow preserve characters and -certain extensions that are not part of the FQDN, but are brought in by -the regex pattern due to the fact that we need to use word boundary -indicators within the pattern. - -Signed-off-by: Jake Hunsaker ---- - sos/cleaner/mappings/hostname_map.py | 59 +++++++++++++++++++++++--- - sos/cleaner/parsers/__init__.py | 3 +- - sos/cleaner/parsers/hostname_parser.py | 30 ++++++++++--- - 3 files changed, 81 insertions(+), 11 deletions(-) - -diff --git a/sos/cleaner/mappings/hostname_map.py b/sos/cleaner/mappings/hostname_map.py -index c9a44d8d..d4b2c88e 100644 ---- a/sos/cleaner/mappings/hostname_map.py -+++ b/sos/cleaner/mappings/hostname_map.py -@@ -104,7 +104,7 @@ class SoSHostnameMap(SoSMap): - host = domain.split('.') - if len(host) == 1: - # don't block on host's shortname -- return True -+ return host[0] in self.hosts.keys() - else: - domain = host[0:-1] - for known_domain in self._domains: -@@ -113,12 +113,59 @@ class SoSHostnameMap(SoSMap): - return False - - def get(self, item): -- if item.startswith(('.', '_')): -- item = item.lstrip('._') -- item = item.strip() -+ prefix = '' -+ suffix = '' -+ final = None -+ # The regex pattern match may include a leading and/or trailing '_' -+ # character due to the need to use word boundary matching, so we need -+ # to strip these from the string during processing, but still keep them -+ # in the returned string to not mangle the string replacement in the -+ # context of the file or filename -+ while item.startswith(('.', '_')): -+ prefix += item[0] -+ item = item[1:] -+ while item.endswith(('.', '_')): -+ suffix += item[-1] -+ item = item[0:-1] - if not self.domain_name_in_loaded_domains(item.lower()): - return item -- return super(SoSHostnameMap, self).get(item) -+ if item.endswith(('.yaml', '.yml', '.crt', '.key', '.pem')): -+ ext = '.' + item.split('.')[-1] -+ item = item.replace(ext, '') -+ suffix += ext -+ if item not in self.dataset.keys(): -+ # try to account for use of '-' in names that include hostnames -+ # and don't create new mappings for each of these -+ for _existing in sorted(self.dataset.keys(), reverse=True, -+ key=lambda x: len(x)): -+ _host_substr = False -+ _test = item.split(_existing) -+ _h = _existing.split('.') -+ # avoid considering a full FQDN match as a new match off of -+ # the hostname of an existing match -+ if _h[0] and _h[0] in self.hosts.keys(): -+ _host_substr = True -+ if len(_test) == 1 or not _test[0]: -+ # does not match existing obfuscation -+ continue -+ elif _test[0].endswith('.') and not _host_substr: -+ # new hostname in known domain -+ final = super(SoSHostnameMap, self).get(item) -+ break -+ elif item.split(_test[0]): -+ # string that includes existing FQDN obfuscation substring -+ # so, only obfuscate the FQDN part -+ try: -+ itm = item.split(_test[0])[1] -+ final = _test[0] + super(SoSHostnameMap, self).get(itm) -+ break -+ except Exception: -+ # fallback to still obfuscating the entire item -+ pass -+ -+ if not final: -+ final = super(SoSHostnameMap, self).get(item) -+ return prefix + final + suffix - - def sanitize_item(self, item): - host = item.split('.') -@@ -146,6 +193,8 @@ class SoSHostnameMap(SoSMap): - """Obfuscate the short name of the host with an incremented counter - based on the total number of obfuscated host names - """ -+ if not hostname: -+ return hostname - if hostname not in self.hosts: - ob_host = "host%s" % self.host_count - self.hosts[hostname] = ob_host -diff --git a/sos/cleaner/parsers/__init__.py b/sos/cleaner/parsers/__init__.py -index 84874475..57d2020a 100644 ---- a/sos/cleaner/parsers/__init__.py -+++ b/sos/cleaner/parsers/__init__.py -@@ -87,7 +87,8 @@ class SoSCleanerParser(): - for match in matches: - match = match.strip() - new_match = self.mapping.get(match) -- line = line.replace(match, new_match) -+ if new_match != match: -+ line = line.replace(match, new_match) - return line, count - - def parse_string_for_keys(self, string_data): -diff --git a/sos/cleaner/parsers/hostname_parser.py b/sos/cleaner/parsers/hostname_parser.py -index 9982024b..3de6bb08 100644 ---- a/sos/cleaner/parsers/hostname_parser.py -+++ b/sos/cleaner/parsers/hostname_parser.py -@@ -18,7 +18,7 @@ class SoSHostnameParser(SoSCleanerParser): - map_file_key = 'hostname_map' - prep_map_file = 'sos_commands/host/hostname' - regex_patterns = [ -- r'(((\b|_)[a-zA-Z0-9-\.]{1,200}\.[a-zA-Z]{1,63}\b))' -+ r'(((\b|_)[a-zA-Z0-9-\.]{1,200}\.[a-zA-Z]{1,63}(\b|_)))' - ] - - def __init__(self, conf_file=None, opt_domains=None): -@@ -66,10 +66,30 @@ class SoSHostnameParser(SoSCleanerParser): - """Override the default parse_line() method to also check for the - shortname of the host derived from the hostname. - """ -+ -+ def _check_line(ln, count, search, repl=None): -+ """Perform a second manual check for substrings that may have been -+ missed by regex matching -+ """ -+ if search in self.mapping.skip_keys: -+ return ln, count -+ if search in ln: -+ count += ln.count(search) -+ ln = ln.replace(search, self.mapping.get(repl or search)) -+ return ln, count -+ - count = 0 - line, count = super(SoSHostnameParser, self).parse_line(line) -- for short_name in self.short_names: -- if short_name in line: -- count += 1 -- line = line.replace(short_name, self.mapping.get(short_name)) -+ # make an additional pass checking for '_' formatted substrings that -+ # the regex patterns won't catch -+ hosts = [h for h in self.mapping.dataset.keys() if '.' in h] -+ for host in sorted(hosts, reverse=True, key=lambda x: len(x)): -+ fqdn = host -+ for c in '.-': -+ fqdn = fqdn.replace(c, '_') -+ line, count = _check_line(line, count, fqdn, host) -+ -+ for short_name in sorted(self.short_names, reverse=True): -+ line, count = _check_line(line, count, short_name) -+ - return line, count --- -2.26.3 - - -From ec46e6a8fac58ed757344be3751eb1f925eab981 Mon Sep 17 00:00:00 2001 -From: Jake Hunsaker -Date: Mon, 14 Jun 2021 09:31:07 -0400 -Subject: [PATCH 08/10] [ocp] Refine OCP node options in cluster profile - -Adds explicit setting of primary/node sos options for the `openshift` -plugin within the cluster, rather than relying on default configurations -and best practices to avoid duplicate collections. - -Signed-off-by: Jake Hunsaker ---- - sos/collector/clusters/ocp.py | 65 +++++++++++++++++++++++++++++++++-- - sos/collector/sosnode.py | 4 +-- - 2 files changed, 65 insertions(+), 4 deletions(-) - -diff --git a/sos/collector/clusters/ocp.py b/sos/collector/clusters/ocp.py -index 283fcfd1..ddff84a4 100644 ---- a/sos/collector/clusters/ocp.py -+++ b/sos/collector/clusters/ocp.py -@@ -8,6 +8,8 @@ - # - # See the LICENSE file in the source distribution for further information. - -+import os -+ - from pipes import quote - from sos.collector.clusters import Cluster - -@@ -18,10 +20,14 @@ class ocp(Cluster): - cluster_name = 'OpenShift Container Platform v4' - packages = ('openshift-hyperkube', 'openshift-clients') - -+ api_collect_enabled = False -+ token = None -+ - option_list = [ - ('label', '', 'Colon delimited list of labels to select nodes with'), - ('role', '', 'Colon delimited list of roles to select nodes with'), -- ('kubeconfig', '', 'Path to the kubeconfig file') -+ ('kubeconfig', '', 'Path to the kubeconfig file'), -+ ('token', '', 'Service account token to use for oc authorization') - ] - - def fmt_oc_cmd(self, cmd): -@@ -32,9 +38,20 @@ class ocp(Cluster): - return "oc --config %s %s" % (self.get_option('kubeconfig'), cmd) - return "oc %s" % cmd - -+ def _attempt_oc_login(self): -+ """Attempt to login to the API using the oc command using a provided -+ token -+ """ -+ _res = self.exec_primary_cmd("oc login --insecure-skip-tls-verify=True" -+ " --token=%s" % self.token) -+ return _res['status'] == 0 -+ - def check_enabled(self): - if super(ocp, self).check_enabled(): - return True -+ self.token = self.get_option('token') or os.getenv('SOSOCPTOKEN', None) -+ if self.token: -+ self._attempt_oc_login() - _who = self.fmt_oc_cmd('whoami') - return self.exec_master_cmd(_who)['status'] == 0 - -@@ -106,4 +123,48 @@ class ocp(Cluster): - return 'master' in self.node_dict[sosnode.address]['roles'] - - def set_master_options(self, node): -- node.opts.enable_plugins.append('openshift') -+ node.enable_plugins.append('openshift') -+ if self.api_collect_enabled: -+ # a primary has already been enabled for API collection, disable -+ # it among others -+ node.plugin_options.append('openshift.no-oc=on') -+ else: -+ _oc_cmd = 'oc' -+ if node.host.containerized: -+ _oc_cmd = '/host/bin/oc' -+ # when run from a container, the oc command does not inherit -+ # the default config, so if it's present then pass it here to -+ # detect a funcitonal oc command. This is sidestepped in sos -+ # report by being able to chroot the `oc` execution which we -+ # cannot do remotely -+ if node.file_exists('/root/.kube/config', need_root=True): -+ _oc_cmd += ' --kubeconfig /host/root/.kube/config' -+ can_oc = node.run_command("%s whoami" % _oc_cmd, -+ use_container=node.host.containerized, -+ # container is available only to root -+ # and if rhel, need to run sos as root -+ # anyways which will run oc as root -+ need_root=True) -+ if can_oc['status'] == 0: -+ # the primary node can already access the API -+ self.api_collect_enabled = True -+ elif self.token: -+ node.sos_env_vars['SOSOCPTOKEN'] = self.token -+ self.api_collect_enabled = True -+ elif self.get_option('kubeconfig'): -+ kc = self.get_option('kubeconfig') -+ if node.file_exists(kc): -+ if node.host.containerized: -+ kc = "/host/%s" % kc -+ node.sos_env_vars['KUBECONFIG'] = kc -+ self.api_collect_enabled = True -+ if self.api_collect_enabled: -+ msg = ("API collections will be performed on %s\nNote: API " -+ "collections may extend runtime by 10s of minutes\n" -+ % node.address) -+ self.soslog.info(msg) -+ self.ui_log.info(msg) -+ -+ def set_node_options(self, node): -+ # don't attempt OC API collections on non-primary nodes -+ node.plugin_options.append('openshift.no-oc=on') -diff --git a/sos/collector/sosnode.py b/sos/collector/sosnode.py -index 1c25cc34..6597d236 100644 ---- a/sos/collector/sosnode.py -+++ b/sos/collector/sosnode.py -@@ -202,11 +202,11 @@ class SosNode(): - self.opts.registry_authfile or self.host.container_authfile - ) - -- def file_exists(self, fname): -+ def file_exists(self, fname, need_root=False): - """Checks for the presence of fname on the remote node""" - if not self.local: - try: -- res = self.run_command("stat %s" % fname) -+ res = self.run_command("stat %s" % fname, need_root=need_root) - return res['status'] == 0 - except Exception: - return False --- -2.26.3 - - -From eea8e15845a8bcba91b93a5310ba693e8c20ab9c Mon Sep 17 00:00:00 2001 -From: Jake Hunsaker -Date: Thu, 17 Jun 2021 09:52:36 -0400 -Subject: [PATCH 09/10] [cleaner] Don't obfuscate default 'core' user - -The 'core' user is a common default user on containerized hosts, and -obfuscation of it is not advantageous, much like the default 'ubuntu' -user for that distribution. - -Signed-off-by: Jake Hunsaker ---- - sos/cleaner/parsers/username_parser.py | 1 + - 1 file changed, 1 insertion(+) - -diff --git a/sos/cleaner/parsers/username_parser.py b/sos/cleaner/parsers/username_parser.py -index 0c3bbac4..64843205 100644 ---- a/sos/cleaner/parsers/username_parser.py -+++ b/sos/cleaner/parsers/username_parser.py -@@ -28,6 +28,7 @@ class SoSUsernameParser(SoSCleanerParser): - prep_map_file = 'sos_commands/login/lastlog_-u_1000-60000' - regex_patterns = [] - skip_list = [ -+ 'core', - 'nobody', - 'nfsnobody', - 'root' --- -2.26.3 - - -From 581429ca65131711c96f9d56bf2f0e18779aec2e Mon Sep 17 00:00:00 2001 -From: Jake Hunsaker -Date: Fri, 18 Jun 2021 14:26:55 -0400 -Subject: [PATCH 10/10] [cleaner] Fix checksum and archive pruning from archive - list - -Fixes an issue where checksums may have gotten into the list of archives -to be cleaned, which would cause further issues later. Additionally, -prevents nested sosreports from top-level archives (such as from -`collect`) from being removed for being a binary file when that -top-level archive gets obfuscated. ---- - sos/cleaner/__init__.py | 5 +++-- - sos/cleaner/obfuscation_archive.py | 1 + - 2 files changed, 4 insertions(+), 2 deletions(-) - -diff --git a/sos/cleaner/__init__.py b/sos/cleaner/__init__.py -index 88d4d0ea..8280bc50 100644 ---- a/sos/cleaner/__init__.py -+++ b/sos/cleaner/__init__.py -@@ -226,8 +226,7 @@ third party. - nested_archives = [] - for _file in archive.getmembers(): - if (re.match('sosreport-.*.tar', _file.name.split('/')[-1]) and not -- (_file.name.endswith('.md5') or -- _file.name.endswith('.sha256'))): -+ (_file.name.endswith(('.md5', '.sha256')))): - nested_archives.append(_file.name.split('/')[-1]) - - if nested_archives: -@@ -235,6 +234,8 @@ third party. - nested_path = self.extract_archive(archive) - for arc_file in os.listdir(nested_path): - if re.match('sosreport.*.tar.*', arc_file): -+ if arc_file.endswith(('.md5', '.sha256')): -+ continue - self.report_paths.append(os.path.join(nested_path, - arc_file)) - # add the toplevel extracted archive -diff --git a/sos/cleaner/obfuscation_archive.py b/sos/cleaner/obfuscation_archive.py -index 90188358..e357450b 100644 ---- a/sos/cleaner/obfuscation_archive.py -+++ b/sos/cleaner/obfuscation_archive.py -@@ -58,6 +58,7 @@ class SoSObfuscationArchive(): - Returns: list of files and file regexes - """ - return [ -+ 'sosreport-', - 'sys/firmware', - 'sys/fs', - 'sys/kernel/debug', --- -2.26.3 - diff --git a/sos-bz1985985-sos-log-effective-options.patch b/sos-bz1985985-sos-log-effective-options.patch deleted file mode 100644 index 120df02..0000000 --- a/sos-bz1985985-sos-log-effective-options.patch +++ /dev/null @@ -1,284 +0,0 @@ -From 00d12ad3cf24dcc6c73e9bcf63db1d3f17e58bb1 Mon Sep 17 00:00:00 2001 -From: Jake Hunsaker -Date: Thu, 1 Jul 2021 10:50:54 -0400 -Subject: [PATCH] [sosnode] Properly format skip-commands and skip-files on - nodes - -Fixes an issue where options provided for `skip-commands` and -`skip-files` were not properly formatted, thus causing an exception -during the finalization of the node's sos command. - -Signed-off-by: Jake Hunsaker ---- - sos/collector/sosnode.py | 5 +++-- - 1 file changed, 3 insertions(+), 2 deletions(-) - -diff --git a/sos/collector/sosnode.py b/sos/collector/sosnode.py -index 6597d236..426edcba 100644 ---- a/sos/collector/sosnode.py -+++ b/sos/collector/sosnode.py -@@ -734,11 +734,12 @@ class SosNode(): - if self.check_sos_version('4.1'): - if self.opts.skip_commands: - sos_opts.append( -- '--skip-commands=%s' % (quote(self.opts.skip_commands)) -+ '--skip-commands=%s' % ( -+ quote(','.join(self.opts.skip_commands))) - ) - if self.opts.skip_files: - sos_opts.append( -- '--skip-files=%s' % (quote(self.opts.skip_files)) -+ '--skip-files=%s' % (quote(','.join(self.opts.skip_files))) - ) - - if self.check_sos_version('4.2'): --- -2.31.1 - -From de7edce3f92ed50abcb28dd0dbcbeb104dc7c679 Mon Sep 17 00:00:00 2001 -From: Pavel Moravec -Date: Fri, 2 Jul 2021 09:52:11 +0200 -Subject: [PATCH] [collector] fix a typo in --plugin-option - -Sos report uses --plugin-option or --plugopts. - -Relevant: #2606 - -Signed-off-by: Pavel Moravec ---- - sos/collector/__init__.py | 2 +- - 1 file changed, 1 insertion(+), 1 deletion(-) - -diff --git a/sos/collector/__init__.py b/sos/collector/__init__.py -index 6d96d692..f072287e 100644 ---- a/sos/collector/__init__.py -+++ b/sos/collector/__init__.py -@@ -272,7 +272,7 @@ class SoSCollector(SoSComponent): - help="chroot executed commands to SYSROOT") - sos_grp.add_argument('-e', '--enable-plugins', action="extend", - help='Enable specific plugins for sosreport') -- sos_grp.add_argument('-k', '--plugin-options', action="extend", -+ sos_grp.add_argument('-k', '--plugin-option', action="extend", - help='Plugin option as plugname.option=value') - sos_grp.add_argument('--log-size', default=0, type=int, - help='Limit the size of individual logs (in MiB)') --- -2.31.1 - -From 24a79ae8df8f29276f6139c68d4ba9b05114f951 Mon Sep 17 00:00:00 2001 -From: Pavel Moravec -Date: Fri, 2 Jul 2021 09:53:47 +0200 -Subject: [PATCH] [options] allow variant option names in config file - -While cmdline allows --plugin-option as well as --plugopts, -it stores the value under `plugopts` key. Therefore parsing -config file ignores --plugin-option. - -Similarly for --name/--label and --profile/--profiles. - -When processing config file, we must unify those potentially duplicit -keys. - -Resolves: #2606 - -Signed-off-by: Pavel Moravec ---- - sos/options.py | 9 +++++++++ - 1 file changed, 9 insertions(+) - -diff --git a/sos/options.py b/sos/options.py -index 1eda55d6..a014a022 100644 ---- a/sos/options.py -+++ b/sos/options.py -@@ -186,9 +186,18 @@ class SoSOptions(): - if 'verbose' in odict.keys(): - odict['verbosity'] = int(odict.pop('verbose')) - # convert options names -+ # unify some of them if multiple variants of the -+ # cmdoption exist -+ rename_opts = { -+ 'name': 'label', -+ 'plugin_option': 'plugopts', -+ 'profile': 'profiles' -+ } - for key in list(odict): - if '-' in key: - odict[key.replace('-', '_')] = odict.pop(key) -+ if key in rename_opts: -+ odict[rename_opts[key]] = odict.pop(key) - # set the values according to the config file - for key, val in odict.items(): - if isinstance(val, str): --- -2.31.1 - -From c7d3644c0c64e9e5439806250592a55c8e2de26f Mon Sep 17 00:00:00 2001 -From: Pavel Moravec -Date: Thu, 1 Jul 2021 08:11:15 +0200 -Subject: [PATCH] [report,collect] unify --map-file arguments - -Unify --map[-file] argument among report/collect/clean. - -Resolves: #2602 - -Signed-off-by: Pavel Moravec ---- - sos/cleaner/__init__.py | 2 +- - sos/collector/__init__.py | 2 +- - sos/report/__init__.py | 2 +- - 3 files changed, 3 insertions(+), 3 deletions(-) - -diff --git a/sos/cleaner/__init__.py b/sos/cleaner/__init__.py -index 7414b55e0..4c9837826 100644 ---- a/sos/cleaner/__init__.py -+++ b/sos/cleaner/__init__.py -@@ -192,7 +192,7 @@ def add_parser_options(cls, parser): - 'file for obfuscation')) - clean_grp.add_argument('--no-update', dest='no_update', default=False, - action='store_true', -- help='Do not update the --map file with new ' -+ help='Do not update the --map-file with new ' - 'mappings from this run') - clean_grp.add_argument('--keep-binary-files', default=False, - action='store_true', -diff --git a/sos/collector/__init__.py b/sos/collector/__init__.py -index 7b8cfcf72..6d96d6923 100644 ---- a/sos/collector/__init__.py -+++ b/sos/collector/__init__.py -@@ -427,7 +427,7 @@ def add_parser_options(cls, parser): - cleaner_grp.add_argument('--no-update', action='store_true', - default=False, dest='no_update', - help='Do not update the default cleaner map') -- cleaner_grp.add_argument('--map', dest='map_file', -+ cleaner_grp.add_argument('--map-file', dest='map_file', - default='/etc/sos/cleaner/default_mapping', - help=('Provide a previously generated mapping' - ' file for obfuscation')) -diff --git a/sos/report/__init__.py b/sos/report/__init__.py -index 7ad2d24a4..411c4eb03 100644 ---- a/sos/report/__init__.py -+++ b/sos/report/__init__.py -@@ -341,7 +341,7 @@ def add_parser_options(cls, parser): - cleaner_grp.add_argument('--no-update', action='store_true', - default=False, dest='no_update', - help='Do not update the default cleaner map') -- cleaner_grp.add_argument('--map', dest='map_file', -+ cleaner_grp.add_argument('--map-file', dest='map_file', - default='/etc/sos/cleaner/default_mapping', - help=('Provide a previously generated mapping' - ' file for obfuscation')) -From fd75745e7a5a6c5def8e6d23190227872b9912c3 Mon Sep 17 00:00:00 2001 -From: Jake Hunsaker -Date: Wed, 11 Aug 2021 10:48:41 -0400 -Subject: [PATCH] [sosnode] Fix passing of plugin options when using - `--only-plugins` - -Fixes the handling of plugin options passed by `sos collect` to each -node by first aligning the SoSOption name to those of `report` -(`plugopts`), and second re-arranges the handling of plugin options and -preset options passed by the user when also using `--only-plugins` so -that the former are preserved and passed only with the `--only-plugins` -option value. - -Resolves: #2641 - -Signed-off-by: Jake Hunsaker ---- - sos/collector/__init__.py | 5 +++-- - sos/collector/sosnode.py | 34 +++++++++++++++++----------------- - 2 files changed, 20 insertions(+), 19 deletions(-) - -diff --git a/sos/collector/__init__.py b/sos/collector/__init__.py -index 57ef074e..70b7a69e 100644 ---- a/sos/collector/__init__.py -+++ b/sos/collector/__init__.py -@@ -84,7 +84,7 @@ class SoSCollector(SoSComponent): - 'only_plugins': [], - 'password': False, - 'password_per_node': False, -- 'plugin_options': [], -+ 'plugopts': [], - 'plugin_timeout': None, - 'cmd_timeout': None, - 'preset': '', -@@ -273,7 +273,8 @@ class SoSCollector(SoSComponent): - help="chroot executed commands to SYSROOT") - sos_grp.add_argument('-e', '--enable-plugins', action="extend", - help='Enable specific plugins for sosreport') -- sos_grp.add_argument('-k', '--plugin-option', action="extend", -+ sos_grp.add_argument('-k', '--plugin-option', '--plugopts', -+ action="extend", dest='plugopts', - help='Plugin option as plugname.option=value') - sos_grp.add_argument('--log-size', default=0, type=int, - help='Limit the size of individual logs (in MiB)') -diff --git a/sos/collector/sosnode.py b/sos/collector/sosnode.py -index 426edcba..5d05c297 100644 ---- a/sos/collector/sosnode.py -+++ b/sos/collector/sosnode.py -@@ -667,10 +667,10 @@ class SosNode(): - - if self.cluster.sos_plugin_options: - for opt in self.cluster.sos_plugin_options: -- if not any(opt in o for o in self.plugin_options): -+ if not any(opt in o for o in self.plugopts): - option = '%s=%s' % (opt, - self.cluster.sos_plugin_options[opt]) -- self.plugin_options.append(option) -+ self.plugopts.append(option) - - # set master-only options - if self.cluster.check_node_is_master(self): -@@ -688,7 +688,7 @@ class SosNode(): - self.only_plugins = list(self.opts.only_plugins) - self.skip_plugins = list(self.opts.skip_plugins) - self.enable_plugins = list(self.opts.enable_plugins) -- self.plugin_options = list(self.opts.plugin_options) -+ self.plugopts = list(self.opts.plugopts) - self.preset = list(self.opts.preset) - - def finalize_sos_cmd(self): -@@ -754,6 +754,20 @@ class SosNode(): - os.path.join(self.host.sos_bin_path, self.sos_bin) - ) - -+ if self.plugopts: -+ opts = [o for o in self.plugopts -+ if self._plugin_exists(o.split('.')[0]) -+ and self._plugin_option_exists(o.split('=')[0])] -+ if opts: -+ sos_opts.append('-k %s' % quote(','.join(o for o in opts))) -+ -+ if self.preset: -+ if self._preset_exists(self.preset): -+ sos_opts.append('--preset=%s' % quote(self.preset)) -+ else: -+ self.log_debug('Requested to enable preset %s but preset does ' -+ 'not exist on node' % self.preset) -+ - if self.only_plugins: - plugs = [o for o in self.only_plugins if self._plugin_exists(o)] - if len(plugs) != len(self.only_plugins): -@@ -792,20 +806,6 @@ class SosNode(): - if enable: - sos_opts.append('--enable-plugins=%s' % quote(enable)) - -- if self.plugin_options: -- opts = [o for o in self.plugin_options -- if self._plugin_exists(o.split('.')[0]) -- and self._plugin_option_exists(o.split('=')[0])] -- if opts: -- sos_opts.append('-k %s' % quote(','.join(o for o in opts))) -- -- if self.preset: -- if self._preset_exists(self.preset): -- sos_opts.append('--preset=%s' % quote(self.preset)) -- else: -- self.log_debug('Requested to enable preset %s but preset does ' -- 'not exist on node' % self.preset) -- - self.sos_cmd = "%s %s" % (sos_cmd, ' '.join(sos_opts)) - self.log_info('Final sos command set to %s' % self.sos_cmd) - self.manifest.add_field('final_sos_command', self.sos_cmd) --- -2.31.1 - diff --git a/sos-bz1992859-rhui-plugin.patch b/sos-bz1992859-rhui-plugin.patch deleted file mode 100644 index e884396..0000000 --- a/sos-bz1992859-rhui-plugin.patch +++ /dev/null @@ -1,387 +0,0 @@ -From 94b9b90c818eb18f0ca8d78fe063dc5b0677c885 Mon Sep 17 00:00:00 2001 -From: Pavel Moravec -Date: Tue, 22 Jun 2021 12:58:03 +0200 -Subject: [PATCH] [rhui] add plugin to RHUI - -Add a new/revoked plugin for RHUI (newly based on python3 and pulp-3). - -Edditionally, collect /etc/pki/pulp certificates except for RSA keys. - -Resolves: #2590 - -Signed-off-by: Pavel Moravec ---- - sos/report/plugins/pulpcore.py | 7 ++++- - sos/report/plugins/rhui.py | 49 ++++++++++++++++++++++++++++++++++ - 2 files changed, 55 insertions(+), 1 deletion(-) - create mode 100644 sos/report/plugins/rhui.py - -diff --git a/sos/report/plugins/pulpcore.py b/sos/report/plugins/pulpcore.py -index ccaac3185..77ceacb92 100644 ---- a/sos/report/plugins/pulpcore.py -+++ b/sos/report/plugins/pulpcore.py -@@ -77,7 +77,12 @@ def separate_value(line, sep=':'): - def setup(self): - self.parse_settings_config() - -- self.add_copy_spec("/etc/pulp/settings.py") -+ self.add_copy_spec([ -+ "/etc/pulp/settings.py", -+ "/etc/pki/pulp/*" -+ ]) -+ # skip collecting certificate keys -+ self.add_forbidden_path("/etc/pki/pulp/*.key") - - self.add_cmd_output("rq info -u redis://localhost:6379/8", - env={"LC_ALL": "en_US.UTF-8"}, -diff --git a/sos/report/plugins/rhui.py b/sos/report/plugins/rhui.py -new file mode 100644 -index 000000000..7acd3f49e ---- /dev/null -+++ b/sos/report/plugins/rhui.py -@@ -0,0 +1,49 @@ -+# Copyright (C) 2021 Red Hat, Inc., Pavel Moravec -+ -+# This file is part of the sos project: https://github.com/sosreport/sos -+# -+# This copyrighted material is made available to anyone wishing to use, -+# modify, copy, or redistribute it subject to the terms and conditions of -+# version 2 of the GNU General Public License. -+# -+# See the LICENSE file in the source distribution for further information. -+ -+from sos.report.plugins import Plugin, RedHatPlugin -+ -+ -+class Rhui(Plugin, RedHatPlugin): -+ -+ short_desc = 'Red Hat Update Infrastructure' -+ -+ plugin_name = "rhui" -+ commands = ("rhui-manager",) -+ files = ("/etc/ansible/facts.d/rhui_auth.fact", "/usr/lib/rhui/cds.py") -+ -+ def setup(self): -+ self.add_copy_spec([ -+ "/etc/rhui/rhui-tools.conf", -+ "/etc/rhui/registered_subscriptions.conf", -+ "/etc/pki/rhui/*", -+ "/var/log/rhui-subscription-sync.log", -+ "/var/cache/rhui/*", -+ "/root/.rhui/*", -+ ]) -+ # skip collecting certificate keys -+ self.add_forbidden_path("/etc/pki/rhui/*.key") -+ -+ self.add_cmd_output([ -+ "rhui-manager status", -+ "rhui-manager cert info", -+ "ls -lR /var/lib/rhui/remote_share", -+ ]) -+ -+ def postproc(self): -+ # obfuscate admin_pw and secret_key values -+ for prop in ["admin_pw", "secret_key"]: -+ self.do_path_regex_sub( -+ "/etc/ansible/facts.d/rhui_auth.fact", -+ r"(%s\s*=\s*)(.*)" % prop, -+ r"\1********") -+ -+ -+# vim: set et ts=4 sw=4 : -From bd15dc764c9d4554d8e8f08163228d65ca099985 Mon Sep 17 00:00:00 2001 -From: Pavel Moravec -Date: Thu, 24 Jun 2021 17:53:27 +0200 -Subject: [PATCH 1/4] [plugins] Allow add_forbidden_path to apply glob - recursively - -Add option to apply glob.glob to forbidden path recursively. - -Signed-off-by: Pavel Moravec ---- - sos/report/plugins/__init__.py | 6 ++++-- - 1 file changed, 4 insertions(+), 2 deletions(-) - -diff --git a/sos/report/plugins/__init__.py b/sos/report/plugins/__init__.py -index 06923300..6fd1a3b2 100644 ---- a/sos/report/plugins/__init__.py -+++ b/sos/report/plugins/__init__.py -@@ -1187,12 +1187,14 @@ class Plugin(object): - 'symlink': "no" - }) - -- def add_forbidden_path(self, forbidden): -+ def add_forbidden_path(self, forbidden, recursive=False): - """Specify a path, or list of paths, to not copy, even if it's part of - an ``add_copy_spec()`` call - - :param forbidden: A filepath to forbid collection from - :type forbidden: ``str`` or a ``list`` of strings -+ -+ :param recursive: Should forbidden glob be applied recursively - """ - if isinstance(forbidden, str): - forbidden = [forbidden] -@@ -1202,7 +1204,7 @@ class Plugin(object): - - for forbid in forbidden: - self._log_info("adding forbidden path '%s'" % forbid) -- for path in glob.glob(forbid): -+ for path in glob.glob(forbid, recursive=recursive): - self.forbidden_paths.append(path) - - def get_all_options(self): --- -2.31.1 - - -From b695201baeb629a6543445d98dbb04f357670621 Mon Sep 17 00:00:00 2001 -From: Pavel Moravec -Date: Thu, 24 Jun 2021 17:57:48 +0200 -Subject: [PATCH 2/4] [pulpcore] improve settings.py parsing - -- deal with /etc/pulp/settings.py as a one-line string -- parse dbname from it as well -- dont collect any *.key file from whole /etc/pki/pulp dir - -Related: #2593 - -Signed-off-by: Pavel Moravec ---- - sos/report/plugins/pulpcore.py | 23 +++++++++++++++-------- - 1 file changed, 15 insertions(+), 8 deletions(-) - -diff --git a/sos/report/plugins/pulpcore.py b/sos/report/plugins/pulpcore.py -index 77ceacb9..be526035 100644 ---- a/sos/report/plugins/pulpcore.py -+++ b/sos/report/plugins/pulpcore.py -@@ -28,9 +28,10 @@ class PulpCore(Plugin, IndependentPlugin): - databases_scope = False - self.dbhost = "localhost" - self.dbport = 5432 -+ self.dbname = "pulpcore" - self.dbpasswd = "" - # TODO: read also redis config (we dont expect much customisations) -- # TODO: read also db user (pulp) and database name (pulpcore) -+ # TODO: read also db user (pulp) - self.staticroot = "/var/lib/pulp/assets" - self.uploaddir = "/var/lib/pulp/media/upload" - -@@ -44,7 +45,10 @@ class PulpCore(Plugin, IndependentPlugin): - return val - - try: -- for line in open("/etc/pulp/settings.py").read().splitlines(): -+ # split the lines to "one option per line" format -+ for line in open("/etc/pulp/settings.py").read() \ -+ .replace(',', ',\n').replace('{', '{\n') \ -+ .replace('}', '\n}').splitlines(): - # skip empty lines and lines with comments - if not line or line[0] == '#': - continue -@@ -53,11 +57,14 @@ class PulpCore(Plugin, IndependentPlugin): - continue - # example HOST line to parse: - # 'HOST': 'localhost', -- if databases_scope and match(r"\s+'HOST'\s*:\s+\S+", line): -+ pattern = r"\s*['|\"]%s['|\"]\s*:\s*\S+" -+ if databases_scope and match(pattern % 'HOST', line): - self.dbhost = separate_value(line) -- if databases_scope and match(r"\s+'PORT'\s*:\s+\S+", line): -+ if databases_scope and match(pattern % 'PORT', line): - self.dbport = separate_value(line) -- if databases_scope and match(r"\s+'PASSWORD'\s*:\s+\S+", line): -+ if databases_scope and match(pattern % 'NAME', line): -+ self.dbname = separate_value(line) -+ if databases_scope and match(pattern % 'PASSWORD', line): - self.dbpasswd = separate_value(line) - # if line contains closing '}' database_scope end - if databases_scope and '}' in line: -@@ -82,7 +89,7 @@ class PulpCore(Plugin, IndependentPlugin): - "/etc/pki/pulp/*" - ]) - # skip collecting certificate keys -- self.add_forbidden_path("/etc/pki/pulp/*.key") -+ self.add_forbidden_path("/etc/pki/pulp/**/*.key", recursive=True) - - self.add_cmd_output("rq info -u redis://localhost:6379/8", - env={"LC_ALL": "en_US.UTF-8"}, -@@ -104,8 +111,8 @@ class PulpCore(Plugin, IndependentPlugin): - _query = "select * from %s where pulp_last_updated > NOW() - " \ - "interval '%s days' order by pulp_last_updated" % \ - (table, task_days) -- _cmd = "psql -h %s -p %s -U pulp -d pulpcore -c %s" % \ -- (self.dbhost, self.dbport, quote(_query)) -+ _cmd = "psql -h %s -p %s -U pulp -d %s -c %s" % \ -+ (self.dbhost, self.dbport, self.dbname, quote(_query)) - self.add_cmd_output(_cmd, env=self.env, suggest_filename=table) - - def postproc(self): --- -2.31.1 - - -From 0286034da44bce43ab368dfc6815da7d74d60719 Mon Sep 17 00:00:00 2001 -From: Pavel Moravec -Date: Thu, 24 Jun 2021 17:59:36 +0200 -Subject: [PATCH 3/4] [rhui] call rhui-* commands with proper env and timeout - -rhui-manager commands timeout when not being logged in, which -should be reacted by adding proper cmd timeout. - -Adding the env.variable ensures potentially unaswered "RHUI Username:" -is also printed/colected. - -Further, prevent collecting any *.key file from the whole /etc/pki/rhui -dir. - -Related: #2593 - -Signed-off-by: Pavel Moravec ---- - sos/report/plugins/rhui.py | 7 +++++-- - 1 file changed, 5 insertions(+), 2 deletions(-) - -diff --git a/sos/report/plugins/rhui.py b/sos/report/plugins/rhui.py -index 7acd3f49..5a152427 100644 ---- a/sos/report/plugins/rhui.py -+++ b/sos/report/plugins/rhui.py -@@ -29,13 +29,16 @@ class Rhui(Plugin, RedHatPlugin): - "/root/.rhui/*", - ]) - # skip collecting certificate keys -- self.add_forbidden_path("/etc/pki/rhui/*.key") -+ self.add_forbidden_path("/etc/pki/rhui/**/*.key", recursive=True) - -+ # call rhui-manager commands with 1m timeout and -+ # with an env. variable ensuring that "RHUI Username:" -+ # even unanswered prompt gets collected - self.add_cmd_output([ - "rhui-manager status", - "rhui-manager cert info", - "ls -lR /var/lib/rhui/remote_share", -- ]) -+ ], timeout=60, env={'PYTHONUNBUFFERED': '1'}) - - def postproc(self): - # obfuscate admin_pw and secret_key values --- -2.31.1 - - -From a656bd239ab86dfd8973f733ae2c0fbd0c57d416 Mon Sep 17 00:00:00 2001 -From: Pavel Moravec -Date: Thu, 24 Jun 2021 18:01:14 +0200 -Subject: [PATCH 4/4] [rhui] fix broken obfuscation - -- /etc/ansible/facts.d/rhui_*.fact must be collected by -rhui plugin to let some file to be obfuscated there -- obfuscate also cookies values that can grant login access - -Resolves: #2593 - -Signed-off-by: Pavel Moravec ---- - sos/report/plugins/ansible.py | 3 +++ - sos/report/plugins/rhui.py | 7 +++++++ - 2 files changed, 10 insertions(+) - -diff --git a/sos/report/plugins/ansible.py b/sos/report/plugins/ansible.py -index 3e5d3d37..5991b786 100644 ---- a/sos/report/plugins/ansible.py -+++ b/sos/report/plugins/ansible.py -@@ -29,4 +29,7 @@ class Ansible(Plugin, RedHatPlugin, UbuntuPlugin): - "ansible --version" - ]) - -+ # let rhui plugin collects the RHUI specific files -+ self.add_forbidden_path("/etc/ansible/facts.d/rhui_*.fact") -+ - # vim: set et ts=4 sw=4 : -diff --git a/sos/report/plugins/rhui.py b/sos/report/plugins/rhui.py -index 5a152427..1d479f85 100644 ---- a/sos/report/plugins/rhui.py -+++ b/sos/report/plugins/rhui.py -@@ -27,6 +27,7 @@ class Rhui(Plugin, RedHatPlugin): - "/var/log/rhui-subscription-sync.log", - "/var/cache/rhui/*", - "/root/.rhui/*", -+ "/etc/ansible/facts.d/rhui_*.fact", - ]) - # skip collecting certificate keys - self.add_forbidden_path("/etc/pki/rhui/**/*.key", recursive=True) -@@ -47,6 +48,12 @@ class Rhui(Plugin, RedHatPlugin): - "/etc/ansible/facts.d/rhui_auth.fact", - r"(%s\s*=\s*)(.*)" % prop, - r"\1********") -+ # obfuscate twoo cookies for login session -+ for cookie in ["csrftoken", "sessionid"]: -+ self.do_path_regex_sub( -+ r"/root/\.rhui/.*/cookies.txt", -+ r"(%s\s+)(\S+)" % cookie, -+ r"\1********") - - - # vim: set et ts=4 sw=4 : --- -2.31.1 - -From 4e5bebffca9936bcdf4d38aad9989970a15dd72b Mon Sep 17 00:00:00 2001 -From: Pavel Moravec -Date: Tue, 3 Aug 2021 21:54:33 +0200 -Subject: [PATCH] [rhui] Update the plugin on several places - -- obfuscate "rhui_manager_password: xxx" in /root/.rhui/answers.yaml* -- no need to collect or obfuscate anything from /etc/ansible/facts.d -- newly detect the plugin via /etc/rhui/rhui-tools.conf file or rhui-manager - command (only) - -Resolves: #2637 - -Signed-off-by: Pavel Moravec ---- - sos/report/plugins/rhui.py | 14 ++++++-------- - 1 file changed, 6 insertions(+), 8 deletions(-) - -diff --git a/sos/report/plugins/rhui.py b/sos/report/plugins/rhui.py -index 1d479f85..52065fb4 100644 ---- a/sos/report/plugins/rhui.py -+++ b/sos/report/plugins/rhui.py -@@ -16,8 +16,8 @@ class Rhui(Plugin, RedHatPlugin): - short_desc = 'Red Hat Update Infrastructure' - - plugin_name = "rhui" -- commands = ("rhui-manager",) -- files = ("/etc/ansible/facts.d/rhui_auth.fact", "/usr/lib/rhui/cds.py") -+ commands = ("rhui-manager", ) -+ files = ("/etc/rhui/rhui-tools.conf", ) - - def setup(self): - self.add_copy_spec([ -@@ -27,7 +27,6 @@ class Rhui(Plugin, RedHatPlugin): - "/var/log/rhui-subscription-sync.log", - "/var/cache/rhui/*", - "/root/.rhui/*", -- "/etc/ansible/facts.d/rhui_*.fact", - ]) - # skip collecting certificate keys - self.add_forbidden_path("/etc/pki/rhui/**/*.key", recursive=True) -@@ -42,11 +41,10 @@ class Rhui(Plugin, RedHatPlugin): - ], timeout=60, env={'PYTHONUNBUFFERED': '1'}) - - def postproc(self): -- # obfuscate admin_pw and secret_key values -- for prop in ["admin_pw", "secret_key"]: -- self.do_path_regex_sub( -- "/etc/ansible/facts.d/rhui_auth.fact", -- r"(%s\s*=\s*)(.*)" % prop, -+ # hide rhui_manager_password value in (also rotated) answers file -+ self.do_path_regex_sub( -+ r"/root/\.rhui/answers.yaml.*", -+ r"(\s*rhui_manager_password\s*:)\s*(\S+)", - r"\1********") - # obfuscate twoo cookies for login session - for cookie in ["csrftoken", "sessionid"]: --- -2.31.1 - diff --git a/sos-bz1992861-cleaner-AD-users-obfuscation.patch b/sos-bz1992861-cleaner-AD-users-obfuscation.patch deleted file mode 100644 index 2e5835a..0000000 --- a/sos-bz1992861-cleaner-AD-users-obfuscation.patch +++ /dev/null @@ -1,142 +0,0 @@ -From 7e471676fe41dab155a939c60446cc7b7dab773b Mon Sep 17 00:00:00 2001 -From: Jake Hunsaker -Date: Tue, 20 Jul 2021 11:09:29 -0400 -Subject: [PATCH] [username parser] Load usernames from `last` for LDAP users - -AD/LDAP users are not reported into `lastlog` generally, however they -are reported in `last`. Conversely, `last` does not report local users -who have not logged in but still exist. - -In order to obfuscate both kinds of users, we need to look at both -sources. - -For this, first allow parsers to specify multiple prep files. Second, -update the username parser to search through all `lastlog` collections -as well as the `last` collection. - -Also includes a small update to the username parser's prep loading logic -to ensure we are iterating over each username discovered only once. - -Signed-off-by: Jake Hunsaker ---- - sos/cleaner/__init__.py | 38 ++++++++++++++------------ - sos/cleaner/parsers/__init__.py | 2 +- - sos/cleaner/parsers/username_parser.py | 24 +++++++++++++--- - 3 files changed, 42 insertions(+), 22 deletions(-) - -diff --git a/sos/cleaner/__init__.py b/sos/cleaner/__init__.py -index ca5f93e5..6aadfe79 100644 ---- a/sos/cleaner/__init__.py -+++ b/sos/cleaner/__init__.py -@@ -518,23 +518,27 @@ third party. - for _parser in self.parsers: - if not _parser.prep_map_file: - continue -- _arc_path = os.path.join(_arc_name, _parser.prep_map_file) -- try: -- if is_dir: -- _pfile = open(_arc_path, 'r') -- content = _pfile.read() -- else: -- _pfile = archive.extractfile(_arc_path) -- content = _pfile.read().decode('utf-8') -- _pfile.close() -- if isinstance(_parser, SoSUsernameParser): -- _parser.load_usernames_into_map(content) -- for line in content.splitlines(): -- if isinstance(_parser, SoSHostnameParser): -- _parser.load_hostname_into_map(line) -- self.obfuscate_line(line) -- except Exception as err: -- self.log_debug("Could not prep %s: %s" % (_arc_path, err)) -+ if isinstance(_parser.prep_map_file, str): -+ _parser.prep_map_file = [_parser.prep_map_file] -+ for parse_file in _parser.prep_map_file: -+ _arc_path = os.path.join(_arc_name, parse_file) -+ try: -+ if is_dir: -+ _pfile = open(_arc_path, 'r') -+ content = _pfile.read() -+ else: -+ _pfile = archive.extractfile(_arc_path) -+ content = _pfile.read().decode('utf-8') -+ _pfile.close() -+ if isinstance(_parser, SoSUsernameParser): -+ _parser.load_usernames_into_map(content) -+ for line in content.splitlines(): -+ if isinstance(_parser, SoSHostnameParser): -+ _parser.load_hostname_into_map(line) -+ self.obfuscate_line(line) -+ except Exception as err: -+ self.log_debug("Could not prep %s: %s" -+ % (_arc_path, err)) - - def obfuscate_report(self, report): - """Individually handle each archive or directory we've discovered by -diff --git a/sos/cleaner/parsers/__init__.py b/sos/cleaner/parsers/__init__.py -index 3076db39..af6e375e 100644 ---- a/sos/cleaner/parsers/__init__.py -+++ b/sos/cleaner/parsers/__init__.py -@@ -50,7 +50,7 @@ class SoSCleanerParser(): - skip_line_patterns = [] - skip_files = [] - map_file_key = 'unset' -- prep_map_file = 'unset' -+ prep_map_file = [] - - def __init__(self, conf_file=None): - # attempt to load previous run data into the mapping for the parser -diff --git a/sos/cleaner/parsers/username_parser.py b/sos/cleaner/parsers/username_parser.py -index 96ce5f0c..b142e371 100644 ---- a/sos/cleaner/parsers/username_parser.py -+++ b/sos/cleaner/parsers/username_parser.py -@@ -25,13 +25,24 @@ class SoSUsernameParser(SoSCleanerParser - - name = 'Username Parser' - map_file_key = 'username_map' -- prep_map_file = 'sos_commands/login/lastlog_-u_1000-60000' -+ prep_map_file = [ -+ 'sos_commands/login/lastlog_-u_1000-60000', -+ 'sos_commands/login/lastlog_-u_60001-65536', -+ 'sos_commands/login/lastlog_-u_65537-4294967295', -+ # AD users will be reported here, but favor the lastlog files since -+ # those will include local users who have not logged in -+ 'sos_commands/login/last' -+ ] - regex_patterns = [] - skip_list = [ - 'core', - 'nobody', - 'nfsnobody', -- 'root' -+ 'shutdown', -+ 'reboot', -+ 'root', -+ 'ubuntu', -+ 'wtmp' - ] - - def __init__(self, conf_file=None, opt_names=None): -@@ -44,11 +54,17 @@ class SoSUsernameParser(SoSCleanerParser): - """Since we don't get the list of usernames from a straight regex for - this parser, we need to override the initial parser prepping here. - """ -+ users = set() - for line in content.splitlines()[1:]: -- user = line.split()[0] -+ try: -+ user = line.split()[0] -+ except Exception: -+ continue - if user in self.skip_list: - continue -- self.mapping.get(user) -+ users.add(user) -+ for each in users: -+ self.mapping.get(each) - - def parse_line(self, line): - count = 0 --- -2.31.1 - diff --git a/sos-bz2011533-unpackaged-recursive-symlink.patch b/sos-bz2011533-unpackaged-recursive-symlink.patch new file mode 100644 index 0000000..35cc89d --- /dev/null +++ b/sos-bz2011533-unpackaged-recursive-symlink.patch @@ -0,0 +1,42 @@ +From e2ca3d02f36c0db4efaacfb2c1b7d502f38e371c Mon Sep 17 00:00:00 2001 +From: Pavel Moravec +Date: Mon, 30 Aug 2021 10:18:29 +0200 +Subject: [PATCH] [unpackaged] deal with recursive loop of symlinks properly + +When the plugin processes a recursive loop of symlinks, it currently +hangs in an infinite loop trying to follow the symlinks. Use +pathlib.Path.resolve() method to return the target directly. + +Resolves: #2664 + +Signed-off-by: Pavel Moravec +--- + sos/report/plugins/unpackaged.py | 5 +++-- + 1 file changed, 3 insertions(+), 2 deletions(-) + +diff --git a/sos/report/plugins/unpackaged.py b/sos/report/plugins/unpackaged.py +index e5cc6191..9d68077c 100644 +--- a/sos/report/plugins/unpackaged.py ++++ b/sos/report/plugins/unpackaged.py +@@ -10,6 +10,7 @@ from sos.report.plugins import Plugin, RedHatPlugin + + import os + import stat ++from pathlib import Path + + + class Unpackaged(Plugin, RedHatPlugin): +@@ -41,8 +42,8 @@ class Unpackaged(Plugin, RedHatPlugin): + for name in files: + path = os.path.join(root, name) + try: +- while stat.S_ISLNK(os.lstat(path).st_mode): +- path = os.path.abspath(os.readlink(path)) ++ if stat.S_ISLNK(os.lstat(path).st_mode): ++ path = Path(path).resolve() + except Exception: + continue + file_list.append(os.path.realpath(path)) +-- +2.31.1 + diff --git a/sos-bz2011534-opacapture-under-allow-system-changes.patch b/sos-bz2011534-opacapture-under-allow-system-changes.patch new file mode 100644 index 0000000..39f9c8a --- /dev/null +++ b/sos-bz2011534-opacapture-under-allow-system-changes.patch @@ -0,0 +1,49 @@ +From 66ebb8256b1326573cbcb2d134545635dfead3bc Mon Sep 17 00:00:00 2001 +From: Jose Castillo +Date: Sun, 29 Aug 2021 15:35:09 +0200 +Subject: [PATCH] [omnipath_client] Ensure opacapture runs only with + allow-system-changes + +While omnipath_client plugin is collecting "opacapture", +`depmod -a` command is executed to regenerates some files +under /usr/lib/modules/$kernel. + +modules.dep +modules.dep.bin +modules.devname +modules.softdep +modules.symbols +modules.symbols.bin + +This patch ensures that the command is only run when +the option --allow-system-changes is used. + +Fixes: RHBZ#1998433 + +Signed-off-by: Jose Castillo +--- + sos/report/plugins/omnipath_client.py | 9 +++++++-- + 1 file changed, 7 insertions(+), 2 deletions(-) + +diff --git a/sos/report/plugins/omnipath_client.py b/sos/report/plugins/omnipath_client.py +index 1ec01384..4e988c5c 100644 +--- a/sos/report/plugins/omnipath_client.py ++++ b/sos/report/plugins/omnipath_client.py +@@ -45,7 +45,12 @@ class OmnipathClient(Plugin, RedHatPlugin): + # rather than storing it somewhere under /var/tmp and copying it via + # add_copy_spec, add it directly to sos_commands/ dir by + # building a path argument using self.get_cmd_output_path(). +- self.add_cmd_output("opacapture %s" % join(self.get_cmd_output_path(), +- "opacapture.tgz")) ++ # This command calls 'depmod -a', so lets make sure we ++ # specified the 'allow-system-changes' option before running it. ++ if self.get_option('allow_system_changes'): ++ self.add_cmd_output("opacapture %s" % ++ join(self.get_cmd_output_path(), ++ "opacapture.tgz"), ++ changes=True) + + # vim: set et ts=4 sw=4 : +-- +2.31.1 + diff --git a/sos-bz2011535-kernel-psi.patch b/sos-bz2011535-kernel-psi.patch new file mode 100644 index 0000000..1a9d5e0 --- /dev/null +++ b/sos-bz2011535-kernel-psi.patch @@ -0,0 +1,33 @@ +From 23e523b6b9784390c7ce2c5af654ab497fb10aaf Mon Sep 17 00:00:00 2001 +From: Jose Castillo +Date: Wed, 8 Sep 2021 09:25:24 +0200 +Subject: [PATCH] [kernel] Capture Pressure Stall Information + +Kernel 4.20 includes PSI metrics for CPU, memeory and IO. +The feature is enabled after adding "psi=1" as +kernel boot parameter. +The information is captured in files +in the directory /proc/pressure. + +Signed-off-by: Jose Castillo +--- + sos/report/plugins/kernel.py | 3 ++- + 1 file changed, 2 insertions(+), 1 deletion(-) + +diff --git a/sos/report/plugins/kernel.py b/sos/report/plugins/kernel.py +index 8c5e5e11..803f5e30 100644 +--- a/sos/report/plugins/kernel.py ++++ b/sos/report/plugins/kernel.py +@@ -112,7 +112,8 @@ class Kernel(Plugin, IndependentPlugin): + "/sys/kernel/debug/extfrag/unusable_index", + "/sys/kernel/debug/extfrag/extfrag_index", + clocksource_path + "available_clocksource", +- clocksource_path + "current_clocksource" ++ clocksource_path + "current_clocksource", ++ "/proc/pressure/" + ]) + + if self.get_option("with-timer"): +-- +2.31.1 + diff --git a/sos-bz2011537-estimate-only-option.patch b/sos-bz2011537-estimate-only-option.patch new file mode 100644 index 0000000..eebde8f --- /dev/null +++ b/sos-bz2011537-estimate-only-option.patch @@ -0,0 +1,179 @@ +From 5b245b1e449c6a05d09034bcb8290bffded79327 Mon Sep 17 00:00:00 2001 +From: Pavel Moravec +Date: Wed, 8 Sep 2021 17:04:58 +0200 +Subject: [PATCH] [report] Implement --estimate-only + +Add report option --estimate-only to estimate disk space requirements +when running a sos report. + +Resolves: #2673 + +Signed-off-by: Pavel Moravec +--- + man/en/sos-report.1 | 13 +++++++- + sos/report/__init__.py | 74 ++++++++++++++++++++++++++++++++++++++++-- + 2 files changed, 84 insertions(+), 3 deletions(-) + +diff --git a/man/en/sos-report.1 b/man/en/sos-report.1 +index 36b337df..e8efc8f8 100644 +--- a/man/en/sos-report.1 ++++ b/man/en/sos-report.1 +@@ -14,7 +14,7 @@ sos report \- Collect and package diagnostic and support data + [--preset preset] [--add-preset add_preset]\fR + [--del-preset del_preset] [--desc description]\fR + [--batch] [--build] [--debug] [--dry-run]\fR +- [--label label] [--case-id id]\fR ++ [--estimate-only] [--label label] [--case-id id]\fR + [--threads threads]\fR + [--plugin-timeout TIMEOUT]\fR + [--cmd-timeout TIMEOUT]\fR +@@ -317,6 +317,17 @@ output, or string data from the system. The resulting logs may be used + to understand the actions that sos would have taken without the dry run + option. + .TP ++.B \--estimate-only ++Estimate disk space requirements when running sos report. This can be valuable ++to prevent sosreport working dir to consume all free disk space. No plugin data ++is available at the end. ++ ++Plugins will be collected sequentially, size of collected files and commands outputs ++will be calculated and the plugin files will be immediatelly deleted prior execution ++of the next plugin. This still can consume whole free disk space, though. Please note, ++size estimations may not be accurate for highly utilized systems due to changes between ++an estimate and a real execution. ++.TP + .B \--upload + If specified, attempt to upload the resulting archive to a vendor defined location. + +diff --git a/sos/report/__init__.py b/sos/report/__init__.py +index 82484f1d..b033f621 100644 +--- a/sos/report/__init__.py ++++ b/sos/report/__init__.py +@@ -86,6 +86,7 @@ class SoSReport(SoSComponent): + 'desc': '', + 'domains': [], + 'dry_run': False, ++ 'estimate_only': False, + 'experimental': False, + 'enable_plugins': [], + 'keywords': [], +@@ -137,6 +138,7 @@ class SoSReport(SoSComponent): + self._args = args + self.sysroot = "/" + self.preset = None ++ self.estimated_plugsizes = {} + + self.print_header() + self._set_debug() +@@ -223,6 +225,11 @@ class SoSReport(SoSComponent): + help="Description for a new preset",) + report_grp.add_argument("--dry-run", action="store_true", + help="Run plugins but do not collect data") ++ report_grp.add_argument("--estimate-only", action="store_true", ++ help="Approximate disk space requirements for " ++ "a real sos run; disables --clean and " ++ "--collect, sets --threads=1 and " ++ "--no-postproc") + report_grp.add_argument("--experimental", action="store_true", + dest="experimental", default=False, + help="enable experimental plugins") +@@ -700,6 +700,33 @@ class SoSReport(SoSComponent): + self.all_options.append((plugin, plugin_name, optname, + optparm)) + ++ def _set_estimate_only(self): ++ # set estimate-only mode by enforcing some options settings ++ # and return a corresponding log messages string ++ msg = "\nEstimate-only mode enabled" ++ ext_msg = [] ++ if self.opts.threads > 1: ++ ext_msg += ["--threads=%s overriden to 1" % self.opts.threads, ] ++ self.opts.threads = 1 ++ if not self.opts.build: ++ ext_msg += ["--build enabled", ] ++ self.opts.build = True ++ if not self.opts.no_postproc: ++ ext_msg += ["--no-postproc enabled", ] ++ self.opts.no_postproc = True ++ if self.opts.clean: ++ ext_msg += ["--clean disabled", ] ++ self.opts.clean = False ++ if self.opts.upload: ++ ext_msg += ["--upload* options disabled", ] ++ self.opts.upload = False ++ if ext_msg: ++ msg += ", which overrides some options:\n " + "\n ".join(ext_msg) ++ else: ++ msg += "." ++ msg += "\n\n" ++ return msg ++ + def _report_profiles_and_plugins(self): + self.ui_log.info("") + if len(self.loaded_plugins): +@@ -875,10 +909,12 @@ class SoSReport(SoSComponent): + return True + + def batch(self): ++ msg = self.policy.get_msg() ++ if self.opts.estimate_only: ++ msg += self._set_estimate_only() + if self.opts.batch: +- self.ui_log.info(self.policy.get_msg()) ++ self.ui_log.info(msg) + else: +- msg = self.policy.get_msg() + msg += _("Press ENTER to continue, or CTRL-C to quit.\n") + try: + input(msg) +@@ -1011,6 +1047,22 @@ class SoSReport(SoSComponent): + self.running_plugs.remove(plugin[1]) + self.loaded_plugins[plugin[0]-1][1].set_timeout_hit() + pool._threads.clear() ++ if self.opts.estimate_only: ++ from pathlib import Path ++ tmpdir_path = Path(self.archive.get_tmp_dir()) ++ self.estimated_plugsizes[plugin[1]] = sum( ++ [f.stat().st_size for f in tmpdir_path.glob('**/*') ++ if (os.path.isfile(f) and not os.path.islink(f))]) ++ # remove whole tmp_dir content - including "sos_commands" and ++ # similar dirs that will be re-created on demand by next plugin ++ # if needed; it is less error-prone approach than skipping ++ # deletion of some dirs but deleting their content ++ for f in os.listdir(self.archive.get_tmp_dir()): ++ f = os.path.join(self.archive.get_tmp_dir(), f) ++ if os.path.isdir(f): ++ rmtree(f) ++ else: ++ os.unlink(f) + return True + + def collect_plugin(self, plugin): +@@ -1330,6 +1382,24 @@ class SoSReport(SoSComponent): + self.policy.display_results(archive, directory, checksum, + map_file=map_file) + ++ if self.opts.estimate_only: ++ from sos.utilities import get_human_readable ++ _sum = get_human_readable(sum(self.estimated_plugsizes.values())) ++ self.ui_log.info("Estimated disk space requirement for whole " ++ "uncompressed sos report directory: %s" % _sum) ++ bigplugins = sorted(self.estimated_plugsizes.items(), ++ key=lambda x: x[1], reverse=True)[:3] ++ bp_out = ", ".join("%s: %s" % ++ (p, get_human_readable(v, precision=0)) ++ for p, v in bigplugins) ++ self.ui_log.info("Three biggest plugins: %s" % bp_out) ++ self.ui_log.info("") ++ self.ui_log.info("Please note the estimation is relevant to the " ++ "current options.") ++ self.ui_log.info("Be aware that the real disk space requirements " ++ "might be different.") ++ self.ui_log.info("") ++ + if self.opts.upload or self.opts.upload_url: + if not self.opts.build: + try: +-- +2.31.1 + diff --git a/sos-bz2011538-iptables-save-under-nf_tables-kmod.patch b/sos-bz2011538-iptables-save-under-nf_tables-kmod.patch new file mode 100644 index 0000000..e234bc6 --- /dev/null +++ b/sos-bz2011538-iptables-save-under-nf_tables-kmod.patch @@ -0,0 +1,73 @@ +From 7d5157aa5071e3620246e2d4aa80acb2d3ed30f0 Mon Sep 17 00:00:00 2001 +From: Pavel Moravec +Date: Tue, 28 Sep 2021 22:44:52 +0200 +Subject: [PATCH] [networking] prevent iptables-save commands to load nf_tables + kmod + +If iptables has built-in nf_tables kmod, then +'ip netns iptables-save' command requires the kmod which must +be guarded by predicate. + +Analogously for ip6tables. + +Resolves: #2703 + +Signed-off-by: Pavel Moravec +--- + sos/report/plugins/networking.py | 29 ++++++++++++++++++++++++----- + 1 file changed, 24 insertions(+), 5 deletions(-) + +diff --git a/sos/report/plugins/networking.py b/sos/report/plugins/networking.py +index c80ae719..1237f629 100644 +--- a/sos/report/plugins/networking.py ++++ b/sos/report/plugins/networking.py +@@ -182,22 +182,41 @@ class Networking(Plugin): + # per-namespace. + self.add_cmd_output("ip netns") + cmd_prefix = "ip netns exec " +- for namespace in self.get_network_namespaces( +- self.get_option("namespace_pattern"), +- self.get_option("namespaces")): ++ namespaces = self.get_network_namespaces( ++ self.get_option("namespace_pattern"), ++ self.get_option("namespaces")) ++ if (namespaces): ++ # 'ip netns exec iptables-save' must be guarded by nf_tables ++ # kmod, if 'iptables -V' output contains 'nf_tables' ++ # analogously for ip6tables ++ co = {'cmd': 'iptables -V', 'output': 'nf_tables'} ++ co6 = {'cmd': 'ip6tables -V', 'output': 'nf_tables'} ++ iptables_with_nft = (SoSPredicate(self, kmods=['nf_tables']) ++ if self.test_predicate(self, ++ pred=SoSPredicate(self, cmd_outputs=co)) ++ else None) ++ ip6tables_with_nft = (SoSPredicate(self, kmods=['nf_tables']) ++ if self.test_predicate(self, ++ pred=SoSPredicate(self, cmd_outputs=co6)) ++ else None) ++ for namespace in namespaces: + ns_cmd_prefix = cmd_prefix + namespace + " " + self.add_cmd_output([ + ns_cmd_prefix + "ip address show", + ns_cmd_prefix + "ip route show table all", + ns_cmd_prefix + "ip -s -s neigh show", + ns_cmd_prefix + "ip rule list", +- ns_cmd_prefix + "iptables-save", +- ns_cmd_prefix + "ip6tables-save", + ns_cmd_prefix + "netstat %s -neopa" % self.ns_wide, + ns_cmd_prefix + "netstat -s", + ns_cmd_prefix + "netstat %s -agn" % self.ns_wide, + ns_cmd_prefix + "nstat -zas", + ], priority=50) ++ self.add_cmd_output([ns_cmd_prefix + "iptables-save"], ++ pred=iptables_with_nft, ++ priority=50) ++ self.add_cmd_output([ns_cmd_prefix + "ip6tables-save"], ++ pred=ip6tables_with_nft, ++ priority=50) + + ss_cmd = ns_cmd_prefix + "ss -peaonmi" + # --allow-system-changes is handled directly in predicate +-- +2.31.1 + diff --git a/sos.spec b/sos.spec index 540db87..cf817a3 100644 --- a/sos.spec +++ b/sos.spec @@ -4,8 +4,8 @@ Summary: A set of tools to gather troubleshooting information from a system Name: sos -Version: 4.1 -Release: 8%{?dist} +Version: 4.2 +Release: 1%{?dist} Group: Applications/System Source0: https://github.com/sosreport/sos/archive/%{version}/sos-%{version}.tar.gz Source1: sos-audit-%{auditversion}.tgz @@ -22,28 +22,14 @@ Requires: xz Recommends: python3-pexpect Conflicts: vdsm < 4.40 Obsoletes: sos-collector <= 1.9 -Patch1: sos-bz1925419-gluster-pubkeys-statusfile.patch -Patch2: sos-bz1967110-collect-cleaning-consistency.patch -Patch3: sos-bz1967111-manpages-see-also.patch -Patch4: sos-bz1967112-add-cmd-timeout.patch -Patch5: sos-bz1967113-ds-mask-password-in-ldif.patch -Patch6: sos-bz1967114-gather-cups-browsed-logs.patch -Patch7: sos-bz1967115-sssd-memcache-and-logs.patch -Patch8: sos-bz1967116-ibmvNIC-dynamic-debugs.patch -Patch9: sos-bz1967117-pulpcore-plugin.patch -Patch10: sos-bz1967118-saphana-traceback.patch -Patch11: sos-bz1967119-collect-nstat.patch -Patch12: sos-bz1967120-snapper-plugin-and-allocation-failures.patch -Patch13: sos-bz1965002-skip-selinux-of-proc-everytime.patch -Patch14: sos-bz1985976-enhance-tc-hw-offload.patch -Patch15: sos-bz1985982-obfuscate-fqdn-from-dnf-log.patch -Patch16: sos-bz1967718-sssd-common.patch -Patch17: sos-bz1985983-ocp-cluster-cleaner.patch -Patch18: sos-bz1985985-sos-log-effective-options.patch -Patch19: sos-bz1938874-potential-issues-static-analyse.patch -Patch20: sos-bz1959779-conversions-and-upgrades.patch -Patch21: sos-bz1992859-rhui-plugin.patch -Patch22: sos-bz1992861-cleaner-AD-users-obfuscation.patch +Recommends: python3-pexpect +Recommends: python3-requests +Patch1: sos-bz1869561-cpuX-individual-sizelimits.patch +Patch2: sos-bz2011533-unpackaged-recursive-symlink.patch +Patch3: sos-bz2011534-opacapture-under-allow-system-changes.patch +Patch4: sos-bz2011535-kernel-psi.patch +Patch5: sos-bz2011538-iptables-save-under-nf_tables-kmod.patch +Patch6: sos-bz2011537-estimate-only-option.patch %description Sos is a set of tools that gathers information about system @@ -60,22 +46,6 @@ support technicians and developers. %patch4 -p1 %patch5 -p1 %patch6 -p1 -%patch7 -p1 -%patch8 -p1 -%patch9 -p1 -%patch10 -p1 -%patch11 -p1 -%patch12 -p1 -%patch13 -p1 -%patch14 -p1 -%patch15 -p1 -%patch16 -p1 -%patch17 -p1 -%patch18 -p1 -%patch19 -p1 -%patch20 -p1 -%patch21 -p1 -%patch22 -p1 %build %py3_build @@ -143,6 +113,22 @@ of the system. Currently storage and filesystem commands are audited. %changelog +* Wed Oct 06 2021 Pavel Moravec = 4.2-1 +- Rebase on upstream 4.2 + Resolves: bz1998134 +- [report] Implement --estimate-only + Resolves: bz2011537 +- [omnipath_client] Opacapture to run only with allow changes + Resolves: bz2011534 +- [unpackaged] deal with recursive loop of symlinks properly + Resolves: bz2011533 +- [networking] prevent iptables-save commands to load nf_tables + Resolves: bz2011538 +- [kernel] Capture Pressure Stall Information + Resolves: bz2011535 +- [processor] Apply sizelimit to /sys/devices/system/cpu/cpuX + Resolves: bz1869561 + * Wed Aug 11 2021 Pavel Moravec = 4.1-8 - [report,collect] unify --map-file arguments Resolves: bz1985985 diff --git a/sources b/sources index c6be755..bff9db8 100644 --- a/sources +++ b/sources @@ -1,2 +1,2 @@ -SHA512 (sos-4.1.tar.gz) = dff6526383531d615a3b8ffe5d9f10b5f5f342220efc3d94ab2490106c52db0ed0a1947596790c2ef8432d2831332885396cf0d22027b4a2d3079fb22a37d4a7 +SHA512 (sos-4.2.tar.gz) = 549a5a0c916ea342680df48ad88b620238dc7e9e32531992020a684046774b54c40b8201733537f0fc0b7245f6b67608ff39db83fc0eaf4c3bc48928100081d9 SHA512 (sos-audit-0.3.tgz) = 32597baf6350804d08179a0dbe48470a93df148e83d2e49bb3288f6bcc2d151bb1433761913bfbccd912c14de92435939fef5bcd7e091dfe33a345d61ea842ea