From 816f2ecc3269c7f00bd28fd5254269bb52b57a99 Mon Sep 17 00:00:00 2001 From: Pavel Moravec Date: Tue, 25 Jan 2022 21:45:56 +0100 Subject: [PATCH] sos updates to RHEL9.0 Resolves: bz2037350 bz2041855 bz2043104 bz2043488 Signed-off-by: Pavel Moravec --- sos-bz2037350-ocp-backports.patch | 237 ++++++++++++++++++ sos-bz2041855-virsh-in-foreground.patch | 58 +++++ sos-bz2043104-foreman-tasks-msgpack.patch | 59 +++++ ...043488-ovn-proper-package-enablement.patch | 45 ++++ sos.spec | 20 +- 5 files changed, 417 insertions(+), 2 deletions(-) create mode 100644 sos-bz2041855-virsh-in-foreground.patch create mode 100644 sos-bz2043104-foreman-tasks-msgpack.patch create mode 100644 sos-bz2043488-ovn-proper-package-enablement.patch diff --git a/sos-bz2037350-ocp-backports.patch b/sos-bz2037350-ocp-backports.patch index a47dc91..5a44250 100644 --- a/sos-bz2037350-ocp-backports.patch +++ b/sos-bz2037350-ocp-backports.patch @@ -5647,4 +5647,241 @@ index 30f99a1140..3aca0c7460 100644 # There are some incompatible changes in nmcli since # the release of NetworkManager >= 0.9.9. In addition, # NetworkManager >= 0.9.9 will use the long names of +From 9eb60f0bb6ea36f9c1cf099c1fd20cf3938b4b26 Mon Sep 17 00:00:00 2001 +From: Jake Hunsaker +Date: Mon, 17 Jan 2022 11:11:24 -0500 +Subject: [PATCH] [clean] Ignore empty items for obfuscation better + +This commit fixes a couple edge cases where an item empty (e.g. and +empty string '') was not being properly ignored, which in turned caused +failures in writing both obfuscations and replacement files. + +This should no longer be possible. + +Signed-off-by: Jake Hunsaker +--- + sos/cleaner/mappings/__init__.py | 5 ++++- + sos/cleaner/mappings/username_map.py | 2 +- + sos/cleaner/parsers/username_parser.py | 2 +- + 3 files changed, 6 insertions(+), 3 deletions(-) + +diff --git a/sos/cleaner/mappings/__init__.py b/sos/cleaner/mappings/__init__.py +index 5cf5c8b2d..48171a052 100644 +--- a/sos/cleaner/mappings/__init__.py ++++ b/sos/cleaner/mappings/__init__.py +@@ -49,6 +49,8 @@ def add(self, item): + :param item: The plaintext object to obfuscate + """ + with self.lock: ++ if not item: ++ return item + self.dataset[item] = self.sanitize_item(item) + return self.dataset[item] + +@@ -67,7 +69,8 @@ def get(self, item): + """Retrieve an item's obfuscated counterpart from the map. If the item + does not yet exist in the map, add it by generating one on the fly + """ +- if self.ignore_item(item) or self.item_in_dataset_values(item): ++ if (not item or self.ignore_item(item) or ++ self.item_in_dataset_values(item)): + return item + if item not in self.dataset: + return self.add(item) +diff --git a/sos/cleaner/mappings/username_map.py b/sos/cleaner/mappings/username_map.py +index 7ecccd7bc..ed6dc0912 100644 +--- a/sos/cleaner/mappings/username_map.py ++++ b/sos/cleaner/mappings/username_map.py +@@ -24,7 +24,7 @@ class SoSUsernameMap(SoSMap): + + def load_names_from_options(self, opt_names): + for name in opt_names: +- if name not in self.dataset.keys(): ++ if name and name not in self.dataset.keys(): + self.add(name) + + def sanitize_item(self, username): +diff --git a/sos/cleaner/parsers/username_parser.py b/sos/cleaner/parsers/username_parser.py +index 49640f7fd..2853c860f 100644 +--- a/sos/cleaner/parsers/username_parser.py ++++ b/sos/cleaner/parsers/username_parser.py +@@ -55,7 +55,7 @@ def load_usernames_into_map(self, content): + user = line.split()[0] + except Exception: + continue +- if user.lower() in self.skip_list: ++ if not user or user.lower() in self.skip_list: + continue + users.add(user) + for each in users: +From ed618678fd3d07e68e1a430eb7d225a9701332e0 Mon Sep 17 00:00:00 2001 +From: Jake Hunsaker +Date: Thu, 13 Jan 2022 13:52:34 -0500 +Subject: [PATCH] [clean,parsers] Build regex lists for static items only once + +For parsers such as the username and keyword parsers, we don't discover +new items through parsing archives - these parsers use static lists +determined before we begin the actual obfuscation process. + +As such, we can build a list of regexes for these static items once, and +then reference those regexes during execution, rather than rebuilding +the regex for each of these items for every obfuscation. + +For use cases where hundreds of items, e.g. hundreds of usernames, are +being obfuscated this results in a significant performance increase. +Individual per-file gains are minor - fractions of a second - however +these gains build up over the course of the hundreds to thousands of +files a typical archive can be expected to contain. + +Signed-off-by: Jake Hunsaker +--- + sos/cleaner/__init__.py | 9 +++++++++ + sos/cleaner/parsers/__init__.py | 10 ++++++++++ + sos/cleaner/parsers/keyword_parser.py | 15 ++++++++++----- + sos/cleaner/parsers/username_parser.py | 14 ++++++++------ + tests/unittests/cleaner_tests.py | 1 + + 5 files changed, 38 insertions(+), 11 deletions(-) + +diff --git a/sos/cleaner/__init__.py b/sos/cleaner/__init__.py +index 5686e2131..b76bef644 100644 +--- a/sos/cleaner/__init__.py ++++ b/sos/cleaner/__init__.py +@@ -294,6 +294,7 @@ def execute(self): + # we have at least one valid target to obfuscate + self.completed_reports = [] + self.preload_all_archives_into_maps() ++ self.generate_parser_item_regexes() + self.obfuscate_report_paths() + + if not self.completed_reports: +@@ -498,6 +499,14 @@ def _replace_obfuscated_archives(self): + shutil.move(archive.final_archive_path, dest) + archive.final_archive_path = dest_name + ++ def generate_parser_item_regexes(self): ++ """For the parsers that use prebuilt lists of items, generate those ++ regexes now since all the parsers should be preloaded by the archive(s) ++ as well as being handed cmdline options and mapping file configuration. ++ """ ++ for parser in self.parsers: ++ parser.generate_item_regexes() ++ + def preload_all_archives_into_maps(self): + """Before doing the actual obfuscation, if we have multiple archives + to obfuscate then we need to preload each of them into the mappings +diff --git a/sos/cleaner/parsers/__init__.py b/sos/cleaner/parsers/__init__.py +index e62fd9384..6def863a6 100644 +--- a/sos/cleaner/parsers/__init__.py ++++ b/sos/cleaner/parsers/__init__.py +@@ -46,9 +46,19 @@ class SoSCleanerParser(): + map_file_key = 'unset' + + def __init__(self, config={}): ++ self.regexes = {} + if self.map_file_key in config: + self.mapping.conf_update(config[self.map_file_key]) + ++ def generate_item_regexes(self): ++ """Generate regexes for items the parser will be searching for ++ repeatedly without needing to generate them for every file and/or line ++ we process ++ ++ Not used by all parsers. ++ """ ++ pass ++ + def parse_line(self, line): + """This will be called for every line in every file we process, so that + every parser has a chance to scrub everything. +diff --git a/sos/cleaner/parsers/keyword_parser.py b/sos/cleaner/parsers/keyword_parser.py +index 694c6073a..362a1929e 100644 +--- a/sos/cleaner/parsers/keyword_parser.py ++++ b/sos/cleaner/parsers/keyword_parser.py +@@ -9,6 +9,7 @@ + # See the LICENSE file in the source distribution for further information. + + import os ++import re + + from sos.cleaner.parsers import SoSCleanerParser + from sos.cleaner.mappings.keyword_map import SoSKeywordMap +@@ -33,16 +34,20 @@ def __init__(self, config, keywords=None, keyword_file=None): + # pre-generate an obfuscation mapping for each keyword + # this is necessary for cases where filenames are being + # obfuscated before or instead of file content +- self.mapping.get(keyword) ++ self.mapping.get(keyword.lower()) + self.user_keywords.append(keyword) + if keyword_file and os.path.exists(keyword_file): + with open(keyword_file, 'r') as kwf: + self.user_keywords.extend(kwf.read().splitlines()) + ++ def generate_item_regexes(self): ++ for kw in self.user_keywords: ++ self.regexes[kw] = re.compile(kw, re.I) ++ + def parse_line(self, line): + count = 0 +- for keyword in sorted(self.user_keywords, reverse=True): +- if keyword in line: +- line = line.replace(keyword, self.mapping.get(keyword)) +- count += 1 ++ for kwrd, reg in sorted(self.regexes.items(), key=len, reverse=True): ++ if reg.search(line): ++ line, _count = reg.subn(self.mapping.get(kwrd.lower()), line) ++ count += _count + return line, count +diff --git a/sos/cleaner/parsers/username_parser.py b/sos/cleaner/parsers/username_parser.py +index 3208a6557..49640f7fd 100644 +--- a/sos/cleaner/parsers/username_parser.py ++++ b/sos/cleaner/parsers/username_parser.py +@@ -61,12 +61,14 @@ def load_usernames_into_map(self, content): + for each in users: + self.mapping.get(each) + ++ def generate_item_regexes(self): ++ for user in self.mapping.dataset: ++ self.regexes[user] = re.compile(user, re.I) ++ + def parse_line(self, line): + count = 0 +- for username in sorted(self.mapping.dataset.keys(), reverse=True): +- _reg = re.compile(username, re.I) +- if _reg.search(line): +- line, count = _reg.subn( +- self.mapping.get(username.lower()), line +- ) ++ for user, reg in sorted(self.regexes.items(), key=len, reverse=True): ++ if reg.search(line): ++ line, _count = reg.subn(self.mapping.get(user.lower()), line) ++ count += _count + return line, count +diff --git a/tests/unittests/cleaner_tests.py b/tests/unittests/cleaner_tests.py +index cb20772fd..b59eade9a 100644 +--- a/tests/unittests/cleaner_tests.py ++++ b/tests/unittests/cleaner_tests.py +@@ -105,6 +105,7 @@ def setUp(self): + self.host_parser = SoSHostnameParser(config={}, opt_domains='foobar.com') + self.kw_parser = SoSKeywordParser(config={}, keywords=['foobar']) + self.kw_parser_none = SoSKeywordParser(config={}) ++ self.kw_parser.generate_item_regexes() + + def test_ip_parser_valid_ipv4_line(self): + line = 'foobar foo 10.0.0.1/24 barfoo bar' +From: Pavel Moravec +Subject: downstream-only patch to allow container_runtime change on 4.2 +sos cluster/collector already, as any 4.2 released version will support +it. +diff -rup a/sos/collector/sosnode.py b/sos/collector/sosnode.py +--- a/sos/collector/sosnode.py ++++ b/sos/collector/sosnode.py +@@ -586,8 +586,6 @@ class SosNode(): + if self.opts.cmd_timeout: + sos_opts.append('--cmd-timeout=%s' + % quote(str(self.opts.cmd_timeout))) +- +- if self.check_sos_version('4.3'): + if self.opts.container_runtime != 'auto': + sos_opts.append( + "--container-runtime=%s" % self.opts.container_runtime diff --git a/sos-bz2041855-virsh-in-foreground.patch b/sos-bz2041855-virsh-in-foreground.patch new file mode 100644 index 0000000..8eca3fb --- /dev/null +++ b/sos-bz2041855-virsh-in-foreground.patch @@ -0,0 +1,58 @@ +From 747fef695e4ff08f320c5f03090bdefa7154c761 Mon Sep 17 00:00:00 2001 +From: Pavel Moravec +Date: Fri, 14 Jan 2022 20:10:22 +0100 +Subject: [PATCH] [virsh] Call virsh commands in the foreground / with a TTY + +In some virsh errors (like unable to connect to a hypervisor), +the tool requires to communicate to TTY otherwise it can get stuck +(when called via Popen with a timeout). + +Calling it on foreground prevents the stuck / waiting on cmd timeout. + +Resolves: #2825 + +Signed-off-by: Pavel Moravec +--- + sos/report/plugins/virsh.py | 14 +++++++++----- + 1 file changed, 9 insertions(+), 5 deletions(-) + +diff --git a/sos/report/plugins/virsh.py b/sos/report/plugins/virsh.py +index d6b7c16761..08f9a8488c 100644 +--- a/sos/report/plugins/virsh.py ++++ b/sos/report/plugins/virsh.py +@@ -39,26 +39,30 @@ def setup(self): + ] + + for subcmd in subcmds: +- self.add_cmd_output('%s %s' % (cmd, subcmd)) ++ self.add_cmd_output('%s %s' % (cmd, subcmd), foreground=True) + + # get network, pool and nwfilter elements + for k in ['net', 'nwfilter', 'pool']: +- k_list = self.collect_cmd_output('%s %s-list' % (cmd, k)) ++ k_list = self.collect_cmd_output('%s %s-list' % (cmd, k), ++ foreground=True) + if k_list['status'] == 0: + k_lines = k_list['output'].splitlines() + # the 'Name' column position changes between virsh cmds + pos = k_lines[0].split().index('Name') + for j in filter(lambda x: x, k_lines[2:]): + n = j.split()[pos] +- self.add_cmd_output('%s %s-dumpxml %s' % (cmd, k, n)) ++ self.add_cmd_output('%s %s-dumpxml %s' % (cmd, k, n), ++ foreground=True) + + # cycle through the VMs/domains list, ignore 2 header lines and latest + # empty line, and dumpxml domain name in 2nd column +- domains_output = self.exec_cmd('%s list --all' % cmd) ++ domains_output = self.exec_cmd('%s list --all' % cmd, foreground=True) + if domains_output['status'] == 0: + domains_lines = domains_output['output'].splitlines()[2:] + for domain in filter(lambda x: x, domains_lines): + d = domain.split()[1] + for x in ['dumpxml', 'dominfo', 'domblklist']: +- self.add_cmd_output('%s %s %s' % (cmd, x, d)) ++ self.add_cmd_output('%s %s %s' % (cmd, x, d), ++ foreground=True) ++ + # vim: et ts=4 sw=4 diff --git a/sos-bz2043104-foreman-tasks-msgpack.patch b/sos-bz2043104-foreman-tasks-msgpack.patch new file mode 100644 index 0000000..900389c --- /dev/null +++ b/sos-bz2043104-foreman-tasks-msgpack.patch @@ -0,0 +1,59 @@ +From 5634f7dd77eff821f37daa953fa86cc783d3b937 Mon Sep 17 00:00:00 2001 +From: Pavel Moravec +Date: Fri, 21 Jan 2022 16:27:33 +0100 +Subject: [PATCH] [foreman] Use psql-msgpack-decode wrapper for dynflow >= 1.6 + +In dynflow >=1.6.3, dynflow* tables in postgres are encoded by +msgpack which makes plain CSV dumps unreadable. In such a case, +psql-msgpack-decode wrapper tool from dynflow-utils (of any +version) must be used instead of the plain psql command. + +Resolves: #2830 + +Signed-off-by: Pavel Moravec +--- + sos/report/plugins/foreman.py | 16 ++++++++++++---- + 1 file changed, 12 insertions(+), 4 deletions(-) + +diff --git a/sos/report/plugins/foreman.py b/sos/report/plugins/foreman.py +index 314a651d1..3fd80e6a8 100644 +--- a/sos/report/plugins/foreman.py ++++ b/sos/report/plugins/foreman.py +@@ -244,8 +244,16 @@ def setup(self): + self.add_cmd_output(_cmd, suggest_filename=table, timeout=600, + sizelimit=100, env=self.env) + ++ # dynflow* tables on dynflow >=1.6.3 are encoded and hence in that ++ # case, psql-msgpack-decode wrapper tool from dynflow-utils (any ++ # version) must be used instead of plain psql command ++ dynutils = self.is_installed('dynflow-utils') + for dyn in foremancsv: +- _cmd = self.build_query_cmd(foremancsv[dyn], csv=True) ++ binary = "psql" ++ if dyn != 'foreman_tasks_tasks' and dynutils: ++ binary = "/usr/libexec/psql-msgpack-decode" ++ _cmd = self.build_query_cmd(foremancsv[dyn], csv=True, ++ binary=binary) + self.add_cmd_output(_cmd, suggest_filename=dyn, timeout=600, + sizelimit=100, env=self.env) + +@@ -270,7 +278,7 @@ def setup(self): + # collect http[|s]_proxy env.variables + self.add_env_var(["http_proxy", "https_proxy"]) + +- def build_query_cmd(self, query, csv=False): ++ def build_query_cmd(self, query, csv=False, binary="psql"): + """ + Builds the command needed to invoke the pgsql query as the postgres + user. +@@ -281,8 +289,8 @@ def build_query_cmd(self, query, csv=False): + if csv: + query = "COPY (%s) TO STDOUT " \ + "WITH (FORMAT 'csv', DELIMITER ',', HEADER)" % query +- _dbcmd = "psql --no-password -h %s -p 5432 -U foreman -d foreman -c %s" +- return _dbcmd % (self.dbhost, quote(query)) ++ _dbcmd = "%s --no-password -h %s -p 5432 -U foreman -d foreman -c %s" ++ return _dbcmd % (binary, self.dbhost, quote(query)) + + def postproc(self): + self.do_path_regex_sub( diff --git a/sos-bz2043488-ovn-proper-package-enablement.patch b/sos-bz2043488-ovn-proper-package-enablement.patch new file mode 100644 index 0000000..945eda9 --- /dev/null +++ b/sos-bz2043488-ovn-proper-package-enablement.patch @@ -0,0 +1,45 @@ +From 210b83e1d1164d29b1f6198675b8b596c4af8336 Mon Sep 17 00:00:00 2001 +From: Daniel Alvarez Sanchez +Date: Thu, 20 Jan 2022 12:58:44 +0100 +Subject: [PATCH] [ovn_central] Account for Red Hat ovn package naming + +Previous ovn packages were 'ovn2xxx' and now they have +been renamed to 'ovn-2xxx'. This causes sos tool to not +recognize that the packages are installed and it won't +collect the relevant data. + +This patch is changing the match to be compatible +with the previous and newer naming conventions. + +Signed-off-by: Daniel Alvarez Sanchez +--- + sos/report/plugins/ovn_central.py | 2 +- + sos/report/plugins/ovn_host.py | 2 +- + 2 files changed, 2 insertions(+), 2 deletions(-) + +diff --git a/sos/report/plugins/ovn_central.py b/sos/report/plugins/ovn_central.py +index ddbf288da..0f947d4c5 100644 +--- a/sos/report/plugins/ovn_central.py ++++ b/sos/report/plugins/ovn_central.py +@@ -147,7 +147,7 @@ def setup(self): + + class RedHatOVNCentral(OVNCentral, RedHatPlugin): + +- packages = ('openvswitch-ovn-central', 'ovn2.*-central', ) ++ packages = ('openvswitch-ovn-central', 'ovn.*-central', ) + ovn_sbdb_sock_path = '/var/run/openvswitch/ovnsb_db.ctl' + + +diff --git a/sos/report/plugins/ovn_host.py b/sos/report/plugins/ovn_host.py +index 78604a15a..25c38cccc 100644 +--- a/sos/report/plugins/ovn_host.py ++++ b/sos/report/plugins/ovn_host.py +@@ -55,7 +55,7 @@ def check_enabled(self): + + class RedHatOVNHost(OVNHost, RedHatPlugin): + +- packages = ('openvswitch-ovn-host', 'ovn2.*-host', ) ++ packages = ('openvswitch-ovn-host', 'ovn.*-host', ) + + + class DebianOVNHost(OVNHost, DebianPlugin, UbuntuPlugin): diff --git a/sos.spec b/sos.spec index 5094ccc..3b90d30 100644 --- a/sos.spec +++ b/sos.spec @@ -5,7 +5,7 @@ Summary: A set of tools to gather troubleshooting information from a system Name: sos Version: 4.2 -Release: 11%{?dist} +Release: 12%{?dist} Group: Applications/System Source0: https://github.com/sosreport/sos/archive/%{version}/sos-%{version}.tar.gz Source1: sos-audit-%{auditversion}.tgz @@ -42,6 +42,9 @@ Patch15: sos-bz2025611-RHTS-api-change.patch Patch16: sos-bz2034001-nvidia-GPU-info.patch Patch17: sos-bz2031777-rhui-logs.patch Patch18: sos-bz2037350-ocp-backports.patch +Patch19: sos-bz2043104-foreman-tasks-msgpack.patch +Patch20: sos-bz2041855-virsh-in-foreground.patch +Patch21: sos-bz2043488-ovn-proper-package-enablement.patch %description Sos is a set of tools that gathers information about system @@ -70,6 +73,9 @@ support technicians and developers. %patch16 -p1 %patch17 -p1 %patch18 -p1 +%patch19 -p1 +%patch20 -p1 +%patch21 -p1 %build %py3_build @@ -137,8 +143,18 @@ of the system. Currently storage and filesystem commands are audited. %changelog +* Tue Jan 25 2022 Pavel Moravec = 4.2-12 +- [foreman] Use psql-msgpack-decode wrapper for dynflow >= 1.6 + Resolves: bz2043104 +- [virsh] Call virsh commands in the foreground / with a TTY + Resolves: bz2041855 +- [ovn_central] Account for Red Hat ovn package naming + Resolves: bz2043488 +- [clean,parsers] Build regex lists for static items only once + Resolves: bz2037350 + * Mon Jan 10 2022 Pavel Moravec = 4.2-11 -- [reort] Add journal logs for NetworkManager plugin +- [report] Add journal logs for NetworkManager plugin Resolves: bz2037350 * Fri Jan 07 2022 Pavel Moravec = 4.2-9