--- a/sos/report/plugins/charmed_cruise_control.py 2026-07-17 13:13:53.703490321 +0200 +++ /dev/null 2026-07-04 09:21:02.628033198 +0200 @@ -1,142 +0,0 @@ -# This file is part of the sos project: https://github.com/sosreport/sos -# -# This copyrighted material is made available to anyone wishing to use, -# modify, copy, or redistribute it subject to the terms and conditions of -# version 2 of the GNU General Public License. -# -# See the LICENSE file in the source distribution for further information.# - -import glob -import re -from datetime import datetime -from functools import cached_property - -from sos.report.plugins import Plugin, UbuntuPlugin - -PATHS = { - "CONF": "/var/snap/charmed-kafka/current/etc/cruise-control", - "LOGS": "/var/snap/charmed-kafka/common/var/log/cruise-control", -} - -DATE_FORMAT = "%Y-%m-%d-%H" - - -class CharmedCruiseControl(Plugin, UbuntuPlugin): - short_desc = "Cruise Control (from Charmed Kafka)" - plugin_name = "charmed_cruise_control" - packages = ("charmed-kafka",) - - @cached_property - def credentials_args(self) -> str: - try: - with open( - f"{PATHS['CONF']}/cruisecontrol.credentials", - encoding="utf-8", - ) as f: - content = f.read().strip() - - match = re.match(r"balancer: (?P\w+),ADMIN", content) - if match: - pwd = match.group("pwd") - return f"-u balancer:{pwd}" - - return "" - except FileNotFoundError: - return "" - - def setup(self): - if not self.credentials_args: - # service not properly set-up, skip - return - - # --- FILE EXCLUSIONS --- - - all_logs = self.get_option("all_logs") - since = self.get_option("since") - - for file in glob.glob(f"{PATHS['LOGS']}/*"): - date = re.search( - pattern=r"([0-9]{4}-[0-9]{2}-[0-9]{2}-[0-9]{2})", string=file - ) - - # include files without date, aka current files - if not date: - continue - - file_dt = datetime.strptime(date.group(1), DATE_FORMAT) - - if ( - since - and not all_logs - and file_dt < datetime.strptime(str(since), DATE_FORMAT) - ): - # skip files outside given range - self.add_forbidden_path(file) - - # hide keys/stores - self.add_forbidden_path([ - f"{PATHS['CONF']}/*.pem", - f"{PATHS['CONF']}/*.key", - f"{PATHS['CONF']}/*.p12", - f"{PATHS['CONF']}/*.jks", - ]) - - # --- FILE INCLUSIONS --- - - self.add_copy_spec( - [ - f"{PATHS['CONF']}", - f"{PATHS['LOGS']}", - ] - ) - - # --- SNAP LOGS --- - - if all_logs: - self.add_cmd_output( - "snap logs -n all charmed-kafka.cruise-control", - suggest_filename="snap_logs_charmed-kafka_cruise-control", - ) - else: - self.add_cmd_output( - "snap logs -n 500 charmed-kafka.cruise-control", - suggest_filename="snap_logs_charmed-kafka_cruise-control", - ) - - # --- STATE, TASKS, PARTITION LOAD --- - - endpoints = { - 'cruise-control-state': 'state?super_verbose=true', - 'cluster-state': 'kafka_cluster_state?verbose=true', - 'partition_load': 'partition_load', - 'user-tasks': 'user_tasks', - } - - url = 'localhost:9090/kafkacruisecontrol' - - for fname, api in endpoints.items(): - self.add_cmd_output( - f"curl {self.credentials_args} {url}/{api}", - suggest_filename=fname, - ) - - # --- JMX METRICS --- - - self.add_cmd_output( - "curl localhost:9102/metrics", - suggest_filename="jmx-metrics" - ) - - def postproc(self): - if not self.credentials_args: - # service not properly set-up, skip - return - - # --- SCRUB PASSWORDS --- - - for scrub_pattern in [r'(password=")[^"]*', r"(balancer: )[^,]*"]: - self.do_path_regex_sub( - f"{PATHS['CONF']}/*", - scrub_pattern, - r"\1*********", - ) --- a/sos/report/plugins/charmed_kafka.py 2026-06-16 10:22:51.000000000 +0200 +++ /dev/null 2026-07-04 09:21:02.628033198 +0200 @@ -1,222 +0,0 @@ -# This file is part of the sos project: https://github.com/sosreport/sos -# -# This copyrighted material is made available to anyone wishing to use, -# modify, copy, or redistribute it subject to the terms and conditions of -# version 2 of the GNU General Public License. -# -# See the LICENSE file in the source distribution for further information.# - -import glob -import json -import re -from datetime import datetime -from functools import cached_property -from typing import Optional - -from sos.report.plugins import Plugin, UbuntuPlugin - -PATHS = { - "CONF": "/var/snap/charmed-kafka/current/etc/kafka", - "LOGS": "/var/snap/charmed-kafka/common/var/log/kafka", -} - -DATE_FORMAT = "%Y-%m-%d-%H" - - -class CharmedKafka(Plugin, UbuntuPlugin): - short_desc = "Charmed Kafka" - plugin_name = "charmed_kafka" - packages = ("charmed-kafka",) - - @cached_property - def bootstrap_server(self) -> Optional[str]: - try: - lines = [] - with open( - f"{PATHS['CONF']}/client.properties", - encoding="utf-8", - ) as f: - lines = f.readlines() - - for line in lines: - if "bootstrap" in line: - # ensure using internal address if - # not set in client.properties - return re.sub( - r":(?!1)(\d+)", r":1\1", - line.split("=")[1] - ) - - return None - except FileNotFoundError: - return None - - @cached_property - def default_bin_args(self) -> str: - if not self.bootstrap_server: - return "" - - return ( - f"--bootstrap-server {self.bootstrap_server}" - f" --command-config {PATHS['CONF']}/client.properties" - ) - - def setup(self): - if not self.bootstrap_server: - # service not properly set-up by the charm, skip - return - - # --- FILE EXCLUSIONS --- - - all_logs = self.get_option("all_logs") - since = self.get_option("since") - - for file in glob.glob(f"{PATHS['LOGS']}/*"): - date = re.search( - pattern=r"([0-9]{4}-[0-9]{2}-[0-9]{2}-[0-9]{2})", string=file - ) - - # include files without date, aka current files - if not date: - continue - - file_dt = datetime.strptime(date.group(1), DATE_FORMAT) - - if ( - since - and not all_logs - and file_dt < datetime.strptime(str(since), DATE_FORMAT) - ): - # skip files outside given range - self.add_forbidden_path(file) - - # hide keys/stores - self.add_forbidden_path([ - f"{PATHS['CONF']}/*.pem", - f"{PATHS['CONF']}/*.key", - f"{PATHS['CONF']}/*.p12", - f"{PATHS['CONF']}/*.jks", - ]) - - # --- FILE INCLUSIONS --- - - self.add_copy_spec( - [ - f"{PATHS['CONF']}", - f"{PATHS['LOGS']}", - ] - ) - - # --- SNAP LOGS --- - - if all_logs: - self.add_cmd_output( - "snap logs -n all charmed-kafka.daemon", - suggest_filename="snap_logs_charmed-kafka_daemon", - ) - else: - self.add_cmd_output( - "snap logs -n 500 all charmed-kafka.daemon", - suggest_filename="snap_logs_charmed-kafka_daemon", - ) - - # --- TOPICS --- - - self.add_cmd_output( - f"charmed-kafka.topics --describe {self.default_bin_args}", - env={"KAFKA_OPTS": ""}, - suggest_filename="kafka-topics", - ) - - # --- CONFIGS --- - - for entity in ["topics", "clients", "users", "brokers", "ips"]: - self.add_cmd_output( - ( - "charmed-kafka.configs --describe --all " - f"--entity-type {entity} {self.default_bin_args}" - ), - env={"KAFKA_OPTS": ""}, - suggest_filename=f"kafka-configs-{entity}", - ) - - # --- ACLs --- - - self.add_cmd_output( - f"charmed-kafka.acls --list {self.default_bin_args}", - env={"KAFKA_OPTS": ""}, - suggest_filename="kafka-acls", - ) - - # --- JMX METRICS --- - - self.add_cmd_output( - "curl localhost:9101/metrics", - suggest_filename="jmx-metrics" - ) - - def collect(self): - - # --- LOG DIRS --- - - log_dirs_output = self.exec_cmd( - f"charmed-kafka.log-dirs --describe {self.default_bin_args}", - env={"KAFKA_OPTS": ""}, - ) - log_dirs = {} - - # output has leading non-json lines that need cleaning, e.g: - - # Querying brokers for log directories information - # Received log directory information from brokers 100,101,102 - # {"brokers":[{"broker":100,"logDirs":... - - if log_dirs_output and log_dirs_output["status"] == 0: - for line in log_dirs_output["output"].splitlines(): - try: - log_dirs = json.loads(line) - break - except json.JSONDecodeError: - continue - - with self.collection_file("kafka-log-dirs") as f: - f.write(json.dumps(log_dirs, indent=4)) - - # --- TRANSACTIONS --- - - transactions_list = self.exec_cmd( - f"charmed-kafka.transactions {self.default_bin_args} list", - env={"KAFKA_OPTS": ""}, - ) - transactional_ids = [] - - if transactions_list and transactions_list["status"] == 0: - transactional_ids = transactions_list["output"].splitlines()[1:] - - transactions_outputs = [] - for transactional_id in transactional_ids: - transactions_describe = self.exec_cmd( - ( - f"charmed-kafka.transactions {self.default_bin_args}", - f"describe --transactional-id {transactional_id}", - ), - ) - - if transactions_describe and transactions_describe["status"] == 0: - transactions_outputs.append(transactions_describe["output"]) - - with self.collection_file("kafka-transactions") as f: - f.write("\n".join(transactions_outputs)) - - def postproc(self): - if not self.bootstrap_server: - # service not properly set-up by the charm, skip - return - - # --- SCRUB PASSWORDS --- - - self.do_path_regex_sub( - f"{PATHS['CONF']}/*", - r'(password=")[^"]*', - r"\1*********", - ) --- a/sos/report/plugins/charmed_mongodb.py 2026-07-17 13:13:53.703669427 +0200 +++ /dev/null 2026-07-04 09:21:02.628033198 +0200 @@ -1,456 +0,0 @@ -# This file is part of the sos project: https://github.com/sosreport/sos -# -# This copyrighted material is made available to anyone wishing to use, -# modify, copy, or redistribute it subject to the terms and conditions of -# version 2 of the GNU General Public License. -# -# See the LICENSE file in the source distribution for further information.# - -from enum import Enum -from urllib.parse import quote_plus, urlencode -import os -import shutil -from pathlib import Path -import typing -from typing import Dict, Optional, Tuple - -import yaml - -from sos.report.plugins import Plugin, PluginOpt, UbuntuPlugin -from sos.utilities import is_executable - -DATE_FORMAT = "%Y-%m-%d-%H" - - -class Substrate(Enum): - VM = "vm" - K8S = "k8s" - - -Role = typing.Literal["replication", "shard", "config-server"] - - -class CharmedMongoDB(Plugin, UbuntuPlugin): - """The Charmed MongoDB plugin is used to collect MongoDB configuration - and logs from the Charmed MongoDB snap package or K8s deployment. - - If all_logs is set to True, it collects all logs by default. - The parameters `dbuser` and `dbpass` are used to dump database information, - replicaset status, shard status, etc. You can provide those parameters with - the environment variables `MONGODB_USER` and `MONGODB_PASSWORD` - """ - - short_desc = "Charmed MongoDB" - plugin_name = "charmed_mongodb" - - # Triggers - - packages = ("charmed-mongodb",) - containers = ("mongod",) - - snap_package = "charmed-mongodb" - snap_path_common = "/var/snap/charmed-mongodb/common" - snap_path_current = "/var/snap/charmed-mongodb/current" - - kube_cmd = "kubectl" - selector = "app.kubernetes.io/name=mongodb-k8s" - - conf_paths = { - "MONGODB_CONF": "/etc/mongod", - "MONGODB_LOGS": "/var/log/mongodb", - } - - option_list = [ - PluginOpt( - name="dumpdbs", - default=False, - val_type=bool, - desc="Set to true to dump server information.", - ), - PluginOpt( - "dbuser", - default="", - val_type=str, - desc="Username for database dump collection", - ), - PluginOpt( - "dbpass", - default="", - val_type=str, - desc="Password for database dump collection", - ), - ] - - regular_commands: Tuple[Tuple[str, str], ...] = ( - ("EJSON.stringify(db.serverStatus())", "server_status.txt"), - ("EJSON.stringify(rs.status())", "replicaset_status.txt"), - ("EJSON.stringify(db.getUsers())", "db_users.txt"), - ("EJSON.stringify(db.getRoles())", "db_roles.txt"), - ( - "EJSON.stringify(db.adminCommand({listDatabases: 1}))", - "db_databases.txt", - ), - ) - - config_server_commands: Tuple[Tuple[str, str], ...] = ( - ("EJSON.stringify(sh.status())", "shard_cluster_status.txt"), - ("EJSON.stringify(sh.listShards())", "shard_shards.txt"), - ) - - def _join_conf_path(self, base: str, *parts: str): - stripped_parts = [p.lstrip(os.path.sep) for p in parts] - return self.path_join(base, *stripped_parts) - - @staticmethod - def _match_role(role: str) -> Optional[Role]: - if role == "configsvr": - return "config-server" - if role == "shardsvr": - return "shard" - if role == "replication": - return "replication" - return None - - def vm_role(self) -> Optional[Role]: - role: str = "replication" - conf_path = self._join_conf_path( - self.snap_path_current, self.conf_paths["MONGODB_CONF"] - ) - try: - with open( - f"{conf_path}/mongod.conf", - encoding="utf-8", - ) as f: - data = yaml.safe_load(f) - sharding_conf = data.get("sharding", {}) - if sharding_conf: - role = sharding_conf.get("clusterRole", "") - - return self._match_role(role) - except FileNotFoundError: - return None - - def k8s_role(self, kube_cmd: str, cont: str, pod: str) -> Optional[Role]: - role: str = "replication" - - # Cat the configuration file. - cat_conf_cmd = ( - f"{kube_cmd} exec -c {cont} {pod} -- " - f"cat {self.conf_paths['MONGODB_CONF']}/mongod.conf" - ) - result = self.exec_cmd(cat_conf_cmd) - - if result.get("status") != 0: - return None - - data = yaml.safe_load(result.get("output", "")) - sharding_conf = data.get("sharding", {}) - if sharding_conf: - role = sharding_conf.get("clusterRole", "") - return self._match_role(role) - - def _get_db_credentials(self) -> Tuple[Optional[str], Optional[str]]: - db_user = self.get_option("dbuser") - db_pass = self.get_option("dbpass") - - if not db_user: - if "MONGODB_USER" in os.environ: - self.soslog.info( - "MONGODB_USER present: Using MONGODB_USER environment" - "variable, user did not provide username." - ) - db_user = os.environ["MONGODB_USER"] - else: - self.soslog.warning("error: Missing credentials (username)") - return None, None - - if not db_pass: - if "MONGODB_PWD" in os.environ: - self.soslog.info( - "MONGODB_PWD present: Using MONGODB_PWD environment " - "variable, user did not provide password." - ) - db_pass = os.environ["MONGODB_PWD"] - else: - self.soslog.warning("error: Missing credentials (password)") - return None, None - - return db_user, db_pass - - def _process_snap(self): - role = self.vm_role() - - if not role: - # The service is not properly set up by the charm, exiting. - return - - base_conf = self._join_conf_path( - self.snap_path_current, self.conf_paths["MONGODB_CONF"] - ) - base_logs = self._join_conf_path( - self.snap_path_common, self.conf_paths["MONGODB_LOGS"] - ) - - all_logs = self.get_option("all_logs") - - # Hide certificates and cluster keyfile. - self.add_forbidden_path( - [ - f"{base_conf}/*.pem", - f"{base_conf}/*.crt", - f"{base_conf}/keyFile", - ] - ) - self.add_copy_spec([base_conf]) - - if all_logs: - self.add_copy_spec([f"{base_logs}/*"]) - else: - self.add_copy_spec([f"{base_logs}/*.log"]) - - lines = None if all_logs else 500 - self.add_journal("snap.charmed-mongodb.*", lines=lines) - - self.add_cmd_output("snap info charmed-mongodb") - - # METRICS - self.add_cmd_output( - "curl http://127.0.0.1:9216/metrics", "mongodb_exporter-metrics" - ) - - if not self.get_option("dumpdbs"): - return - - db_user, db_pass = self._get_db_credentials() - if not db_user or not db_pass: - return - - mongodb_uri = self._build_uri(db_user, Substrate.VM) - mongos_uri = self._build_uri(db_user, Substrate.VM, 27018) - mongodb_cmd = "charmed-mongodb.mongosh" - env = {"MONGODB_PWD": db_pass} - - # --- REGULAR INFORMATION --- - for command, suggest_filename in self.regular_commands: - self.add_cmd_output( - ( - f"sh -c '{mongodb_cmd} {mongodb_uri} --quiet " - f"--eval \"{command}\"'" - ), - suggest_filename=suggest_filename, - env=env, - ) - - # --- SHARDING INFORMATION --- - if role == "config-server": - for command, suggest_filename in self.config_server_commands: - command = ( - f"sh -c '{mongodb_cmd} {mongos_uri} --quiet " - f"--eval \"{command}\"'" - ) - self.add_cmd_output( - command, - suggest_filename=suggest_filename, - env=env, - ) - - def _build_uri( - self, - db_user: str, - substrate: Substrate, - port: int = 27017, - ) -> str: - base_conf = self.conf_paths["MONGODB_CONF"] - args: Dict[str, str] = {"authSource": "admin"} - if substrate == Substrate.VM: - base_conf = self._join_conf_path(self.snap_path_current, base_conf) - - external_ca = Path(f"{base_conf}/external-ca.crt") - external_cert = Path(f"{base_conf}/external-cert.pem") - if external_ca.exists() and external_cert.exists(): - args |= { - "tls": "true", - "tlsCertificateKeyFile": f"{external_cert}", - "tlsCaFile": f"{external_ca}", - } - _args = urlencode(args) - user = quote_plus(db_user) - host = "127.0.0.1" - return f"mongodb://{user}:${{MONGODB_PWD}}@{host}:{port}/admin?{_args}" - - def _determine_namespaces(self) -> list[str]: - namespaces = self.exec_cmd( - f"{self.kube_cmd} get pods -A -l {self.selector} " - "-o jsonpath='{.items[*].metadata.namespace}'" - ) - if namespaces["status"] == 0: - return list(set(namespaces["output"].strip().split())) - return [] - - def _get_pod_names(self, namespace) -> list[str]: - pods = self.exec_cmd( - f"{self.kube_cmd} -n {namespace} get pods -l {self.selector} " - "-o jsonpath='{.items[*].metadata.name}'" - ) - if pods["status"] == 0: - return pods["output"].strip().split() - return [] - - def _remote_exec( - self, - kube_cmd: str, - cont: str, - pod: str, - mongod_cmd: str, - uri: str, - password: str, - cmd: str, - cmd_name: str, - ): - output_file = f"/tmp/eval_{cmd_name}" # nosec: B108 - # We first execute the command and write into a file - query_cmd = ( - f"{kube_cmd} exec -c {cont} {pod} -- " - f'sh -lc \'export MONGODB_PWD="{password}"; {mongod_cmd} {uri} ' - f'--quiet --eval "{cmd}" > {output_file}\'' - ) - - self.exec_cmd(query_cmd) - - # We then cat that file to have the command output. - cat_cmd = ( - f"{kube_cmd} exec -c {cont} {pod} -- " - f"sh -lc 'cat {output_file} && rm {output_file}'" - ) - - self.add_cmd_output( - cmds=cat_cmd, - suggest_filename=f"{pod}_{cmd_name}", - ) - - def _collect_per_namespace(self, ns: str, all_logs: bool): - kube_cmd = f"{self.kube_cmd} -n {ns}" - - mongodb_cont = "mongod" - pods = self._get_pod_names(ns) - logs_path = self.conf_paths["MONGODB_LOGS"] - conf_path = self.conf_paths["MONGODB_CONF"] - - # Get the config and logs from each pod - dump_files_path = self.get_cmd_output_path() - for path in self.conf_paths.values(): - for pod in pods: - name_prefix = f"{dump_files_path}/pods/{pod}/{path}" - os.makedirs(name_prefix, exist_ok=True) - copy_cmd = ( - f"{kube_cmd} cp -c {mongodb_cont} " - f"{pod}:{path} {name_prefix}" - ) - self.exec_cmd(copy_cmd) - - for pod in pods: - if all_logs: # This is all_logs - self.add_copy_spec([f"{dump_files_path}/{pod}/{logs_path}/*"]) - else: - self.add_copy_spec( - [f"{dump_files_path}/{pod}/{logs_path}/*.log"] - ) - - self.add_forbidden_path( - [ - f"{dump_files_path}/pods/{pod}/{conf_path}/*.pem", - f"{dump_files_path}/pods/{pod}/{conf_path}/*.crt", - f"{dump_files_path}/pods/{pod}/{conf_path}/keyFile", - ] - ) - self.add_copy_spec([f"{dump_files_path}/pods/{pod}/{conf_path}"]) - - # METRICS - for pod in pods: - query_cmd = ( - f"{kube_cmd} exec -c {mongodb_cont} {pod} -- " - "sh -lc curl localhost:9216/metrics" - ) - self.add_cmd_output( - cmds=query_cmd, - suggest_filename=f"mongodb_exporter-metrics-{pod}", - ) - - if not self.get_option("dumpdbs"): - return - - db_user, db_pass = self._get_db_credentials() - if not db_user or not db_pass: - return - - mongodb_uri = self._build_uri(db_user, Substrate.K8S) - mongos_uri = self._build_uri(db_user, Substrate.K8S, 27018) - mongodb_cmd = "mongosh" - - for pod in pods: - role = self.k8s_role(kube_cmd, mongodb_cont, pod) - for command, suggest_filename in self.regular_commands: - self._remote_exec( - kube_cmd=kube_cmd, - cont=mongodb_cont, - pod=pod, - mongod_cmd=mongodb_cmd, - uri=mongodb_uri, - password=db_pass, - cmd=command, - cmd_name=suggest_filename, - ) - - # --- SHARDING INFORMATION --- - if role == "config-server": - for command, suggest_filename in self.config_server_commands: - self._remote_exec( - kube_cmd=kube_cmd, - cont=mongodb_cont, - pod=pod, - mongod_cmd=mongodb_cmd, - uri=mongos_uri, - password=db_pass, - cmd=command, - cmd_name=suggest_filename, - ) - - def _process_k8s(self): - all_logs = self.get_option("all_logs") or False - namespaces = self._determine_namespaces() - for namespace in namespaces: - self._collect_per_namespace(namespace, all_logs) - - def setup(self) -> None: - if self.is_installed(self.snap_package): - self._process_snap() - - if is_executable(self.kube_cmd, self.sysroot): - self._process_k8s() - - def postproc(self): - if self.is_installed(self.snap_package): - substrate = Substrate.VM - if not self.vm_role(): - # Service was not properly set up. - return - else: - substrate = Substrate.K8S - - base_conf = self.conf_paths["MONGODB_CONF"] - - if substrate == Substrate.VM: - base_conf = self._join_conf_path(self.snap_path_current, base_conf) - else: - base_conf = f"{self.get_cmd_output_path()}/pods/*/{base_conf}" - shutil.rmtree( - f"{self.get_cmd_output_path()}/pods", - ignore_errors=True, - ) - - # --- SCRUB PASSWORDS --- - self.do_path_regex_sub( - f"{base_conf}/*", - regexp=r'("queryPassword": ")[^"]*"', - subst=r"\1*********", - ) --- a/sos/report/plugins/charmed_mongos.py 2026-06-16 10:22:51.000000000 +0200 +++ /dev/null 2026-07-04 09:21:02.628033198 +0200 @@ -1,396 +0,0 @@ -# This file is part of the sos project: https://github.com/sosreport/sos -# -# This copyrighted material is made available to anyone wishing to use, -# modify, copy, or redistribute it subject to the terms and conditions of -# version 2 of the GNU General Public License. -# -# See the LICENSE file in the source distribution for further information. - -from enum import Enum -import shlex -import shutil -import secrets -import string -import subprocess -from urllib.parse import quote, quote_plus, urlencode -import os -from pathlib import Path -from typing import Dict, Optional, Tuple - -import yaml - -from sos.report.plugins import Plugin, PluginOpt, UbuntuPlugin -from sos.utilities import is_executable - - -class Substrate(Enum): - VM = "vm" - K8S = "k8s" - - -class CharmedMongos(Plugin, UbuntuPlugin): - """The Charmed Mongos plugin is used to collect Mongos configuration - and logs from the Charmed Mongos snap package or K8s deployment. - - If all_logs is set to True, it collects all logs by default. - The parameters `dbuser` and `dbpass` are used to dump database information, - replicaset status, shard status, etc. You can provide those parameters with - the environment variables `MONGOS_USER` and `MONGOS_PWD` - """ - - short_desc = "Charmed Mongos" - plugin_name = "charmed_mongos" - - # Triggers - packages = ("charmed-mongodb",) - containers = ("mongos",) - - snap_package = "charmed-mongodb" - snap_path_common = "/var/snap/charmed-mongodb/common" - snap_path_current = "/var/snap/charmed-mongodb/current" - - kube_cmd = "kubectl" - selector = "app.kubernetes.io/name=mongos-k8s" - - conf_paths = { - "MONGODB_CONF": "/etc/mongod", - "MONGODB_LOGS": "/var/log/mongodb", - } - - option_list = [ - PluginOpt( - name="dumpdbs", - default=False, - val_type=bool, - desc="Set to true to dump server information.", - ), - PluginOpt( - "dbuser", - default="", - val_type=str, - desc="Username for database dump collection", - ), - PluginOpt( - "dbpass", - default="", - val_type=str, - desc="Password for database dump collection", - ), - ] - - mongos_commands: Tuple[Tuple[str, str], ...] = ( - ("EJSON.stringify(db.serverStatus())", "server_status.txt"), - ("EJSON.stringify(db.getUsers())", "db_users.txt"), - ("EJSON.stringify(db.getRoles())", "db_roles.txt"), - ( - "EJSON.stringify(db.adminCommand({listDatabases: 1}))", - "db_databases.txt", - ), - ("EJSON.stringify(sh.status())", "shard_cluster_status.txt"), - ("EJSON.stringify(sh.listShards())", "shard_shards.txt"), - ) - - def _join_conf_path(self, base: str, *parts: str): - stripped_parts = [p.lstrip(os.path.sep) for p in parts] - return self.path_join(base, *stripped_parts) - - def _get_db_credentials(self) -> Tuple[Optional[str], Optional[str]]: - db_user = self.get_option("dbuser") - db_pass = self.get_option("dbpass") - - if not db_user: - if "MONGOS_USER" in os.environ: - self.soslog.info( - "MONGOS_USER present: Using MONGOS_USER environment " - "variable, user did not provide username." - ) - db_user = os.environ["MONGOS_USER"] - else: - self.soslog.warning("error: Missing credentials (username)") - return None, None - - if not db_pass: - if "MONGOS_PWD" in os.environ: - self.soslog.info( - "MONGOS_PWD present: Using MONGOS_PWD environment " - "variable, user did not provide password." - ) - db_pass = os.environ["MONGOS_PWD"] - else: - self.soslog.warning("error: Missing credentials (password)") - return None, None - - return db_user, db_pass - - def vm_config_db(self) -> Optional[str]: - _conf_file = f"{self.conf_paths['MONGODB_CONF']}/mongos.conf" - conf_path = self._join_conf_path(self.snap_path_current, _conf_file) - try: - with open(conf_path, encoding="utf-8") as f: - data = yaml.safe_load(f) - - return data.get("sharding", {}).get("configDB", None) - except FileNotFoundError: - return None - - def k8s_config_db( - self, kube_cmd: str, cont: str, pod: str - ) -> Optional[str]: - # Cat the configuration file. - cat_conf_cmd = ( - f"{kube_cmd} exec -c {cont} {pod} -- " - f"cat {self.conf_paths['MONGODB_CONF']}/mongos.conf" - ) - result = self.exec_cmd(cat_conf_cmd) - - if result.get("status") != 0: - return None - - data = yaml.safe_load(result.get("output", "")) - return data.get("sharding", {}).get("configDB", None) - - def _process_snap(self): - config_db = self.vm_config_db() - - if not config_db: - # The service is not properly set up by the charm, exiting. - return - - base_conf = self._join_conf_path( - self.snap_path_current, self.conf_paths["MONGODB_CONF"] - ) - base_logs = self._join_conf_path( - self.snap_path_common, self.conf_paths["MONGODB_LOGS"] - ) - - all_logs = self.get_option("all_logs") - - # Hide certificates and cluster keyfile. - self.add_forbidden_path( - [ - f"{base_conf}/*.pem", - f"{base_conf}/*.crt", - f"{base_conf}/keyFile", - f"{base_conf}/mongod.conf", - ] - ) - self.add_copy_spec([base_conf]) - - if all_logs: - self.add_copy_spec([f"{base_logs}/*"]) - else: - self.add_copy_spec([f"{base_logs}/*.log"]) - - lines = None if all_logs else 500 - self.add_journal("snap.charmed-mongodb.mongos", lines=lines) - - self.add_cmd_output("snap info charmed-mongodb") - - if not self.get_option("dumpdbs"): - return - - db_user, db_pass = self._get_db_credentials() - if not db_user or not db_pass: - return - - mongos_uri = self._build_uri(db_user, Substrate.VM) - mongodb_cmd = "charmed-mongodb.mongosh" - env = {"MONGOS_PWD": db_pass} - - # --- REGULAR INFORMATION --- - for command, suggest_filename in self.mongos_commands: - self.add_cmd_output( - ( - f"sh -c '{mongodb_cmd} {mongos_uri} " - f"--quiet --eval \"{command}\"'" - ), - suggest_filename=suggest_filename, - env=env, - ) - - def _build_uri( - self, - db_user: str, - substrate: Substrate, - ) -> str: - base_conf = self.conf_paths["MONGODB_CONF"] - args: Dict[str, str] = {"authSource": "admin"} - - if substrate == Substrate.VM: - base_conf = self._join_conf_path(self.snap_path_current, base_conf) - - external_ca = Path(f"{base_conf}/external-ca.crt") - external_cert = Path(f"{base_conf}/external-cert.pem") - if external_ca.exists() and external_cert.exists(): - args |= { - "tls": "true", - "tlsCertificateKeyFile": f"{external_cert}", - "tlsCaFile": f"{external_ca}", - } - _args = urlencode(args) - user = quote_plus(db_user) - host = "127.0.0.1:27018" - socket_path = Path(f"{self.snap_path_current}/var/mongodb-27018.sock") - if substrate == Substrate.VM and socket_path.exists(): - host = quote(f"{socket_path}", safe="") - - if substrate == Substrate.K8S: - return f"mongodb://{user}@{host}/admin?{_args}" - return f"mongodb://{user}:${{MONGOS_PWD}}@{host}/admin?{_args}" - - def _determine_namespaces(self) -> list[str]: - namespaces = self.exec_cmd( - f"{self.kube_cmd} get pods -A -l {self.selector} " - "-o jsonpath='{.items[*].metadata.namespace}'" - ) - if namespaces["status"] == 0: - return list(set(namespaces["output"].strip().split())) - return [] - - def _get_pod_names(self, namespace) -> list[str]: - pods = self.exec_cmd( - f"{self.kube_cmd} -n {namespace} get pods -l {self.selector} " - "-o jsonpath='{.items[*].metadata.name}'" - ) - if pods["status"] == 0: - return pods["output"].strip().split() - return [] - - def _remote_exec( - self, - kube_cmd: str, - cont: str, - pod: str, - mongod_cmd: str, - uri: str, - password: str, - cmd: str, - cmd_name: str, - ): - choices = string.ascii_letters + string.digits - randstring = "".join([secrets.choice(choices) for _ in range(16)]) - output_file = f"/tmp/eval_{cmd_name}_{randstring}.txt" # nosec: B108 - # We first execute the command and write into a file - query_cmd = shlex.split( - f"{kube_cmd} exec -i -c {cont} {pod} -- " - f'sh -lc \'{mongod_cmd} {uri} ' - f'--quiet --eval "{cmd}" --password > {output_file}\'' - ) - - # Pass the password as stdin that is forwarded to the - # container through kubectl exec - subprocess.run(query_cmd, input=password.encode(), check=False) - - # We then cat that file to have the command output. - cat_cmd = ( - f"{kube_cmd} exec -c {cont} {pod} -- " - f"sh -lc 'cat {output_file} && rm {output_file}'" - ) - - self.add_cmd_output( - cmds=cat_cmd, - suggest_filename=f"{pod}_{cmd_name}", - ) - - def _collect_per_namespace(self, ns: str, all_logs: bool): - kube_cmd = f"{self.kube_cmd} -n {ns}" - - mongodb_cont = "mongos" - pods = self._get_pod_names(ns) - logs_path = self.conf_paths["MONGODB_LOGS"] - conf_path = self.conf_paths["MONGODB_CONF"] - - # Get the config and logs from each pod - dump_files_path = self.get_cmd_output_path() - for path in self.conf_paths.values(): - for pod in pods: - name_prefix = f"{dump_files_path}/pods/{pod}/{path}" - os.makedirs(name_prefix, exist_ok=True) - copy_cmd = ( - f"{kube_cmd} cp -c {mongodb_cont} " - f"{pod}:{path} {name_prefix}" - ) - self.exec_cmd(copy_cmd) - - for pod in pods: - if all_logs: # This is all_logs - self.add_copy_spec([f"{dump_files_path}/{pod}/{logs_path}/*"]) - else: - self.add_copy_spec( - [f"{dump_files_path}/{pod}/{logs_path}/*.log"] - ) - - self.add_forbidden_path( - [ - f"{dump_files_path}/pods/{pod}/{conf_path}/*.pem", - f"{dump_files_path}/pods/{pod}/{conf_path}/*.crt", - f"{dump_files_path}/pods/{pod}/{conf_path}/keyFile", - f"{dump_files_path}/pods/{pod}/{conf_path}/mongod.conf", - ] - ) - self.add_copy_spec([f"{dump_files_path}/pods/{pod}/{conf_path}"]) - - if not self.get_option("dumpdbs"): - return - - db_user, db_pass = self._get_db_credentials() - if not db_user or not db_pass: - return - - mongos_uri = self._build_uri(db_user, Substrate.K8S) - mongodb_cmd = "mongosh" - - for pod in pods: - config_db = self.k8s_config_db(kube_cmd, mongodb_cont, pod) - if not config_db: - continue - for command, suggest_filename in self.mongos_commands: - self._remote_exec( - kube_cmd=kube_cmd, - cont=mongodb_cont, - pod=pod, - mongod_cmd=mongodb_cmd, - uri=mongos_uri, - password=db_pass, - cmd=command, - cmd_name=suggest_filename, - ) - - def _process_k8s(self): - all_logs = self.get_option("all_logs") or False - namespaces = self._determine_namespaces() - for namespace in namespaces: - self._collect_per_namespace(namespace, all_logs) - - def setup(self) -> None: - if self.is_installed(self.snap_package): - self._process_snap() - - if is_executable(self.kube_cmd, self.sysroot): - self._process_k8s() - - def postproc(self): - if self.is_installed(self.snap_package): - substrate = Substrate.VM - if not self.vm_config_db(): - # Service was not properly set up. - return - else: - substrate = Substrate.K8S - - base_conf = self.conf_paths["MONGODB_CONF"] - if substrate == Substrate.VM: - base_conf = self._join_conf_path(self.snap_path_current, base_conf) - else: - base_conf = f"{self.get_cmd_output_path()}/pods/*/{base_conf}" - shutil.rmtree( - f"{self.get_cmd_output_path()}/pods", - ignore_errors=True - ) - - # --- SCRUB PASSWORDS --- - self.do_path_regex_sub( - f"{base_conf}/*", - regexp=r'("queryPassword": ")[^"]*"', - subst=r"\1*********", - ) --- a/sos/report/plugins/charmed_zookeeper.py 2026-06-16 10:22:51.000000000 +0200 +++ /dev/null 2026-07-04 09:21:02.628033198 +0200 @@ -1,205 +0,0 @@ -# This file is part of the sos project: https://github.com/sosreport/sos -# -# This copyrighted material is made available to anyone wishing to use, -# modify, copy, or redistribute it subject to the terms and conditions of -# version 2 of the GNU General Public License. -# -# See the LICENSE file in the source distribution for further information.# - -import glob -import json -import os -import re -from datetime import datetime -from functools import cached_property - -from sos.report.plugins import Plugin, UbuntuPlugin - -PATHS = { - "CONF": "/var/snap/charmed-zookeeper/current/etc/zookeeper", - "LOGS": "/var/snap/charmed-zookeeper/common/var/log/zookeeper", - "DATA-LOG": "/var/snap/charmed-zookeeper/common/var/lib/zookeeper/data-log", # noqa: E501 # pylint:disable=line-too-long - "DATA": "/var/snap/charmed-zookeeper/common/var/lib/zookeeper/data", - "BIN": "/snap/charmed-zookeeper/current/opt/zookeeper/bin", - "JRE": "/snap/charmed-zookeeper/current/usr/lib/jvm/java-11-openjdk-amd64/jre", # noqa: E501 # pylint:disable=line-too-long -} - -DATE_FORMAT = "%Y-%m-%d-%H" -CLIENT_JAAS = "client-jaas.cfg" - - -class CharmedZooKeeper(Plugin, UbuntuPlugin): - short_desc = "Charmed ZooKeeper" - plugin_name = "charmed_zookeeper" - packages = ("charmed-zookeeper",) - - default_env = { - "JAVA_HOME": PATHS["JRE"], - "CLIENT_JVMFLAGS": f"-Djava.security.auth.login.config={PATHS['CONF']}/{CLIENT_JAAS}", # noqa: E501 # pylint:disable=line-too-long - "SERVER_JVMFLAGS": "", - } - - @cached_property - def super_password(self) -> str: - try: - with open( - f"{PATHS['CONF']}/{CLIENT_JAAS}", - "r", - encoding="utf-8" - ) as f: - for line in f.readlines(): - if "super" in line: - pw = line.split("=")[1].replace(";", "") - return pw.replace('"', "").strip() - - return "" - except FileNotFoundError: - return "" - - def setup(self): - if not self.super_password: - # service not properly set-up by the charm, skip - return - - # --- FILE EXCLUSIONS --- - - all_logs = self.get_option("all_logs") - since = self.get_option("since") - - for file in glob.glob(f"{PATHS['LOGS']}/*"): - date = re.search( - pattern=r"([0-9]{4}-[0-9]{2}-[0-9]{2}-[0-9]{2})", string=file - ) - - # include files without date, aka current files - if not date: - continue - - file_dt = datetime.strptime(date.group(1), DATE_FORMAT) - - if ( - since - and not all_logs - and file_dt < datetime.strptime(str(since), DATE_FORMAT) - ): - # skip files outside given range - self.add_forbidden_path(file) - - # hide keys/stores - self.add_forbidden_path([ - f"{PATHS['CONF']}/*.pem", - f"{PATHS['CONF']}/*.key", - f"{PATHS['CONF']}/*.p12", - f"{PATHS['CONF']}/*.jks", - f"{PATHS['CONF']}/{CLIENT_JAAS}", - ]) - - # --- FILE INCLUSIONS --- - - self.add_copy_spec( - [ - f"{PATHS['CONF']}", - f"{PATHS['LOGS']}", - ] - ) - - # --- SNAP LOGS --- - - if all_logs: - self.add_cmd_output( - "snap logs -n all charmed-zookeeper.daemon", - suggest_filename="snap_logs_charmed-zookeeper_daemon", - ) - else: - self.add_cmd_output( - "snap logs -n 500 charmed-zookeeper.daemon", - suggest_filename="snap_logs_charmed-zookeeper_daemon", - ) - - # --- JMX METRICS --- - - self.add_cmd_output( - "curl localhost:9998/metrics", - suggest_filename="jmx-metrics" - ) - - # --- PROVIDER METRICS --- - - self.add_cmd_output( - "curl localhost:7000/metrics", - suggest_filename="provider-metrics" - ) - - def collect(self): - # --- TRANSACTIONS --- - all_logs = self.get_option("all_logs") - since = self.get_option("since") - - for file in glob.glob(f"{PATHS['DATA-LOG']}/version-2/*"): - transactions = self.exec_cmd( - f"{PATHS['BIN']}/zkTxnLogToolkit.sh -d {file}", - env=self.default_env, - ) - - if not (transactions and transactions["status"] == 0): - continue - - valid_dt_transactions = [] - for transaction in transactions["output"].splitlines(): - try: - log_dt = datetime.strptime( - " ".join(transaction.split()[:4]), - "%m/%d/%y %I:%M:%S %p %Z", - ) - except ValueError: # must've been a bad line - continue - - if ( - since - and not all_logs - and log_dt < datetime.strptime(str(since), DATE_FORMAT) - ): - continue - - valid_dt_transactions.append(transaction) - - with self.collection_file( - f"zookeeper-transaction-{os.path.basename(file)}" - ) as f: - f.write("\n".join(valid_dt_transactions)) - - # --- SNAPSHOT --- - files = glob.glob(f"{PATHS['DATA']}/version-2/*") - if not files: - return - - most_recent_file = sorted(files)[-1] - snapshot_json = self.exec_cmd( - f"{PATHS['BIN']}/zkSnapShotToolkit.sh -json {most_recent_file}", - env=self.default_env, - ) - snapshot = {} - - if snapshot_json and snapshot_json["status"] == 0: - for line in snapshot_json["output"].splitlines(): - try: - snapshot = json.loads(line) - break - except json.JSONDecodeError: - continue - - with self.collection_file("zookeeper-snapshot") as f: - f.write(json.dumps(snapshot, indent=4)) - - def postproc(self): - if not self.super_password: - # service not properly set-up by the charm, skip - return - - # --- SCRUB PASSWORDS --- - - self.do_path_regex_sub( - f"{PATHS['CONF']}/*", - r'(password=")[^"]*', - r"\1*********", - )