sos/sos-bz2011537-estimate-only-option.patch

From 5b245b1e449c6a05d09034bcb8290bffded79327 Mon Sep 17 00:00:00 2001
From: Pavel Moravec <pmoravec@redhat.com>
Date: Wed, 8 Sep 2021 17:04:58 +0200
Subject: [PATCH] [report] Implement --estimate-only

Add report option --estimate-only to estimate disk space requirements
when running a sos report.

Resolves: #2673

Signed-off-by: Pavel Moravec <pmoravec@redhat.com>
---
 man/en/sos-report.1    | 13 +++++++-
 sos/report/__init__.py | 74 ++++++++++++++++++++++++++++++++++++++++--
 2 files changed, 84 insertions(+), 3 deletions(-)

diff --git a/man/en/sos-report.1 b/man/en/sos-report.1
index 36b337df..e8efc8f8 100644
--- a/man/en/sos-report.1
+++ b/man/en/sos-report.1
@@ -14,7 +14,7 @@ sos report \- Collect and package diagnostic and support data
           [--preset preset] [--add-preset add_preset]\fR
           [--del-preset del_preset] [--desc description]\fR
           [--batch] [--build] [--debug] [--dry-run]\fR
-          [--label label] [--case-id id]\fR
+          [--estimate-only] [--label label] [--case-id id]\fR
           [--threads threads]\fR
           [--plugin-timeout TIMEOUT]\fR
           [--cmd-timeout TIMEOUT]\fR
@@ -317,6 +317,17 @@ output, or string data from the system. The resulting logs may be used
 to understand the actions that sos would have taken without the dry run
 option.
 .TP
+.B \--estimate-only
+Estimate disk space requirements when running sos report. This can be valuable
+to prevent sosreport working dir to consume all free disk space. No plugin data
+is available at the end.
+
+Plugins will be collected sequentially, size of collected files and commands outputs
+will be calculated and the plugin files will be immediatelly deleted prior execution
+of the next plugin. This still can consume whole free disk space, though. Please note,
+size estimations may not be accurate for highly utilized systems due to changes between
+an estimate and a real execution.
+.TP
 .B \--upload
 If specified, attempt to upload the resulting archive to a vendor defined location.
 
diff --git a/sos/report/__init__.py b/sos/report/__init__.py
index 82484f1d..b033f621 100644
--- a/sos/report/__init__.py
+++ b/sos/report/__init__.py
@@ -86,6 +86,7 @@ class SoSReport(SoSComponent):
         'desc': '',
         'domains': [],
         'dry_run': False,
+        'estimate_only': False,
         'experimental': False,
         'enable_plugins': [],
         'keywords': [],
@@ -137,6 +138,7 @@ class SoSReport(SoSComponent):
         self._args = args
         self.sysroot = "/"
         self.preset = None
+        self.estimated_plugsizes = {}
 
         self.print_header()
         self._set_debug()
@@ -223,6 +225,11 @@ class SoSReport(SoSComponent):
                                 help="Description for a new preset",)
         report_grp.add_argument("--dry-run", action="store_true",
                                 help="Run plugins but do not collect data")
+        report_grp.add_argument("--estimate-only", action="store_true",
+                                help="Approximate disk space requirements for "
+                                     "a real sos run; disables --clean and "
+                                     "--collect, sets --threads=1 and "
+                                     "--no-postproc")
         report_grp.add_argument("--experimental", action="store_true",
                                 dest="experimental", default=False,
                                 help="enable experimental plugins")
@@ -700,6 +700,33 @@ class SoSReport(SoSComponent):
                 self.all_options.append((plugin, plugin_name, optname,
                                          optparm))
 
+    def _set_estimate_only(self):
+        # set estimate-only mode by enforcing some options settings
+        # and return a corresponding log messages string
+        msg = "\nEstimate-only mode enabled"
+        ext_msg = []
+        if self.opts.threads > 1:
+            ext_msg += ["--threads=%s overriden to 1" % self.opts.threads, ]
+            self.opts.threads = 1
+        if not self.opts.build:
+            ext_msg += ["--build enabled", ]
+            self.opts.build = True
+        if not self.opts.no_postproc:
+            ext_msg += ["--no-postproc enabled", ]
+            self.opts.no_postproc = True
+        if self.opts.clean:
+            ext_msg += ["--clean disabled", ]
+            self.opts.clean = False
+        if self.opts.upload:
+            ext_msg += ["--upload* options disabled", ]
+            self.opts.upload = False
+        if ext_msg:
+            msg += ", which overrides some options:\n  " + "\n  ".join(ext_msg)
+        else:
+            msg += "."
+        msg += "\n\n"
+        return msg
+
     def _report_profiles_and_plugins(self):
         self.ui_log.info("")
         if len(self.loaded_plugins):
@@ -875,10 +909,12 @@ class SoSReport(SoSComponent):
         return True
 
     def batch(self):
+        msg = self.policy.get_msg()
+        if self.opts.estimate_only:
+            msg += self._set_estimate_only()
         if self.opts.batch:
-            self.ui_log.info(self.policy.get_msg())
+            self.ui_log.info(msg)
         else:
-            msg = self.policy.get_msg()
             msg += _("Press ENTER to continue, or CTRL-C to quit.\n")
             try:
                 input(msg)
@@ -1011,6 +1047,22 @@ class SoSReport(SoSComponent):
                 self.running_plugs.remove(plugin[1])
                 self.loaded_plugins[plugin[0]-1][1].set_timeout_hit()
                 pool._threads.clear()
+        if self.opts.estimate_only:
+            from pathlib import Path
+            tmpdir_path = Path(self.archive.get_tmp_dir())
+            self.estimated_plugsizes[plugin[1]] = sum(
+                    [f.stat().st_size for f in tmpdir_path.glob('**/*')
+                     if (os.path.isfile(f) and not os.path.islink(f))])
+            # remove whole tmp_dir content - including "sos_commands" and
+            # similar dirs that will be re-created on demand by next plugin
+            # if needed; it is less error-prone approach than skipping
+            # deletion of some dirs but deleting their content
+            for f in os.listdir(self.archive.get_tmp_dir()):
+                f = os.path.join(self.archive.get_tmp_dir(), f)
+                if os.path.isdir(f):
+                    rmtree(f)
+                else:
+                    os.unlink(f)
         return True
 
     def collect_plugin(self, plugin):
@@ -1330,6 +1382,24 @@ class SoSReport(SoSComponent):
             self.policy.display_results(archive, directory, checksum,
                                         map_file=map_file)
 
+        if self.opts.estimate_only:
+            from sos.utilities import get_human_readable
+            _sum = get_human_readable(sum(self.estimated_plugsizes.values()))
+            self.ui_log.info("Estimated disk space requirement for whole "
+                             "uncompressed sos report directory: %s" % _sum)
+            bigplugins = sorted(self.estimated_plugsizes.items(),
+                                key=lambda x: x[1], reverse=True)[:3]
+            bp_out = ",  ".join("%s: %s" %
+                                (p, get_human_readable(v, precision=0))
+                                for p, v in bigplugins)
+            self.ui_log.info("Three biggest plugins:  %s" % bp_out)
+            self.ui_log.info("")
+            self.ui_log.info("Please note the estimation is relevant to the "
+                             "current options.")
+            self.ui_log.info("Be aware that the real disk space requirements "
+                             "might be different.")
+            self.ui_log.info("")
+
         if self.opts.upload or self.opts.upload_url:
             if not self.opts.build:
                 try:
-- 
2.31.1

From 7ae47e6c0717c0b56c3368008dd99a87f7f436d5 Mon Sep 17 00:00:00 2001
From: Pavel Moravec <pmoravec@redhat.com>
Date: Wed, 13 Oct 2021 20:21:16 +0200
Subject: [PATCH] [report] Count with sos_logs and sos_reports in
 --estimate-only

Currently, we estimate just plugins' disk space and ignore sos_logs
or sos_reports directories - although they can occupy nontrivial disk
space as well.

Resolves: #2723

Signed-off-by: Pavel Moravec <pmoravec@redhat.com>
---
 sos/report/__init__.py | 8 ++++++++
 1 file changed, 8 insertions(+)

diff --git a/sos/report/__init__.py b/sos/report/__init__.py
index e35c7e8d..7feb31ee 100644
--- a/sos/report/__init__.py
+++ b/sos/report/__init__.py
@@ -1380,6 +1380,14 @@ class SoSReport(SoSComponent):
 
         if self.opts.estimate_only:
             from sos.utilities import get_human_readable
+            from pathlib import Path
+            # add sos_logs, sos_reports dirs, etc., basically everything
+            # that remained in self.tmpdir after plugins' contents removal
+            # that still will be moved to the sos report final directory path
+            tmpdir_path = Path(self.tmpdir)
+            self.estimated_plugsizes['sos_logs_reports'] = sum(
+                    [f.stat().st_size for f in tmpdir_path.glob('**/*')])
+
             _sum = get_human_readable(sum(self.estimated_plugsizes.values()))
             self.ui_log.info("Estimated disk space requirement for whole "
                              "uncompressed sos report directory: %s" % _sum)
-- 
2.31.1

From 4293f3317505661e8f32ba94ad87310996fa1626 Mon Sep 17 00:00:00 2001
From: Eric Desrochers <eric.desrochers@canonical.com>
Date: Tue, 19 Oct 2021 12:18:40 -0400
Subject: [PATCH] [report] check for symlink before rmtree when opt
 estimate-only is use

Check if the dir is also symlink before performing rmtree()
method so that unlink() method can be used instead.

Traceback (most recent call last):
  File "./bin/sos", line 22, in <module>
    sos.execute()
  File "/tmp/sos/sos/__init__.py", line 186, in execute
    self._component.execute()
OSError: Cannot call rmtree on a symbolic link

Closes: #2727

Signed-off-by: Eric Desrochers <eric.desrochers@canonical.com>
---
 sos/report/__init__.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/sos/report/__init__.py b/sos/report/__init__.py
index 7feb31ee..1b5bc97d 100644
--- a/sos/report/__init__.py
+++ b/sos/report/__init__.py
@@ -1059,7 +1059,7 @@ class SoSReport(SoSComponent):
             # deletion of some dirs but deleting their content
             for f in os.listdir(self.archive.get_tmp_dir()):
                 f = os.path.join(self.archive.get_tmp_dir(), f)
-                if os.path.isdir(f):
+                if os.path.isdir(f) and not os.path.islink(f):
                     rmtree(f)
                 else:
                     os.unlink(f)
-- 
2.31.1