366 lines
16 KiB
Diff
366 lines
16 KiB
Diff
From 5b245b1e449c6a05d09034bcb8290bffded79327 Mon Sep 17 00:00:00 2001
|
|
From: Pavel Moravec <pmoravec@redhat.com>
|
|
Date: Wed, 8 Sep 2021 17:04:58 +0200
|
|
Subject: [PATCH] [report] Implement --estimate-only
|
|
|
|
Add report option --estimate-only to estimate disk space requirements
|
|
when running a sos report.
|
|
|
|
Resolves: #2673
|
|
|
|
Signed-off-by: Pavel Moravec <pmoravec@redhat.com>
|
|
---
|
|
man/en/sos-report.1 | 13 +++++++-
|
|
sos/report/__init__.py | 74 ++++++++++++++++++++++++++++++++++++++++--
|
|
2 files changed, 84 insertions(+), 3 deletions(-)
|
|
|
|
diff --git a/man/en/sos-report.1 b/man/en/sos-report.1
|
|
index 36b337df..e8efc8f8 100644
|
|
--- a/man/en/sos-report.1
|
|
+++ b/man/en/sos-report.1
|
|
@@ -14,7 +14,7 @@ sos report \- Collect and package diagnostic and support data
|
|
[--preset preset] [--add-preset add_preset]\fR
|
|
[--del-preset del_preset] [--desc description]\fR
|
|
[--batch] [--build] [--debug] [--dry-run]\fR
|
|
- [--label label] [--case-id id]\fR
|
|
+ [--estimate-only] [--label label] [--case-id id]\fR
|
|
[--threads threads]\fR
|
|
[--plugin-timeout TIMEOUT]\fR
|
|
[--cmd-timeout TIMEOUT]\fR
|
|
@@ -317,6 +317,17 @@ output, or string data from the system. The resulting logs may be used
|
|
to understand the actions that sos would have taken without the dry run
|
|
option.
|
|
.TP
|
|
+.B \--estimate-only
|
|
+Estimate disk space requirements when running sos report. This can be valuable
|
|
+to prevent sosreport working dir to consume all free disk space. No plugin data
|
|
+is available at the end.
|
|
+
|
|
+Plugins will be collected sequentially, size of collected files and commands outputs
|
|
+will be calculated and the plugin files will be immediatelly deleted prior execution
|
|
+of the next plugin. This still can consume whole free disk space, though. Please note,
|
|
+size estimations may not be accurate for highly utilized systems due to changes between
|
|
+an estimate and a real execution.
|
|
+.TP
|
|
.B \--upload
|
|
If specified, attempt to upload the resulting archive to a vendor defined location.
|
|
|
|
diff --git a/sos/report/__init__.py b/sos/report/__init__.py
|
|
index 82484f1d..b033f621 100644
|
|
--- a/sos/report/__init__.py
|
|
+++ b/sos/report/__init__.py
|
|
@@ -86,6 +86,7 @@ class SoSReport(SoSComponent):
|
|
'desc': '',
|
|
'domains': [],
|
|
'dry_run': False,
|
|
+ 'estimate_only': False,
|
|
'experimental': False,
|
|
'enable_plugins': [],
|
|
'keywords': [],
|
|
@@ -137,6 +138,7 @@ class SoSReport(SoSComponent):
|
|
self._args = args
|
|
self.sysroot = "/"
|
|
self.preset = None
|
|
+ self.estimated_plugsizes = {}
|
|
|
|
self.print_header()
|
|
self._set_debug()
|
|
@@ -223,6 +225,11 @@ class SoSReport(SoSComponent):
|
|
help="Description for a new preset",)
|
|
report_grp.add_argument("--dry-run", action="store_true",
|
|
help="Run plugins but do not collect data")
|
|
+ report_grp.add_argument("--estimate-only", action="store_true",
|
|
+ help="Approximate disk space requirements for "
|
|
+ "a real sos run; disables --clean and "
|
|
+ "--collect, sets --threads=1 and "
|
|
+ "--no-postproc")
|
|
report_grp.add_argument("--experimental", action="store_true",
|
|
dest="experimental", default=False,
|
|
help="enable experimental plugins")
|
|
@@ -700,6 +700,33 @@ class SoSReport(SoSComponent):
|
|
self.all_options.append((plugin, plugin_name, optname,
|
|
optparm))
|
|
|
|
+ def _set_estimate_only(self):
|
|
+ # set estimate-only mode by enforcing some options settings
|
|
+ # and return a corresponding log messages string
|
|
+ msg = "\nEstimate-only mode enabled"
|
|
+ ext_msg = []
|
|
+ if self.opts.threads > 1:
|
|
+ ext_msg += ["--threads=%s overriden to 1" % self.opts.threads, ]
|
|
+ self.opts.threads = 1
|
|
+ if not self.opts.build:
|
|
+ ext_msg += ["--build enabled", ]
|
|
+ self.opts.build = True
|
|
+ if not self.opts.no_postproc:
|
|
+ ext_msg += ["--no-postproc enabled", ]
|
|
+ self.opts.no_postproc = True
|
|
+ if self.opts.clean:
|
|
+ ext_msg += ["--clean disabled", ]
|
|
+ self.opts.clean = False
|
|
+ if self.opts.upload:
|
|
+ ext_msg += ["--upload* options disabled", ]
|
|
+ self.opts.upload = False
|
|
+ if ext_msg:
|
|
+ msg += ", which overrides some options:\n " + "\n ".join(ext_msg)
|
|
+ else:
|
|
+ msg += "."
|
|
+ msg += "\n\n"
|
|
+ return msg
|
|
+
|
|
def _report_profiles_and_plugins(self):
|
|
self.ui_log.info("")
|
|
if len(self.loaded_plugins):
|
|
@@ -875,10 +909,12 @@ class SoSReport(SoSComponent):
|
|
return True
|
|
|
|
def batch(self):
|
|
+ msg = self.policy.get_msg()
|
|
+ if self.opts.estimate_only:
|
|
+ msg += self._set_estimate_only()
|
|
if self.opts.batch:
|
|
- self.ui_log.info(self.policy.get_msg())
|
|
+ self.ui_log.info(msg)
|
|
else:
|
|
- msg = self.policy.get_msg()
|
|
msg += _("Press ENTER to continue, or CTRL-C to quit.\n")
|
|
try:
|
|
input(msg)
|
|
@@ -1011,6 +1047,22 @@ class SoSReport(SoSComponent):
|
|
self.running_plugs.remove(plugin[1])
|
|
self.loaded_plugins[plugin[0]-1][1].set_timeout_hit()
|
|
pool._threads.clear()
|
|
+ if self.opts.estimate_only:
|
|
+ from pathlib import Path
|
|
+ tmpdir_path = Path(self.archive.get_tmp_dir())
|
|
+ self.estimated_plugsizes[plugin[1]] = sum(
|
|
+ [f.stat().st_size for f in tmpdir_path.glob('**/*')
|
|
+ if (os.path.isfile(f) and not os.path.islink(f))])
|
|
+ # remove whole tmp_dir content - including "sos_commands" and
|
|
+ # similar dirs that will be re-created on demand by next plugin
|
|
+ # if needed; it is less error-prone approach than skipping
|
|
+ # deletion of some dirs but deleting their content
|
|
+ for f in os.listdir(self.archive.get_tmp_dir()):
|
|
+ f = os.path.join(self.archive.get_tmp_dir(), f)
|
|
+ if os.path.isdir(f):
|
|
+ rmtree(f)
|
|
+ else:
|
|
+ os.unlink(f)
|
|
return True
|
|
|
|
def collect_plugin(self, plugin):
|
|
@@ -1330,6 +1382,24 @@ class SoSReport(SoSComponent):
|
|
self.policy.display_results(archive, directory, checksum,
|
|
map_file=map_file)
|
|
|
|
+ if self.opts.estimate_only:
|
|
+ from sos.utilities import get_human_readable
|
|
+ _sum = get_human_readable(sum(self.estimated_plugsizes.values()))
|
|
+ self.ui_log.info("Estimated disk space requirement for whole "
|
|
+ "uncompressed sos report directory: %s" % _sum)
|
|
+ bigplugins = sorted(self.estimated_plugsizes.items(),
|
|
+ key=lambda x: x[1], reverse=True)[:3]
|
|
+ bp_out = ", ".join("%s: %s" %
|
|
+ (p, get_human_readable(v, precision=0))
|
|
+ for p, v in bigplugins)
|
|
+ self.ui_log.info("Three biggest plugins: %s" % bp_out)
|
|
+ self.ui_log.info("")
|
|
+ self.ui_log.info("Please note the estimation is relevant to the "
|
|
+ "current options.")
|
|
+ self.ui_log.info("Be aware that the real disk space requirements "
|
|
+ "might be different.")
|
|
+ self.ui_log.info("")
|
|
+
|
|
if self.opts.upload or self.opts.upload_url:
|
|
if not self.opts.build:
|
|
try:
|
|
--
|
|
2.31.1
|
|
|
|
From 7ae47e6c0717c0b56c3368008dd99a87f7f436d5 Mon Sep 17 00:00:00 2001
|
|
From: Pavel Moravec <pmoravec@redhat.com>
|
|
Date: Wed, 13 Oct 2021 20:21:16 +0200
|
|
Subject: [PATCH] [report] Count with sos_logs and sos_reports in
|
|
--estimate-only
|
|
|
|
Currently, we estimate just plugins' disk space and ignore sos_logs
|
|
or sos_reports directories - although they can occupy nontrivial disk
|
|
space as well.
|
|
|
|
Resolves: #2723
|
|
|
|
Signed-off-by: Pavel Moravec <pmoravec@redhat.com>
|
|
---
|
|
sos/report/__init__.py | 8 ++++++++
|
|
1 file changed, 8 insertions(+)
|
|
|
|
diff --git a/sos/report/__init__.py b/sos/report/__init__.py
|
|
index e35c7e8d..7feb31ee 100644
|
|
--- a/sos/report/__init__.py
|
|
+++ b/sos/report/__init__.py
|
|
@@ -1380,6 +1380,14 @@ class SoSReport(SoSComponent):
|
|
|
|
if self.opts.estimate_only:
|
|
from sos.utilities import get_human_readable
|
|
+ from pathlib import Path
|
|
+ # add sos_logs, sos_reports dirs, etc., basically everything
|
|
+ # that remained in self.tmpdir after plugins' contents removal
|
|
+ # that still will be moved to the sos report final directory path
|
|
+ tmpdir_path = Path(self.tmpdir)
|
|
+ self.estimated_plugsizes['sos_logs_reports'] = sum(
|
|
+ [f.stat().st_size for f in tmpdir_path.glob('**/*')])
|
|
+
|
|
_sum = get_human_readable(sum(self.estimated_plugsizes.values()))
|
|
self.ui_log.info("Estimated disk space requirement for whole "
|
|
"uncompressed sos report directory: %s" % _sum)
|
|
--
|
|
2.31.1
|
|
|
|
From 4293f3317505661e8f32ba94ad87310996fa1626 Mon Sep 17 00:00:00 2001
|
|
From: Eric Desrochers <eric.desrochers@canonical.com>
|
|
Date: Tue, 19 Oct 2021 12:18:40 -0400
|
|
Subject: [PATCH] [report] check for symlink before rmtree when opt
|
|
estimate-only is use
|
|
|
|
Check if the dir is also symlink before performing rmtree()
|
|
method so that unlink() method can be used instead.
|
|
|
|
Traceback (most recent call last):
|
|
File "./bin/sos", line 22, in <module>
|
|
sos.execute()
|
|
File "/tmp/sos/sos/__init__.py", line 186, in execute
|
|
self._component.execute()
|
|
OSError: Cannot call rmtree on a symbolic link
|
|
|
|
Closes: #2727
|
|
|
|
Signed-off-by: Eric Desrochers <eric.desrochers@canonical.com>
|
|
---
|
|
sos/report/__init__.py | 2 +-
|
|
1 file changed, 1 insertion(+), 1 deletion(-)
|
|
|
|
diff --git a/sos/report/__init__.py b/sos/report/__init__.py
|
|
index 7feb31ee..1b5bc97d 100644
|
|
--- a/sos/report/__init__.py
|
|
+++ b/sos/report/__init__.py
|
|
@@ -1059,7 +1059,7 @@ class SoSReport(SoSComponent):
|
|
# deletion of some dirs but deleting their content
|
|
for f in os.listdir(self.archive.get_tmp_dir()):
|
|
f = os.path.join(self.archive.get_tmp_dir(), f)
|
|
- if os.path.isdir(f):
|
|
+ if os.path.isdir(f) and not os.path.islink(f):
|
|
rmtree(f)
|
|
else:
|
|
os.unlink(f)
|
|
--
|
|
2.31.1
|
|
|
|
From 589d47c93257b55bc796ef6ac25b88c974ee3d72 Mon Sep 17 00:00:00 2001
|
|
From: Pavel Moravec <pmoravec@redhat.com>
|
|
Date: Mon, 8 Nov 2021 16:38:24 +0100
|
|
Subject: [PATCH] [report] Calculate sizes of dirs, symlinks and manifest in
|
|
estimate mode
|
|
|
|
Enhance --estimate-mode to calculate sizes of also:
|
|
- symlinks
|
|
- directories themselves
|
|
- manifest.json file
|
|
|
|
Use os.lstat() method instead of os.stat() to properly calculate the
|
|
sizes (and not destinations of symlinks, e.g.).
|
|
|
|
Print five biggest plugins instead of three as sos logs and reports do
|
|
stand as one "plugin" in the list, often.
|
|
|
|
Resolves: #2752
|
|
|
|
Signed-off-by: Pavel Moravec <pmoravec@redhat.com>
|
|
---
|
|
sos/report/__init__.py | 56 +++++++++++++++++++++---------------------
|
|
1 file changed, 28 insertions(+), 28 deletions(-)
|
|
|
|
diff --git a/sos/report/__init__.py b/sos/report/__init__.py
|
|
index 10952566..a4c92acc 100644
|
|
--- a/sos/report/__init__.py
|
|
+++ b/sos/report/__init__.py
|
|
@@ -1050,8 +1050,7 @@ class SoSReport(SoSComponent):
|
|
from pathlib import Path
|
|
tmpdir_path = Path(self.archive.get_tmp_dir())
|
|
self.estimated_plugsizes[plugin[1]] = sum(
|
|
- [f.stat().st_size for f in tmpdir_path.glob('**/*')
|
|
- if (os.path.isfile(f) and not os.path.islink(f))])
|
|
+ [f.lstat().st_size for f in tmpdir_path.glob('**/*')])
|
|
# remove whole tmp_dir content - including "sos_commands" and
|
|
# similar dirs that will be re-created on demand by next plugin
|
|
# if needed; it is less error-prone approach than skipping
|
|
@@ -1273,6 +1272,33 @@ class SoSReport(SoSComponent):
|
|
short_name='manifest.json'
|
|
)
|
|
|
|
+ # print results in estimate mode (to include also just added manifest)
|
|
+ if self.opts.estimate_only:
|
|
+ from sos.utilities import get_human_readable
|
|
+ from pathlib import Path
|
|
+ # add sos_logs, sos_reports dirs, etc., basically everything
|
|
+ # that remained in self.tmpdir after plugins' contents removal
|
|
+ # that still will be moved to the sos report final directory path
|
|
+ tmpdir_path = Path(self.tmpdir)
|
|
+ self.estimated_plugsizes['sos_logs_reports'] = sum(
|
|
+ [f.lstat().st_size for f in tmpdir_path.glob('**/*')])
|
|
+
|
|
+ _sum = get_human_readable(sum(self.estimated_plugsizes.values()))
|
|
+ self.ui_log.info("Estimated disk space requirement for whole "
|
|
+ "uncompressed sos report directory: %s" % _sum)
|
|
+ bigplugins = sorted(self.estimated_plugsizes.items(),
|
|
+ key=lambda x: x[1], reverse=True)[:5]
|
|
+ bp_out = ", ".join("%s: %s" %
|
|
+ (p, get_human_readable(v, precision=0))
|
|
+ for p, v in bigplugins)
|
|
+ self.ui_log.info("Five biggest plugins: %s" % bp_out)
|
|
+ self.ui_log.info("")
|
|
+ self.ui_log.info("Please note the estimation is relevant to the "
|
|
+ "current options.")
|
|
+ self.ui_log.info("Be aware that the real disk space requirements "
|
|
+ "might be different.")
|
|
+ self.ui_log.info("")
|
|
+
|
|
# package up and compress the results
|
|
if not self.opts.build:
|
|
old_umask = os.umask(0o077)
|
|
@@ -1377,32 +1403,6 @@ class SoSReport(SoSComponent):
|
|
self.policy.display_results(archive, directory, checksum,
|
|
map_file=map_file)
|
|
|
|
- if self.opts.estimate_only:
|
|
- from sos.utilities import get_human_readable
|
|
- from pathlib import Path
|
|
- # add sos_logs, sos_reports dirs, etc., basically everything
|
|
- # that remained in self.tmpdir after plugins' contents removal
|
|
- # that still will be moved to the sos report final directory path
|
|
- tmpdir_path = Path(self.tmpdir)
|
|
- self.estimated_plugsizes['sos_logs_reports'] = sum(
|
|
- [f.stat().st_size for f in tmpdir_path.glob('**/*')])
|
|
-
|
|
- _sum = get_human_readable(sum(self.estimated_plugsizes.values()))
|
|
- self.ui_log.info("Estimated disk space requirement for whole "
|
|
- "uncompressed sos report directory: %s" % _sum)
|
|
- bigplugins = sorted(self.estimated_plugsizes.items(),
|
|
- key=lambda x: x[1], reverse=True)[:3]
|
|
- bp_out = ", ".join("%s: %s" %
|
|
- (p, get_human_readable(v, precision=0))
|
|
- for p, v in bigplugins)
|
|
- self.ui_log.info("Three biggest plugins: %s" % bp_out)
|
|
- self.ui_log.info("")
|
|
- self.ui_log.info("Please note the estimation is relevant to the "
|
|
- "current options.")
|
|
- self.ui_log.info("Be aware that the real disk space requirements "
|
|
- "might be different.")
|
|
- self.ui_log.info("")
|
|
-
|
|
if self.opts.upload or self.opts.upload_url:
|
|
if not self.opts.build:
|
|
try:
|
|
--
|
|
2.31.1
|
|
|