From 9374fb264cd05c9f9f190eca5f3fd679173d1d73 Mon Sep 17 00:00:00 2001 From: lmchilton Date: Sun, 7 Jul 2024 22:18:50 -0400 Subject: [PATCH] Resolves: RHEL-39159 Resolves: RHEL-39132 Resolves: RHEL-32983 Resolves: RHEL-39293 --- pcp.spec | 9 +- redhat-issues-39159-39132-32983-39293.patch | 372 ++++++++++++++++++++ 2 files changed, 380 insertions(+), 1 deletion(-) create mode 100644 redhat-issues-39159-39132-32983-39293.patch diff --git a/pcp.spec b/pcp.spec index 26ad7f2..6fbc276 100644 --- a/pcp.spec +++ b/pcp.spec @@ -1,6 +1,6 @@ Name: pcp Version: 6.2.2 -Release: 3%{?dist} +Release: 4%{?dist} Summary: System-level performance monitoring and performance management License: GPL-2.0-or-later AND LGPL-2.1-or-later AND CC-BY-3.0 URL: https://pcp.io @@ -11,6 +11,7 @@ Source0: https://github.com/performancecopilot/pcp/releases/pcp-%{version}.src.t Patch1: redhat-issues-RHEL-2317-default-archive-version.patch Patch2: redhat-issues-RHEL-30198-pmcd-logdir-tmpfiles.patch Patch3: revert-time64_t-i386.patch +Patch4: redhat-issues-39159-39132-32983-39293.patch %if 0%{?fedora} >= 40 || 0%{?rhel} >= 10 ExcludeArch: %{ix86} @@ -3538,6 +3539,12 @@ fi %files zeroconf -f pcp-zeroconf-files.rpm %changelog +* Sun Jul 7 2024 Lauren Chilton - 6.2.2-4 +- Removed excess headers for pcp2openmetrics tool (RHEL-39159) +- Fix pcp2openmetrics '-s' option (RHEL-39132) +- Fix pcp2openmetrics timestamps in archive mode (RHEL-32983) +- Fix pcp2openmetrics output to adhere to prometheus parser (RHEL-39293) + * Mon Jun 10 2024 Nathan Scott - 6.2.2-3 - Revert time64_t related changes breaking i386 (RHEL-30198) diff --git a/redhat-issues-39159-39132-32983-39293.patch b/redhat-issues-39159-39132-32983-39293.patch new file mode 100644 index 0000000..5758e2e --- /dev/null +++ b/redhat-issues-39159-39132-32983-39293.patch @@ -0,0 +1,372 @@ +commit 6cefac3c6560487abad80498240545e6835e503f +Author: lmchilton +Date: Wed Jun 19 10:19:04 2024 +0200 + + Updated pcp2openmetrics tool to fix issues: RHEL-39159, RHEL-39132, RHEL-32983, RHEL-39293. + Added command line option to exclude # PCP 5 comment line + Updated output to follow openmetrics format to include _total metric name ending + Omitted blank line after each metric output to adhere to prometheus openmetrics parser + Added # EOF line to end of text dump to adhere to prometheus openmetrics + parser + Updated relevent qa for pcp2openmetric changes (1131 & 1827) + Updated pcp2openmetrics man page and fixed style errors(2) + +diff --git a/qa/1131 b/qa/1131 +index 413da6fe3..aa1e554fe 100755 +--- a/qa/1131 ++++ b/qa/1131 +@@ -62,9 +62,10 @@ pcp2json -a $A -H -I -Z UTC+0 -x "" | _archive_filter + echo "---" + pcp2json -a $A -H -I -z -X -b GB -P 2 -F $tmp.outfile "" + echo "---" +-pcp2openmetrics -s1 -H -z hinv.ncpu | _filter_pcp2openmetrics ++pcp2openmetrics -s1 -z hinv.ncpu | _filter_pcp2openmetrics ++echo "---" ++pcp2openmetrics -s2 -x hinv.ncpu | _filter_pcp2openmetrics + echo "---" +- + + cat $tmp.outfile | _archive_filter + which json_verify > /dev/null 2>&1 +diff --git a/qa/1131.out b/qa/1131.out +index 098d698a6..90cb9eaa2 100644 +--- a/qa/1131.out ++++ b/qa/1131.out +@@ -2267,13 +2267,18 @@ QA output created by 1131 + } + --- + --- +- +- ++# EOF + # HELP hinv_ncpu number of CPUs in the system + # PCP5 hinv_ncpu 60.0.32 u32 PM_INDOM_NULL discrete + # TYPE hinv_ncpu gauge + hinv_ncpu{domainname="DOMAINID",groupid="GROUPID",hostname="HOST",machineid="MACHINEID",userid="USERID",agent="linux"} NCPU + --- ++# EOF ++# HELP hinv_ncpu number of CPUs in the system ++# TYPE hinv_ncpu gauge ++hinv_ncpu{domainname="DOMAINID",groupid="GROUPID",hostname="HOST",machineid="MACHINEID",userid="USERID",agent="linux"} NCPU ++hinv_ncpu{domainname="DOMAINID",groupid="GROUPID",hostname="HOST",machineid="MACHINEID",userid="USERID",agent="linux"} NCPU ++--- + { + "@pcp": { + "@hosts": [ +diff --git a/qa/1827 b/qa/1827 +index 47ee66d54..ebeaad460 100755 +--- a/qa/1827 ++++ b/qa/1827 +@@ -63,12 +63,20 @@ $PCP_PYTHON_PROG $here/src/pythonserver.py $port >$tmp.python.out 2>&1 & + pid=$! + sleep 2 # let server start up + ++echo "---" + echo "pcp2openmetrics invocation" | tee -a $here/$seq.full + pcp2openmetrics -s1 -u http://localhost:$port hinv.ncpu >$tmp.openmetrics.out 2>$tmp.openmetrics.err + + echo "pcp2openmetrics HTTP POST (sorted):" + _filter_pcp2openmetrics_http <$tmp.python.out + ++cat /dev/null > $tmp.python.out ++ ++echo "---" ++echo "testing -x option" | tee -a $here/$seq.full ++pcp2openmetrics -s1 -x -u http://localhost:$port hinv.ncpu >$tmp.openmetrics2.out 2>$tmp.openmetrics2.err ++_filter_pcp2openmetrics_http <$tmp.python.out ++ + # terminate pythonserver.py now + pmsignal $pid >/dev/null 2>&1 + +diff --git a/qa/1827.out b/qa/1827.out +index e786d8607..191ebcfa8 100644 +--- a/qa/1827.out ++++ b/qa/1827.out +@@ -1,4 +1,5 @@ + QA output created by 1827 ++--- + pcp2openmetrics invocation + pcp2openmetrics HTTP POST (sorted): + +@@ -21,3 +22,22 @@ INFO:root:Starting httpd... + Path: / + User-Agent: python-requests VERSION + hinv_ncpu{domainname="DOMAINID",groupid="GROUPID",hostname="HOST",machineid="MACHINEID",userid="USERID",agent="linux"} N (cpus) ++--- ++testing -x option ++ ++ ++ ++ ++ ++# HELP hinv_ncpu number of CPUs in the system ++# TYPE hinv_ncpu gauge ++Accept: */* ++Body: ++Content-Length: SIZE ++Content-Type: application/openmetrics-text ++Headers: ++Host: localhost:PORT ++INFO:root:POST request, ++Path: / ++User-Agent: python-requests VERSION ++hinv_ncpu{domainname="DOMAINID",groupid="GROUPID",hostname="HOST",machineid="MACHINEID",userid="USERID",agent="linux"} N (cpus) +diff --git a/src/pcp2openmetrics/pcp2openmetrics.1 b/src/pcp2openmetrics/pcp2openmetrics.1 +index 84e81e7af..41e4e2f9c 100644 +--- a/src/pcp2openmetrics/pcp2openmetrics.1 ++++ b/src/pcp2openmetrics/pcp2openmetrics.1 +@@ -181,6 +181,13 @@ Maximum time (in seconds) when sending a HTTP POST. + Corresponding command line option is \fB\-o\fP. + Defaults to \fB2.5\fP seconds. + .RE ++.PP ++no-comment (boolean) ++.RS 4 ++Omit # PCP5 comment line. Omits header for some metric ++data such as type, instance domain, and semantics. ++Corresponding command line option is \fB\-x\fP. ++Defaults to \fBno\fP. + .SH OPTIONS + The available command line options are: + .TP 5 +@@ -343,9 +350,6 @@ on + .IR host , + rather than from the default localhost. + .TP +-\fB\-H\fR, \fB\-\-no\-header\fR +-Do not print any headers. +-.TP + \fB\-i\fR \fIinstances\fR, \fB\-\-instances\fR=\fIinstances\fR + Retrieve and report only the specified metric + .IR instances . +@@ -605,8 +609,8 @@ and + \fB\-V\fR, \fB\-\-version\fR + Display version number and exit. + .TP +-\fB\-x\fR, \fB\-\-with\-extended\fR +-Write extended information. ++\fB\-x\fR, \fB\-no\-comment\fR ++Omit # PCP5 comment line + .TP + \fB\-X\fR, \fB\-\-with\-everything\fR + Write everything known about metrics, including PCP internal IDs. +diff --git a/src/pcp2openmetrics/pcp2openmetrics.py b/src/pcp2openmetrics/pcp2openmetrics.py +index 51b3326d7..d5a3346fe 100755 +--- a/src/pcp2openmetrics/pcp2openmetrics.py ++++ b/src/pcp2openmetrics/pcp2openmetrics.py +@@ -50,24 +50,24 @@ class PCP2OPENMETRICS(object): + self.opts = self.options() + + # Configuration directives +- self.keys = ('source', 'output', 'derived', 'header', 'globals', +- 'samples', 'interval', 'type', 'precision', 'daemonize', ++ self.keys = ('source', 'output', 'derived', 'globals', ++ 'samples', 'interval', 'precision', 'daemonize', + 'timefmt', 'everything', + 'count_scale', 'space_scale', 'time_scale', 'version', + 'count_scale_force', 'space_scale_force', 'time_scale_force', +- 'type_prefer', 'precision_force', 'limit_filter', 'limit_filter_force', ++ 'precision_force', 'limit_filter', 'limit_filter_force', + 'live_filter', 'rank', 'invert_filter', 'predicate', 'names_change', + 'speclocal', 'instances', 'ignore_incompat', 'ignore_unknown', + 'omit_flat', 'include_labels', 'url', 'http_user', 'http_pass', +- 'http_timeout') ++ 'http_timeout', 'no_comment') + + # Ignored for pmrep(1) compatibility + self.keys_ignore = ( +- 'timestamp', 'unitinfo', 'colxrow', 'separate_header', 'fixed_header', ++ 'timestamp','header', 'unitinfo', 'colxrow', 'separate_header', 'fixed_header', + 'delay', 'width', 'delimiter', 'extcsv', 'width_force', + 'extheader', 'repeat_header', 'interpol', + 'dynamic_header', 'overall_rank', 'overall_rank_alt', 'sort_metric', +- 'instinfo', 'include_texts') ++ 'instinfo', 'include_texts', 'type', 'type_prefer') + + # The order of preference for options (as present): + # 1 - command line options +@@ -79,13 +79,12 @@ class PCP2OPENMETRICS(object): + self.output = None # For pmrep conf file compat only + self.speclocal = None + self.derived = None +- self.header = 1 + self.globals = 1 + self.samples = None # forever + self.interval = pmapi.timeval(10) # 10 sec + self.opts.pmSetOptionInterval(str(10)) # 10 sec + self.delay = 0 +- self.type = 0 ++ self.type = 1 + self.type_prefer = self.type + self.ignore_incompat = 0 + self.ignore_unknown = 0 +@@ -118,6 +117,8 @@ class PCP2OPENMETRICS(object): + self.http_user = None + self.http_pass = None + self.http_timeout = TIMEOUT ++ self.no_comment = False ++ self.header_flag = True + + # Internal + self.runtime = -1 +@@ -146,7 +147,7 @@ class PCP2OPENMETRICS(object): + opts = pmapi.pmOptions() + opts.pmSetOptionCallback(self.option) + opts.pmSetOverrideCallback(self.option_override) +- opts.pmSetShortOptions("a:h:LK:c:Ce:D:V?HGA:S:T:O:s:t:rRIi:jJ:4:58:9:nN:vmP:0:q:b:y:Q:B:Y:F:f:Z:zXo:p:U:u:") ++ opts.pmSetShortOptions("a:h:LK:c:Ce:D:V?GA:S:T:O:s:t:Ii:jJ:4:58:9:nN:vmP:0:q:b:y:Q:B:Y:F:f:Z:zXo:p:U:u:x") + opts.pmSetShortUsage("[option...] metricspec [...]") + + opts.pmSetLongOptionHeader("General options") +@@ -166,7 +167,6 @@ class PCP2OPENMETRICS(object): + opts.pmSetLongOptionHelp() # -?/--help + + opts.pmSetLongOptionHeader("Reporting options") +- opts.pmSetLongOption("no-header", 0, "H", "", "omit headers") + opts.pmSetLongOption("no-globals", 0, "G", "", "omit global metrics") + opts.pmSetLongOptionAlign() # -A/--align + opts.pmSetLongOptionStart() # -S/--start +@@ -176,8 +176,6 @@ class PCP2OPENMETRICS(object): + opts.pmSetLongOptionInterval() # -t/--interval + opts.pmSetLongOptionTimeZone() # -Z/--timezone + opts.pmSetLongOptionHostZone() # -z/--hostzone +- opts.pmSetLongOption("raw", 0, "r", "", "output raw counter values (no rate conversion)") +- opts.pmSetLongOption("raw-prefer", 0, "R", "", "prefer output raw counter values (no rate conversion)") + opts.pmSetLongOption("ignore-incompat", 0, "I", "", "ignore incompatible instances (default: abort)") + opts.pmSetLongOption("ignore-unknown", 0, "5", "", "ignore unknown metrics (default: abort)") + opts.pmSetLongOption("names-change", 1, "4", "ACTION", "update/ignore/abort on PMNS change (default: ignore)") +@@ -201,6 +199,7 @@ class PCP2OPENMETRICS(object): + opts.pmSetLongOption("time-scale-force", 1, "Y", "SCALE", "forced time unit") + + opts.pmSetLongOption("with-everything", 0, "X", "", "write everything, incl. internal IDs") ++ opts.pmSetLongOption("no-comment", 0, "x", "", "omit comment lines") + + opts.pmSetLongOption("url", 1, "u", "URL", "URL of endpoint to receive HTTP POST") + opts.pmSetLongOption("http-timeout", 1, "o", "SECONDS", "timeout when sending HTTP POST") +@@ -238,14 +237,8 @@ class PCP2OPENMETRICS(object): + self.derived = ";" + optarg + else: + self.derived = self.derived + ";" + optarg +- elif opt == 'H': +- self.header = 0 + elif opt == 'G': + self.globals = 0 +- elif opt == 'r': +- self.type = 1 +- elif opt == 'R': +- self.type_prefer = 1 + elif opt == 'I': + self.ignore_incompat = 1 + elif opt == '5': +@@ -298,6 +291,8 @@ class PCP2OPENMETRICS(object): + self.time_scale_force = optarg + elif opt == 'X': + self.everything = 1 ++ elif opt == 'x': ++ self.no_comment = True + elif opt == 'u': + self.url = optarg + elif opt == 'o': +@@ -319,6 +314,7 @@ class PCP2OPENMETRICS(object): + + if pmapi.c_api.pmSetContextOptions(self.context.ctx, self.opts.mode, self.opts.delta): + raise pmapi.pmUsageErr() ++ self.context.pmNewZone("UTC") + + def validate_config(self): + """ Validate configuration options """ +@@ -346,11 +342,6 @@ class PCP2OPENMETRICS(object): + # Common preparations + self.context.prepare_execute(self.opts, False, self.interpol, self.interval) + +- # Headers +- if self.header == 1: +- self.header = 0 +- self.write_header() +- + # Just checking + if self.check == 1: + return +@@ -394,29 +385,12 @@ class PCP2OPENMETRICS(object): + + self.write_openmetrics(tstamp) + +- def write_header(self): +- """ Write info header """ +- output = self.outfile if self.outfile else "stdout" +- if self.context.type == PM_CONTEXT_ARCHIVE: +- sys.stdout.write('"# Writing %d archived metrics to %s..." }\n{ "//": "(Ctrl-C to stop)" }\n' % (len(self.metrics), output)) +- return +- +- sys.stdout.write('# "Waiting for %d metrics to be written to %s' % (len(self.metrics), output)) +- if self.runtime != -1: +- sys.stdout.write('\n # "%s samples(s) with %.1f sec interval ~ %d sec runtime." }\n' % (self.samples, float(self.interval), self.runtime)) +- elif self.samples: +- duration = (self.samples - 1) * float(self.interval) +- sys.stdout.write('\n # "%s samples(s) with %.1f sec interval ~ %d sec runtime." }\n' % (self.samples, float(self.interval), duration)) +- else: +- sys.stdout.write('...\n# "(Ctrl-C to stop)" }\n') +- + def write_openmetrics(self, timestamp): + """ Write results in openmetrics format """ + if timestamp is None: + # Silent goodbye, close in finalize() + return + +- self.context.pmNewZone("UTC") + ts = self.context.datetime_to_secs(self.pmfg_ts(), PM_TIME_SEC) + + if self.prev_ts is None: +@@ -500,11 +474,12 @@ class PCP2OPENMETRICS(object): + help_dict = {} + help_dict[metric] = context.pmLookupText(pmid[0]) + +- if self.context.type == PM_CONTEXT_ARCHIVE: +- body += "\nMetric %s details (last fetch: %d)\n:" % (metric, ts) +- body += '# PCP5 %s %s %s %s %s %s\n' % (openmetrics_name(metric), pmIDStr, get_type_string(desc), pmIndomStr, semantics, units) +- body += '# HELP %s %s\n' % (openmetrics_name(metric), help_dict[metric]) +- body += '# TYPE %s %s\n' % (openmetrics_name(metric), openmetrics_type(desc)) ++ if self.header_flag is True: ++ if self.no_comment is False: ++ body += '# PCP5 %s %s %s %s %s %s\n' % (openmetrics_name(metric), pmIDStr, get_type_string(desc), pmIndomStr, semantics, units) ++ body += '# TYPE %s %s\n' % (openmetrics_name(metric), openmetrics_type(desc)) ++ body += '# HELP %s %s\n' % (openmetrics_name(metric), help_dict[metric]) ++ self.header_flag = False + + for inst, name, value in results[metric]: + if isinstance(value, float): +@@ -515,10 +490,16 @@ class PCP2OPENMETRICS(object): + value = format(value, fmt) + else: + str(value) ++ ++ if openmetrics_type(desc) == "counter": ++ openmetrics_name_end = openmetrics_name(metric) + "_total" ++ else: ++ openmetrics_name_end = openmetrics_name(metric) ++ + if self.context.type == PM_CONTEXT_ARCHIVE: +- body += '%s%s %s %s\n' % (openmetrics_name(metric), openmetrics_labels(inst, name, desc, labels), value, ts) ++ body += '%s%s %s %s\n' % (openmetrics_name_end, openmetrics_labels(inst, name, desc, labels), value, ts) + else: +- body += '%s%s %s\n' % (openmetrics_name(metric), openmetrics_labels(inst, name, desc, labels), value) ++ body += '%s%s %s\n' % (openmetrics_name_end, openmetrics_labels(inst, name, desc, labels), value) + + if self.url: + auth = None +@@ -537,13 +518,13 @@ class PCP2OPENMETRICS(object): + elif self.outfile: + self.writer.write(body) + else: +- print(body) ++ sys.stdout.write(body) + + def finalize(self): + """ Finalize and clean up """ + if self.writer: + try: +- self.writer.write("\n") ++ self.writer.write("# EOF\n") + self.writer.flush() + except IOError as write_error: + if write_error.errno != errno.EPIPE: