rpms/pcp
6
0
Fork 0
Code Issues Pull Requests Releases Activity
c9s
pcp/pmda-openmetrics-rollup.patch
lmchilton f5f1fcd82a Resolves: RHEL-101745 
Resolves: RHEL-106772
Resolves: RHEL-79767
2025-11-19 14:00:02 -05:00

422 lines
18 KiB
Diff
Raw Permalink Blame History

diff -Naurp pcp-6.3.7.orig/qa/1976 pcp-6.3.7/qa/1976
--- pcp-6.3.7.orig/qa/1976 1969-12-31 19:00:00.000000000 -0500
+++ pcp-6.3.7/qa/1976 2025-10-14 09:47:42.874408447 -0400
@@ -0,0 +1,85 @@
+#!/bin/sh
+# PCP QA Test No. 1976
+# Test pmdaopenmetrics metric removal
+#
+# Copyright (c) 2017, 2025 Red Hat. All Rights Reserved.
+#
+seq=`basename $0`
+echo "QA output created by $seq"
+
+# get standard environment, filters and checks
+. ./common.openmetrics
+
+_pmdaopenmetrics_check || _notrun "openmetrics pmda not installed"
+
+status=1 # failure is the default!
+
+_cleanup()
+{
+ cd $here
+ _pmdaopenmetrics_cleanup
+ $sudo rm -rf $tmp $tmp.*
+}
+
+_prepare_pmda openmetrics
+trap "_cleanup; exit \$status" 0 1 2 3 15
+_stop_auto_restart pmcd
+
+_pmdaopenmetrics_save_config
+
+# add all the sample text files as urls.
+# need to be a place the user $PCP_USER (pmcd) can read
+#
+( cd $here/openmetrics/samples; ls -1 *.txt ) | sort | while read file
+do
+ cp $here/openmetrics/samples/$file $tmp.$file
+ urlbase=`basename "$file" .txt | tr .- _`
+ echo 'file://'$tmp.$file >$tmp.tmp
+ $sudo cp $tmp.tmp $PCP_PMDAS_DIR/openmetrics/config.d/$urlbase.url
+done
+ls -l $PCP_PMDAS_DIR/openmetrics/config.d >>$seq_full
+
+# add all the sample scripts
+cp -a $here/openmetrics/scripts/* $PCP_PMDAS_DIR/openmetrics/config.d
+find $PCP_PMDAS_DIR/openmetrics/config.d -name GNU\* -exec rm -f {} ";"
+
+_pmdaopenmetrics_install
+
+if ! _pmdaopenmetrics_wait_for_metric openmetrics.thermostat
+then
+ status=1
+ exit
+fi
+
+echo "-- metric removal of new source/metric --"
+$sudo rm $PCP_PMDAS_DIR/openmetrics/config.d/simple_metric.url
+pminfo openmetrics.simple_metric
+echo
+
+echo "-- source re-addition --"
+# same access controls logic as above, user $PCP_USER needs to be
+# able to read the file at the end of the URL
+#
+cp $here/openmetrics/samples/simple_metric.txt $tmp.simple_metric.txt
+echo 'file:///'$tmp.simple_metric.txt >$PCP_PMDAS_DIR/openmetrics/config.d/simple_metric.url
+pminfo openmetrics.simple_metric
+echo
+
+echo "-- metric removal of recognized source/metric --"
+$sudo rm $PCP_PMDAS_DIR/openmetrics/config.d/simple_metric.url
+pminfo openmetrics.simple_metric
+echo
+
+echo "-- source re-addition with epoch timestamp --"
+txtpath=$here/openmetrics/samples/simple_metric.txt
+urlfile=$PCP_PMDAS_DIR/openmetrics/config.d/simple_metric.url
+echo 'file:///'$txtpath > $urlfile
+$sudo touch -t 197001010000 $urlfile
+pminfo openmetrics.simple_metric
+echo
+
+_pmdaopenmetrics_remove >/dev/null 2>&1
+
+# success, all done
+status=0
+exit
diff -Naurp pcp-6.3.7.orig/qa/1976.out pcp-6.3.7/qa/1976.out
--- pcp-6.3.7.orig/qa/1976.out 1969-12-31 19:00:00.000000000 -0500
+++ pcp-6.3.7/qa/1976.out 2025-10-14 09:47:42.874575552 -0400
@@ -0,0 +1,17 @@
+QA output created by 1976
+
+=== openmetrics agent installation ===
+-- metric removal of new source/metric --
+Error: openmetrics.simple_metric: Unknown metric name
+
+-- source re-addition --
+openmetrics.simple_metric.metric2
+openmetrics.simple_metric.metric1
+
+-- metric removal of recognized source/metric --
+Error: openmetrics.simple_metric: Unknown metric name
+
+-- source re-addition with epoch timestamp --
+openmetrics.simple_metric.metric2
+openmetrics.simple_metric.metric1
+
diff -Naurp pcp-6.3.7.orig/qa/group pcp-6.3.7/qa/group
--- pcp-6.3.7.orig/qa/group 2025-03-30 21:41:26.000000000 -0400
+++ pcp-6.3.7/qa/group 2025-10-14 09:47:42.874918504 -0400
@@ -2205,6 +2205,7 @@ pmcd.pdu
1963 pmda.linux local
1970 pmda.bpf local
1973 pcp zoneinfo python local
+1976 pmdaopenmetrics python local
1978 atop local pmlogrewrite
1984 pmlogconf pmda.redis local
1985 pmfind local valgrind
diff -Naurp pcp-6.3.7.orig/qa/openmetrics/scripts/curl/script.sh pcp-6.3.7/qa/openmetrics/scripts/curl/script.sh
--- pcp-6.3.7.orig/qa/openmetrics/scripts/curl/script.sh 2023-11-16 01:51:39.000000000 -0500
+++ pcp-6.3.7/qa/openmetrics/scripts/curl/script.sh 2025-10-14 09:47:42.875376079 -0400
@@ -1,4 +1,4 @@
#! /bin/sh
. /etc/pcp.conf
-curl -Gq file://$PCP_PMDAS_DIR/openmetrics/config.d/some_metric.txt
+curl -Gqs file://$PCP_PMDAS_DIR/openmetrics/config.d/some_metric.txt
diff -Naurp pcp-6.3.7.orig/qa/openmetrics/scripts/curl_filtered.sh pcp-6.3.7/qa/openmetrics/scripts/curl_filtered.sh
--- pcp-6.3.7.orig/qa/openmetrics/scripts/curl_filtered.sh 2023-11-16 01:51:39.000000000 -0500
+++ pcp-6.3.7/qa/openmetrics/scripts/curl_filtered.sh 2025-10-14 09:47:42.875590832 -0400
@@ -7,6 +7,6 @@
. /etc/pcp.conf
-( curl -Gq file://$PCP_PMDAS_DIR/openmetrics/config.d/some_metric.txt ; \
-curl -Gq file://$PCP_PMDAS_DIR/openmetrics/config.d/some_other_metric.txt) \
+( curl -Gqs file://$PCP_PMDAS_DIR/openmetrics/config.d/some_metric.txt ; \
+curl -Gqs file://$PCP_PMDAS_DIR/openmetrics/config.d/some_other_metric.txt) \
| sed -e '/metric2/d'
diff -Naurp pcp-6.3.7.orig/qa/openmetrics/scripts/curl_hostname_label.sh pcp-6.3.7/qa/openmetrics/scripts/curl_hostname_label.sh
--- pcp-6.3.7.orig/qa/openmetrics/scripts/curl_hostname_label.sh 2023-11-16 01:51:39.000000000 -0500
+++ pcp-6.3.7/qa/openmetrics/scripts/curl_hostname_label.sh 2025-10-14 09:47:42.875772077 -0400
@@ -12,5 +12,5 @@
# here for QA purposes we're fetching from a local file
# and just pretending it came from a remote host.
-curl -q -G file://$PCP_PMDAS_DIR/openmetrics/config.d/some_metric.txt 2>/dev/null | \
+curl -Gqs file://$PCP_PMDAS_DIR/openmetrics/config.d/some_metric.txt 2>/dev/null | \
sed -e 's/[a-z0-9]*=/hostname="remotehost",&/'
diff -Naurp pcp-6.3.7.orig/qa/openmetrics/scripts/curl_scripted.sh pcp-6.3.7/qa/openmetrics/scripts/curl_scripted.sh
--- pcp-6.3.7.orig/qa/openmetrics/scripts/curl_scripted.sh 2023-11-16 01:51:39.000000000 -0500
+++ pcp-6.3.7/qa/openmetrics/scripts/curl_scripted.sh 2025-10-14 09:47:42.875946730 -0400
@@ -1,5 +1,5 @@
#! /bin/sh
. /etc/pcp.conf
-curl -Gq file://$PCP_PMDAS_DIR/openmetrics/config.d/some_metric.txt
-curl -Gq file://$PCP_PMDAS_DIR/openmetrics/config.d/some_other_metric.txt
+curl -Gqs file://$PCP_PMDAS_DIR/openmetrics/config.d/some_metric.txt
+curl -Gqs file://$PCP_PMDAS_DIR/openmetrics/config.d/some_other_metric.txt
diff -Naurp pcp-6.3.7.orig/src/pmdas/openmetrics/pmdaopenmetrics.python pcp-6.3.7/src/pmdas/openmetrics/pmdaopenmetrics.python
--- pcp-6.3.7.orig/src/pmdas/openmetrics/pmdaopenmetrics.python 2025-03-27 02:01:59.000000000 -0400
+++ pcp-6.3.7/src/pmdas/openmetrics/pmdaopenmetrics.python 2025-10-14 09:51:21.874137509 -0400
@@ -32,7 +32,7 @@ import subprocess
import sys
from ctypes import c_int
from socket import gethostname
-from stat import ST_MODE, S_IXUSR, ST_CTIME
+from stat import ST_MODE, S_IXUSR
import requests
from pcp.pmapi import pmUnits, pmContext
@@ -127,15 +127,9 @@ class Metric(object):
(name, pmContext.pmIDStr(self.pmid), self.mtype, self.msem, self.singular, self.mindom, self.labels))
self.obj = pmdaMetric(self.pmid, self.mtype, self.mindom, self.msem, self.munits)
+ self.source.pmda.all_metrics[self.mname] = self.obj
- if helpline: # it could be None!
- unescaped = helpline.replace('\\\\', '\\').replace('\\n', '\n')
- split = unescaped.split('\n')
- help_oneline = split[0] # must have at least one entry
- help_text = '\n'.join(split[1:]) # may have other entries
- else:
- help_oneline = ''
- help_text = ''
+ help_text, help_oneline = self.source.helptext(helpline)
try:
self.source.pmda.add_metric(self.mname, self.obj, help_oneline, help_text)
@@ -565,7 +559,7 @@ class Source(object):
self.path = path # pathname to .url or executable file
self.url = None
self.parse_error = False
- self.parse_url_time = 0 # timestamp of config file when it was last parsed
+ self.parse_time = None # last time config file was parsed
self.is_scripted = is_scripted
self.pmda = thispmda # the shared pmda
self.requests = None
@@ -588,6 +582,16 @@ class Source(object):
self.metrics_by_name = {} # name -> Metric
self.metrics_by_num = {} # number (last component of pmid) -> Metric
+ def helptext(self, helpline):
+ if helpline: # it could be None!
+ unescaped = helpline.replace('\\\\', '\\').replace('\\n', '\n')
+ split = unescaped.split('\n')
+ help_oneline = split[0] # must have at least one entry
+ help_text = '\n'.join(split[1:]) # may have other entries
+ else:
+ help_oneline = ''
+ help_text = ''
+ return help_text, help_oneline
def old_enough_for_refresh(self):
'''But what is "old"? If it is empty (no metrics), then it
@@ -685,6 +689,23 @@ class Source(object):
self.pmda.debug("included_labels '%s'" % (included_labels)) if self.pmda.dbg else None
self.pmda.debug("optional_labels '%s'" % (optional_labels)) if self.pmda.dbg else None
if sp.name in self.metrics_by_name:
+ if ("openmetrics.%s.%s" % (self.name, sp.name)) not in self.pmda.all_metrics and self.name in self.pmda.re_add_list:
+ # re-add metric to namespace
+ if pcpline:
+ split = pcpline.split(" ")
+ fullname = "openmetrics.%s.%s" % (self.name, split[1])
+ else:
+ fullname = "openmetrics.%s.%s" % (self.name, sp.name.replace(":", "."))
+ help_oneline, help_text = self.helptext(helpline)
+ try:
+ obj = self.pmda.removed_metrics[fullname]
+ self.pmda.add_metric(fullname, obj, help_oneline, help_text)
+ self.pmda.debug("re-adding metric: %s to namespace" % fullname) if self.pmda.dbg else None
+ self.pmda.all_metrics[fullname] = obj
+ del self.pmda.removed_metrics[fullname]
+ self.pmda.set_need_refresh()
+ except Exception as e:
+ self.pmda.debug("Can't re-add metric: %s, see error: %s" % (fullname, e)) if self.pmda.dbg else None
m = self.metrics_by_name[sp.name]
assert self.metrics_by_num[m.metricnum] == m
if m.singular:
@@ -693,6 +714,7 @@ class Source(object):
else:
m.store_inst(naming_labels, sp.value)
self.pmda.debug("naming_labels '%s'" % (naming_labels)) if self.pmda.dbg else None
+ # new metric case
else:
# check metric is not excluded by filters
fullname = "openmetrics.%s.%s" % (self.name, sp.name)
@@ -800,9 +822,9 @@ class Source(object):
return num_metrics
- def parse_url_config(self, filepath):
+ def parse_config(self, filepath):
'''
- Parse a URL config file. The first line is always the URL.
+ Parse a configuration file. The first line is always the URL.
Remaining lines are prefixed with a keyword. Supported keywords
include '#' for a comment, 'HEADER:' to add to the header passed
to the headers dict parameter to the get() call. Note the ':' are
@@ -900,12 +922,12 @@ class Source(object):
if not s[ST_MODE] & S_IXUSR:
self.pmda.err("cannot execute script '%s'" % self.path)
return
- elif self.parse_url_time < s[ST_CTIME]:
+ elif self.parse_time is None or self.parse_time < s.st_mtime_ns:
# (re)parse the URL from given file
- self.parse_url_config(self.path)
- self.parse_url_time = s[ST_CTIME]
+ self.parse_config(self.path)
+ self.parse_time = s.st_mtime_ns
except Exception as e:
- self.pmda.err("cannot read %s: %s" % (self.path, e))
+ self.pmda.err("cannot stat %s: %s" % (self.path, e))
return
# fetch the document
@@ -1001,7 +1023,7 @@ class OpenMetricsPMDA(PMDA):
# now everything else may take time
self.pmda_name = pmda_name
self.config_dir = os.path.normpath(config)
- self.config_dir_ctime = None
+ self.config_dir_mtime = 0.0
self.timeout = timeout
# a single central Session that all our sources can concurrently reuse
@@ -1014,6 +1036,10 @@ class OpenMetricsPMDA(PMDA):
reserved_cluster = self.cluster_table.intern_lookup_value("control")
assert reserved_cluster == 0
self.source_by_cluster = {}
+ # all metrics added, to be used for removal
+ self.all_metrics = {}
+ # keep track of removed metrics, in case of re-addition
+ self.removed_metrics = {}
# compiled regex cache
self.regex_cache = {}
@@ -1053,7 +1079,9 @@ class OpenMetricsPMDA(PMDA):
self.add_metric('%s.control.debug' % self.pmda_name, pmdaMetric(self.pmid(0, 4),
c_api.PM_TYPE_U32, c_api.PM_INDOM_NULL, c_api.PM_SEM_DISCRETE,
pmUnits(0, 0, 0, 0, 0, 0)),
- 'debug flag to enable verbose log messages, to enable: pmstore %s.control.debug 1' % self.pmda_name)
+ 'debug flag to enable verbose log messages',
+ 'Debugging flag to enable verbose log messages, to enable:\n'
+ '> pmstore %s.control.debug 1' % self.pmda_name)
# response status string, per-source end-point
self.stats_status = {0:"none"} # status string, keyed by cluster number
@@ -1106,26 +1134,26 @@ class OpenMetricsPMDA(PMDA):
assert s == self.source_by_name[s.name]
- def traverse(self, directory, ctime):
+ def traverse(self, directory, mtime):
''' Return list of files below dir, recursively '''
ret = []
- m = os.path.getctime(directory)
- if ctime is None or m > ctime:
- ctime = m
+ m = os.path.getmtime(directory)
+ if mtime is None or m > mtime:
+ mtime = m
for path, subdirs, files in os.walk(directory):
for f in files:
if not f.startswith("."):
fname = os.path.join(path, f)
- m = os.path.getctime(fname)
- if ctime is None or m > ctime:
- ctime = m
+ m = os.path.getmtime(fname)
+ if mtime is None or m > mtime:
+ mtime = m
ret.append(fname)
for d in subdirs:
- m, _ = self.traverse(os.path.join(path, d), ctime)
- if ctime is None or m > ctime:
- ctime = m
- return ctime, ret
+ m, _ = self.traverse(os.path.join(path, d), mtime)
+ if mtime is None or m > mtime:
+ mtime = m
+ return mtime, ret
def rescan_confdir(self):
'''Scan the configuration directories for any new .url files
@@ -1138,18 +1166,50 @@ class OpenMetricsPMDA(PMDA):
'''
traverse_time = time.time()
- dir_ctime, conf_filelist = self.traverse(self.config_dir, self.config_dir_ctime)
+ dir_mtime, conf_filelist = self.traverse(self.config_dir, self.config_dir_mtime)
traverse_time = time.time() - traverse_time
- if self.config_dir_ctime is None or self.config_dir_ctime < dir_ctime:
- self.config_dir_ctime = dir_ctime
+ if self.config_dir_mtime < dir_mtime:
+ self.config_dir_mtime = dir_mtime
else: # no new or changed conf files, don't rescan directory
return
self.log("Config change detected, traversed %d config entries in %.04fs, rescanning ..." % (len(conf_filelist), traverse_time))
nickname_regexp = self.lookup_regex(r"^[A-Za-z][A-Za-z0-9_.]*$")
+ self.re_add_list = []
+
+ # calculate config entry nicknames
+ nicknames = []
+ for file in conf_filelist:
+ file_split = os.path.splitext(file)
+ name = file_split[0].replace(self.config_dir + "/", "").replace("/", ".")
+ nicknames.append(name)
+
+ # check if config change adds a previously removed source
+ for key in self.removed_metrics:
+ split_name = key.split(".")
+ if split_name[1] in nicknames:
+ self.re_add_list.append(split_name[1])
+
+ # if source is not in config directory, remove the metric
+ for key, value in self.all_metrics.items():
+ split_name = key.split(".")
+ if split_name[1] in nicknames:
+ continue
+ try:
+ remove_name = key
+ remove_obj = value
+ self.remove_metric(remove_name, remove_obj)
+ self.removed_metrics[remove_name] = remove_obj
+ self.debug("removed metric name: %s" % remove_name) if self.dbg else None
+ self.set_need_refresh()
+ except Exception as e:
+ self.debug("can't remove metric: %s, see error: %s" % (key, e)) if self.dbg else None
+
+ for key in self.removed_metrics:
+ if key in self.all_metrics:
+ del self.all_metrics[key]
- # TODO: maybe nuke sources related to removed files
save_cluster_table = False
if sort_conf_list:
# sorted for indom cluster consistency
@@ -1177,6 +1237,16 @@ class OpenMetricsPMDA(PMDA):
if name in self.source_by_name:
# this source is already known
self.assert_source_invariants(name=name)
+ s = self.source_by_name[name]
+ cluster_for_refresh = []
+ cluster_for_refresh_names = []
+ if name in self.re_add_list:
+ for key,value in self.source_by_cluster.items():
+ if value == s:
+ cluster_for_refresh.append(key)
+ cluster_for_refresh_names.append(name)
+ self.debug("refreshing cluster list: %s" % cluster_for_refresh_names) if self.dbg else None
+ self.refresh_some_clusters_for_fetch(cluster_for_refresh)
else:
try:
path = file
Reference in New Issue View Git Blame Copy Permalink