From 9249a81849edbc0c60572c58e11d7a9f9dc1d313 Mon Sep 17 00:00:00 2001 From: eabdullin Date: Wed, 27 Sep 2023 13:53:55 +0000 Subject: [PATCH] import CS pcp-5.3.7-18.el8 --- .gitignore | 1 + .pcp.metadata | 1 + ...t-bugzilla-1981886-pcp-ss-fetchgroup.patch | 326 ++ ...zilla-2139325-openmetrics-in-grafana.patch | 317 ++ ...t-bugzilla-2150889-nfsclient-srcport.patch | 706 +++++ ...ugzilla-2159207-pmproxy-rollup-fixes.patch | 2711 +++++++++++++++++ ...t-bugzilla-2211263-pmcd-conf-rewrite.patch | 94 + SPECS/pcp.spec | 26 +- 8 files changed, 4179 insertions(+), 3 deletions(-) create mode 100644 SOURCES/redhat-bugzilla-1981886-pcp-ss-fetchgroup.patch create mode 100644 SOURCES/redhat-bugzilla-2139325-openmetrics-in-grafana.patch create mode 100644 SOURCES/redhat-bugzilla-2150889-nfsclient-srcport.patch create mode 100644 SOURCES/redhat-bugzilla-2159207-pmproxy-rollup-fixes.patch create mode 100644 SOURCES/redhat-bugzilla-2211263-pmcd-conf-rewrite.patch diff --git a/.gitignore b/.gitignore index 052fd3e..4fb7ff5 100644 --- a/.gitignore +++ b/.gitignore @@ -1 +1,2 @@ SOURCES/pcp-5.3.7.src.tar.gz +SOURCES/redhat-bugzilla-2219731-hacluster-metrics.patch diff --git a/.pcp.metadata b/.pcp.metadata index e64a399..0b967ee 100644 --- a/.pcp.metadata +++ b/.pcp.metadata @@ -1 +1,2 @@ a0a05bf501b016cb859fb211ae60ce18be2bbd99 SOURCES/pcp-5.3.7.src.tar.gz +bb84c58586e078343c975ed61a8b53d4fd4393e8 SOURCES/redhat-bugzilla-2219731-hacluster-metrics.patch diff --git a/SOURCES/redhat-bugzilla-1981886-pcp-ss-fetchgroup.patch b/SOURCES/redhat-bugzilla-1981886-pcp-ss-fetchgroup.patch new file mode 100644 index 0000000..1d0693f --- /dev/null +++ b/SOURCES/redhat-bugzilla-1981886-pcp-ss-fetchgroup.patch @@ -0,0 +1,326 @@ +commit c226e98096c7bdb6296a96257439724e1f68217e +Author: Nathan Scott +Date: Tue Apr 26 12:00:32 2022 +1000 + + libpcp: optimize indom handling in fetchgroup code + + Looking into some pcp-ss issues I immediately encountered a + noticable elapsed time difference between executing just ss + versus pcp-ss. It turned out it was due to fetchgroup code + not really understanding how the underlying indom APIs work + and repeatedly refreshing shared indoms (all metrics in the + pcp-ss tool actually share one indom) once for every metric + that happens to use that indom - i.e. many times per fetch. + + These changes resulted in a 5x speedup in pcp-ss and it now + completes instantly as expected. Additionally, we use alot + less memory now for the many-metrics-sharing-an-indom case. + +diff -Naurp pcp-5.3.7.patched/src/libpcp/src/fetchgroup.c pcp-5.3.7/src/libpcp/src/fetchgroup.c +--- pcp-5.3.7.patched/src/libpcp/src/fetchgroup.c 2022-04-05 09:05:43.000000000 +1000 ++++ pcp-5.3.7/src/libpcp/src/fetchgroup.c 2023-03-08 18:46:43.486916630 +1100 +@@ -34,6 +34,19 @@ + /* ------------------------------------------------------------------------ */ + + /* ++ * Cache to avoid unnecessary repeated calls to pmGetInDom for individual ++ * instance domains. This involves a round trip to pmcd and it does not ++ * change frequently (certainly not for processing results of one sample). ++ */ ++struct __pmInDomCache { ++ pmInDom indom; ++ int *codes; /* saved from pmGetInDom */ ++ char **names; ++ unsigned int size; ++ int refreshed; ++}; ++ ++/* + * An individual fetch-group is a linked list of requests tied to a + * particular pcp context (which the fetchgroup owns). NB: This is + * opaque to the PMAPI client. +@@ -43,8 +56,10 @@ struct __pmFetchGroup { + int wrap; /* wrap-handling flag, set at fg-create-time */ + pmResult *prevResult; + struct __pmFetchGroupItem *items; ++ struct __pmInDomCache *unique_indoms; + pmID *unique_pmids; + size_t num_unique_pmids; ++ size_t num_unique_indoms; + }; + + /* +@@ -81,9 +96,6 @@ struct __pmFetchGroupItem { + struct { + pmID metric_pmid; + pmDesc metric_desc; +- int *indom_codes; /* saved from pmGetInDom */ +- char **indom_names; +- unsigned indom_size; + struct __pmFetchGroupConversionSpec conv; + int *output_inst_codes; /* NB: may be NULL */ + char **output_inst_names; /* NB: may be NULL */ +@@ -168,9 +180,7 @@ pmfg_add_pmid(pmFG pmfg, pmID pmid) + /* + * Populate given pmFGI item structure based on common lookup & + * verification for pmfg inputs. Adjust instance profile to +- * include requested instance. If it's a derived metric, we +- * don't know what instance domain(s) it could involve, so we +- * clear the instance profile entirely. ++ * include requested instance. + */ + static int + pmfg_lookup_item(const char *metric, const char *instance, pmFGI item) +@@ -206,9 +216,12 @@ pmfg_lookup_item(const char *metric, con + + /* Same for a whole indom. Add the whole instance profile. */ + static int +-pmfg_lookup_indom(const char *metric, pmFGI item) ++pmfg_lookup_indom(pmFG pmfg, const char *metric, pmFGI item) + { +- int sts; ++ struct __pmInDomCache *indoms; ++ pmInDom indom; ++ size_t size; ++ int i, sts; + + assert(item != NULL); + assert(item->type == pmfg_indom); +@@ -221,14 +234,34 @@ pmfg_lookup_indom(const char *metric, pm + return sts; + + /* As a convenience to users, we also accept non-indom'd metrics */ +- if (item->u.indom.metric_desc.indom == PM_INDOM_NULL) ++ if ((indom = item->u.indom.metric_desc.indom) == PM_INDOM_NULL) + return 0; + + /* ++ * Insert into the instance domain cache if not seen previously ++ */ ++ for (i = 0; i < pmfg->num_unique_indoms; i++) { ++ if (pmfg->unique_indoms[i].indom == indom) ++ return 0; ++ } ++ ++ size = sizeof(struct __pmInDomCache) * (pmfg->num_unique_indoms + 1); ++ indoms = realloc(pmfg->unique_indoms, size); ++ ++ if (indoms == NULL) ++ return -ENOMEM; ++ pmfg->unique_indoms = indoms; ++ pmfg->unique_indoms[pmfg->num_unique_indoms].indom = indom; ++ pmfg->unique_indoms[pmfg->num_unique_indoms].codes = NULL; ++ pmfg->unique_indoms[pmfg->num_unique_indoms].names = NULL; ++ pmfg->unique_indoms[pmfg->num_unique_indoms].refreshed = 0; ++ pmfg->num_unique_indoms++; ++ ++ /* + * Add all instances; this will override any other past or future + * piecemeal instance requests from __pmExtendFetchGroup_lookup. + */ +- return pmAddProfile(item->u.indom.metric_desc.indom, 0, NULL); ++ return pmAddProfile(indom, 0, NULL); + } + + /* Same for an event field. */ +@@ -581,7 +614,7 @@ pmfg_reinit_indom(pmFGI item) + + if (item->u.indom.output_inst_names) + for (i = 0; i < item->u.indom.output_maxnum; i++) +- item->u.indom.output_inst_names[i] = NULL; /* break ref into indom_names[] */ ++ item->u.indom.output_inst_names[i] = NULL; /* ptr into names[] */ + + if (item->u.indom.output_stss) + for (i = 0; i < item->u.indom.output_maxnum; i++) +@@ -857,11 +890,11 @@ pmfg_fetch_timestamp(pmFG pmfg, pmFGI it + static void + pmfg_fetch_indom(pmFG pmfg, pmFGI item, pmResult *newResult) + { +- int sts = 0; +- int i; +- unsigned j; +- int need_indom_refresh; ++ int i, sts = 0; ++ unsigned int j, k; ++ struct __pmInDomCache *cache; + const pmValueSet *iv; ++ pmInDom indom; + + assert(item != NULL); + assert(item->type == pmfg_indom); +@@ -901,36 +934,43 @@ pmfg_fetch_indom(pmFG pmfg, pmFGI item, + + /* + * Analyze newResult to see whether it only contains instances we +- * already know. This is unfortunately an O(N**2) operation. It +- * could be made a bit faster if we build a pmGetInDom()- variant +- * that provides instances in sorted order. ++ * already know. + */ +- need_indom_refresh = 0; +- if (item->u.indom.output_inst_names) { /* Caller interested at all? */ +- for (j = 0; j < (unsigned)iv->numval; j++) { +- unsigned k; +- for (k = 0; k < item->u.indom.indom_size; k++) +- if (item->u.indom.indom_codes[k] == iv->vlist[j].inst) ++ cache = NULL; ++ indom = item->u.indom.metric_desc.indom; ++ for (j = 0; j < pmfg->num_unique_indoms; j++) { ++ if (indom != pmfg->unique_indoms[j].indom) ++ continue; ++ cache = &pmfg->unique_indoms[j]; ++ break; ++ } ++ if (cache && cache->refreshed && ++ item->u.indom.output_inst_names) { /* Caller interested at all? */ ++ for (j = 0; j < (unsigned int)iv->numval; j++) { ++ for (k = 0; k < cache->size; k++) ++ if (cache->codes[k] == iv->vlist[j].inst) + break; +- if (k >= item->u.indom.indom_size) { +- need_indom_refresh = 1; ++ if (k >= cache->size) { ++ cache->refreshed = 0; + break; + } + } + } +- if (need_indom_refresh) { +- free(item->u.indom.indom_codes); +- free(item->u.indom.indom_names); +- sts = pmGetInDom(item->u.indom.metric_desc.indom, +- &item->u.indom.indom_codes, &item->u.indom.indom_names); ++ if (cache && !cache->refreshed) { ++ cache->refreshed = 1; ++ free(cache->codes); ++ free(cache->names); ++ sts = pmGetInDom(indom, &cache->codes, &cache->names); + if (sts < 1) { + /* Need to manually clear; pmGetInDom claims they are undefined. */ +- item->u.indom.indom_codes = NULL; +- item->u.indom.indom_names = NULL; +- item->u.indom.indom_size = 0; ++ cache->codes = NULL; ++ cache->names = NULL; ++ cache->size = 0; + } + else { +- item->u.indom.indom_size = sts; ++ if (sts < 0) ++ cache->refreshed = 0; ++ cache->size = sts; + } + /* + * NB: Even if the pmGetInDom failed, we can proceed with +@@ -965,18 +1005,19 @@ pmfg_fetch_indom(pmFG pmfg, pmFGI item, + * results from pmGetIndom. + */ + if (item->u.indom.output_inst_names) { +- unsigned k; +- +- for (k = 0; k < item->u.indom.indom_size; k++) { +- if (item->u.indom.indom_codes[k] == jv->inst) { +- /* +- * NB: copy the indom name char* by value. +- * The user is not supposed to modify / free this pointer, +- * or use it after a subsequent fetch or delete operation. +- */ +- item->u.indom.output_inst_names[j] = +- item->u.indom.indom_names[k]; +- break; ++ if (cache == NULL) ++ item->u.indom.output_inst_names[j] = NULL; ++ else { ++ for (k = 0; k < cache->size; k++) { ++ if (cache->codes[k] == jv->inst) { ++ /* ++ * NB: copy the indom name char* by value. ++ * User may not modify / free this pointer, ++ * nor use it after subsequent fetch / delete. ++ */ ++ item->u.indom.output_inst_names[j] = cache->names[k]; ++ break; ++ } + } + } + } +@@ -1201,8 +1242,7 @@ out: + static int + pmfg_clear_profile(pmFG pmfg) + { +- int sts; +- pmFGI item; ++ int i, sts; + + sts = pmUseContext(pmfg->ctx); + if (sts != 0) +@@ -1216,10 +1256,9 @@ pmfg_clear_profile(pmFG pmfg) + * Any errors here are benign (or rather there's nothing we can do + * about 'em), so ignore pmDelProfile() return value. + */ +- for (item = pmfg->items; item; item = item->next) { +- if (item->u.item.metric_desc.indom != PM_INDOM_NULL) +- (void)pmDelProfile(item->u.item.metric_desc.indom, 0, NULL); +- } ++ for (i = 0; i < pmfg->num_unique_indoms; i++) ++ (void)pmDelProfile(pmfg->unique_indoms[i].indom, 0, NULL); ++ + return 0; + } + +@@ -1434,7 +1473,7 @@ pmExtendFetchGroup_indom(pmFG pmfg, + + item->type = pmfg_indom; + +- sts = pmfg_lookup_indom(metric, item); ++ sts = pmfg_lookup_indom(pmfg, metric, item); + if (sts != 0) + goto out; + +@@ -1615,7 +1654,6 @@ pmFetchGroup(pmFG pmfg) + + if (pmfg == NULL) + return -EINVAL; +- + /* + * Walk the fetchgroup, reinitializing every output spot, regardless of + * later errors. +@@ -1705,6 +1743,7 @@ int + pmClearFetchGroup(pmFG pmfg) + { + pmFGI item; ++ size_t n; + + if (pmfg == NULL) + return -EINVAL; +@@ -1719,8 +1758,6 @@ pmClearFetchGroup(pmFG pmfg) + break; + case pmfg_indom: + pmfg_reinit_indom(item); +- free(item->u.indom.indom_codes); +- free(item->u.indom.indom_names); + break; + case pmfg_event: + pmfg_reinit_event(item); +@@ -1739,11 +1776,21 @@ pmClearFetchGroup(pmFG pmfg) + if (pmfg->prevResult) + pmFreeResult(pmfg->prevResult); + pmfg->prevResult = NULL; ++ + if (pmfg->unique_pmids) + free(pmfg->unique_pmids); + pmfg->unique_pmids = NULL; + pmfg->num_unique_pmids = 0; + ++ for (n = 0; n < pmfg->num_unique_indoms; n++) { ++ free(pmfg->unique_indoms[n].codes); ++ free(pmfg->unique_indoms[n].names); ++ } ++ if (pmfg->unique_indoms) ++ free(pmfg->unique_indoms); ++ pmfg->unique_indoms = NULL; ++ pmfg->num_unique_indoms = 0; ++ + return pmfg_clear_profile(pmfg); + } + diff --git a/SOURCES/redhat-bugzilla-2139325-openmetrics-in-grafana.patch b/SOURCES/redhat-bugzilla-2139325-openmetrics-in-grafana.patch new file mode 100644 index 0000000..6f906dc --- /dev/null +++ b/SOURCES/redhat-bugzilla-2139325-openmetrics-in-grafana.patch @@ -0,0 +1,317 @@ +commit 1e216d84c6b95b4f9cb7ee6b5adf9591f8af37d5 +Author: Nathan Scott +Date: Wed Dec 7 11:40:43 2022 +1100 + + pmdaopenmetrics: validate given names before using them for metrics + + Its possible pmdaopenmetrics is fed garbage accidentally, e.g. in the + case where a /metrics end point is not made visible and an HTTP error + response is returned (misconfiguration). + + Failure to safeguard this results in the generation of diagnostics in + the openmetrics PMDA log file at an alarming rate until all available + filesystem space is completely consumed. It can also result in bogus + metric names being exposed to PMAPI clients. + + This commit both adds new safeguards and scales back diagnostic noise + from the agent. In some places it switches to more appropriate APIs + for the level of logging being performed (esp. debug messages). + + Resolves Red Hat BZ #2139325 + +diff -Naurp pcp-5.3.7.orig/qa/1191.out pcp-5.3.7/qa/1191.out +--- pcp-5.3.7.orig/qa/1191.out 2021-11-01 13:02:26.000000000 +1100 ++++ pcp-5.3.7/qa/1191.out 2023-03-09 11:49:42.372413973 +1100 +@@ -10884,13 +10884,6 @@ GC Wall Time for b5be5b9f_b0f1_47de_b436 + inst [0 or "0 collector_name:Copy"] value 24462465 + inst [1 or "1 collector_name:MSC"] value 49519 + +-openmetrics.vmware_exporter.[root@ci-vm-10-0-138-196 [vmware_host_net_usage_average] +- Data Type: double InDom: PM_INDOM_NULL 0xffffffff +- Semantics: instant Units: none +-Help: +-vmware_host_net_usage_average +-Error: Try again. Information not currently available +- + openmetrics.vmware_exporter.vmware_datastore_accessible [VMWare datastore accessible (true / false)] + Data Type: double InDom: 144.35859 0x24008c13 + Semantics: instant Units: none +diff -Naurp pcp-5.3.7.orig/qa/1221.out pcp-5.3.7/qa/1221.out +--- pcp-5.3.7.orig/qa/1221.out 2021-11-01 13:02:26.000000000 +1100 ++++ pcp-5.3.7/qa/1221.out 2023-03-09 11:49:42.382414004 +1100 +@@ -7105,9 +7105,6 @@ openmetrics.thermostat.tms_jvm_gc_b5be5b + inst [0 or "0 collector_name:Copy"] labels {"agent":"openmetrics","collector_name":"Copy","domainname":DOMAINNAME,"groupid":NUM,"hostname":HOSTNAME,"instname":"0 collector_name:Copy","machineid":MACHINEID,"source":"thermostat","url":FILEURL,"userid":NUM} + inst [1 or "1 collector_name:MSC"] labels {"agent":"openmetrics","collector_name":"MSC","domainname":DOMAINNAME,"groupid":NUM,"hostname":HOSTNAME,"instname":"1 collector_name:MSC","machineid":MACHINEID,"source":"thermostat","url":FILEURL,"userid":NUM} + +-openmetrics.vmware_exporter.[root@ci-vm-10-0-138-196 +- labels {"agent":"openmetrics","domainname":DOMAINNAME,"groupid":NUM,"hostname":HOSTNAME,"machineid":MACHINEID,"source":"vmware_exporter","url":FILEURL,"userid":NUM} +- + openmetrics.vmware_exporter.vmware_datastore_accessible + labels {"agent":"openmetrics","dc_name":"ha-datacenter","domainname":DOMAINNAME,"ds_cluster":"","ds_name":"name008","groupid":NUM,"hostname":HOSTNAME,"machineid":MACHINEID,"source":"vmware_exporter","url":FILEURL,"userid":NUM} + inst [0 or "0 dc_name:ha-datacenter ds_cluster: ds_name:name026"] labels {"agent":"openmetrics","dc_name":"ha-datacenter","domainname":DOMAINNAME,"ds_cluster":"","ds_name":"name026","groupid":NUM,"hostname":HOSTNAME,"instname":"0 dc_name:ha-datacenter ds_cluster: ds_name:name026","machineid":MACHINEID,"source":"vmware_exporter","url":FILEURL,"userid":NUM} +@@ -15331,9 +15328,6 @@ openmetrics.thermostat.tms_jvm_gc_b5be5b + inst [0 or "0 collector_name:Copy"] labels {"agent":"openmetrics","collector_name":"Copy","domainname":DOMAINNAME,"groupid":NUM,"hostname":HOSTNAME,"instname":"0 collector_name:Copy","machineid":MACHINEID,"source":"thermostat","url":FILEURL,"userid":NUM} + inst [1 or "1 collector_name:MSC"] labels {"agent":"openmetrics","collector_name":"MSC","domainname":DOMAINNAME,"groupid":NUM,"hostname":HOSTNAME,"instname":"1 collector_name:MSC","machineid":MACHINEID,"source":"thermostat","url":FILEURL,"userid":NUM} + +-openmetrics.vmware_exporter.[root@ci-vm-10-0-138-196 +- labels {"agent":"openmetrics","domainname":DOMAINNAME,"groupid":NUM,"hostname":HOSTNAME,"machineid":MACHINEID,"source":"vmware_exporter","url":FILEURL,"userid":NUM} +- + openmetrics.vmware_exporter.vmware_datastore_accessible + labels {"agent":"openmetrics","dc_name":"ha-datacenter","domainname":DOMAINNAME,"ds_cluster":"","ds_name":"name008","groupid":NUM,"hostname":HOSTNAME,"machineid":MACHINEID,"source":"vmware_exporter","url":FILEURL,"userid":NUM} + inst [0 or "0 dc_name:ha-datacenter ds_cluster: ds_name:name026"] labels {"agent":"openmetrics","dc_name":"ha-datacenter","domainname":DOMAINNAME,"ds_cluster":"","ds_name":"name026","groupid":NUM,"hostname":HOSTNAME,"instname":"0 dc_name:ha-datacenter ds_cluster: ds_name:name026","machineid":MACHINEID,"source":"vmware_exporter","url":FILEURL,"userid":NUM} +diff -Naurp pcp-5.3.7.orig/qa/1342 pcp-5.3.7/qa/1342 +--- pcp-5.3.7.orig/qa/1342 2021-02-17 15:27:41.000000000 +1100 ++++ pcp-5.3.7/qa/1342 2023-03-09 11:49:42.382414004 +1100 +@@ -99,6 +99,8 @@ echo == Note: check $seq.full for log en + echo == pmdaopenmetrics LOG == >>$here/$seq.full + cat $PCP_LOG_DIR/pmcd/openmetrics.log >>$here/$seq.full + ++# cleanup preparing for remove (else error messages here) ++$sudo rm $PCP_PMDAS_DIR/openmetrics/config.d/simple_metric.* + _pmdaopenmetrics_remove + + # success, all done +diff -Naurp pcp-5.3.7.orig/qa/1727 pcp-5.3.7/qa/1727 +--- pcp-5.3.7.orig/qa/1727 2021-09-23 14:19:12.000000000 +1000 ++++ pcp-5.3.7/qa/1727 2023-03-09 11:49:42.382414004 +1100 +@@ -2,6 +2,7 @@ + # PCP QA Test No. 1727 + # Test duplicate instname labels in /metrics webapi when a context + # level label such as "hostname" is explicitly specified. ++# Test invalid OpenMetrics metric names also. + # + # Copyright (c) 2021 Red Hat. All Rights Reserved. + # +@@ -54,8 +55,17 @@ echo '# TYPE somemetric gauge' + echo 'somemetric{hostname="$MYHOST"} 1234' + EOF + +-chmod 755 $tmp.script ++cat <$tmp.badxml ++#! /bin/bash ++ ++cat $here/sadist/891688-dash-time.xml ++echo '# TYPE somemetric gauge' ++echo 'somemetric{hostname="$MYHOST"} 1234' ++EOF ++ ++chmod 755 $tmp.script $tmp.badxml + $sudo mv $tmp.script $PCP_PMDAS_DIR/openmetrics/config.d/duplicate_instname_label.sh ++$sudo mv $tmp.badxml $PCP_PMDAS_DIR/openmetrics/config.d/invalid_metrics_badinput.sh + + _pmdaopenmetrics_install + if ! _pmdaopenmetrics_wait_for_metric openmetrics.control.calls +@@ -68,6 +78,14 @@ echo; echo === /metrics webapi listing. + curl -Gs 'http://localhost:44322/metrics?names=openmetrics.duplicate_instname_label.somemetric' \ + | _filter_openmetrics_labels + ++echo; echo === verify metric name validity using pminfo ++pminfo -v openmetrics ++ ++# squash errors for a clean uninstall ++$sudo rm $PCP_PMDAS_DIR/openmetrics/config.d/invalid_metrics_badinput.sh ++# capture openmetrics log for posterity ++cat $PCP_LOG_DIR/pmcd/openmetrics.log >> $here/$seq.full ++ + _pmdaopenmetrics_remove + + # success, all done +diff -Naurp pcp-5.3.7.orig/qa/1727.out pcp-5.3.7/qa/1727.out +--- pcp-5.3.7.orig/qa/1727.out 2021-09-13 14:40:08.000000000 +1000 ++++ pcp-5.3.7/qa/1727.out 2023-03-09 11:49:42.382414004 +1100 +@@ -8,6 +8,8 @@ QA output created by 1727 + # TYPE openmetrics_duplicate_instname_label_somemetric gauge + openmetrics_duplicate_instname_label_somemetric{hostname=HOSTNAME,instid="0",instname="0 hostname:HOSTNAME",domainname=DOMAINNAME,machineid=MACHINEID,source="duplicate_instname_label"} 1234 + ++=== verify metric name validity using pminfo ++ + === remove openmetrics agent === + Culling the Performance Metrics Name Space ... + openmetrics ... done +diff -Naurp pcp-5.3.7.orig/qa/openmetrics/samples/vmware_exporter.txt pcp-5.3.7/qa/openmetrics/samples/vmware_exporter.txt +--- pcp-5.3.7.orig/qa/openmetrics/samples/vmware_exporter.txt 2021-11-01 13:02:26.000000000 +1100 ++++ pcp-5.3.7/qa/openmetrics/samples/vmware_exporter.txt 2023-03-09 11:49:42.382414004 +1100 +@@ -1035,25 +1035,6 @@ vmware_host_net_errorsTx_summation{clust + # HELP vmware_host_net_usage_average vmware_host_net_usage_average + # TYPE vmware_host_net_usage_average gauge + vmware_host_net_usage_average{cluster_name="",dc_name="ha-datacenter",host_name="SOMEHOSTNAME"} 3.0 +-[root@ci-vm-10-0-138-196 ~]# curl -Gs http://localhost:9272/metrics > curl.txt +-[root@ci-vm-10-0-138-196 ~]# +-[root@ci-vm-10-0-138-196 ~]# +-[root@ci-vm-10-0-138-196 ~]# +-[root@ci-vm-10-0-138-196 ~]# +-[root@ci-vm-10-0-138-196 ~]# +-[root@ci-vm-10-0-138-196 ~]# +-[root@ci-vm-10-0-138-196 ~]# +-[root@ci-vm-10-0-138-196 ~]# +-[root@ci-vm-10-0-138-196 ~]# +-[root@ci-vm-10-0-138-196 ~]# +-[root@ci-vm-10-0-138-196 ~]# +-[root@ci-vm-10-0-138-196 ~]# +-[root@ci-vm-10-0-138-196 ~]# +-[root@ci-vm-10-0-138-196 ~]# +-[root@ci-vm-10-0-138-196 ~]# +-[root@ci-vm-10-0-138-196 ~]# +-[root@ci-vm-10-0-138-196 ~]# +-[root@ci-vm-10-0-138-196 ~]# cat curl.txt + # HELP vmware_vm_power_state VMWare VM Power state (On / Off) + # TYPE vmware_vm_power_state gauge + vmware_vm_power_state{cluster_name="",dc_name="ha-datacenter",ds_name="name026",host_name="SOMEHOSTNAME",vm_name="name023-sat61-client"} 0.0 +diff -Naurp pcp-5.3.7.orig/src/pmdas/openmetrics/pmdaopenmetrics.python pcp-5.3.7/src/pmdas/openmetrics/pmdaopenmetrics.python +--- pcp-5.3.7.orig/src/pmdas/openmetrics/pmdaopenmetrics.python 2022-04-05 09:05:43.000000000 +1000 ++++ pcp-5.3.7/src/pmdas/openmetrics/pmdaopenmetrics.python 2023-03-09 11:49:42.382414004 +1100 +@@ -122,7 +122,7 @@ class Metric(object): + # c_units = c_api.pmUnits_int(self.munits.dimSpace, self.munits.dimTime, self.munits.dimCount, + # self.munits.scaleSpace, self.munits.scaleTime, self.munits.scaleCount) + +- self.source.pmda.log("Metric: adding new metric %s pmid=%s type=%d sem=%d singular=%s mindom=0x%x labels=%s" % ++ self.source.pmda.debug("Metric: adding metric %s pmid=%s type=%d sem=%d singular=%s mindom=0x%x labels=%s" % + (name, pmContext.pmIDStr(self.pmid), self.mtype, self.msem, self.singular, self.mindom, self.labels)) + + self.obj = pmdaMetric(self.pmid, self.mtype, self.mindom, self.msem, self.munits) +@@ -565,6 +565,7 @@ class Source(object): + self.cluster = cluster # unique/persistent id# for nickname + self.path = path # pathname to .url or executable file + self.url = None ++ self.parse_error = False + self.parse_url_time = 0 # timestamp of config file when it was last parsed + self.is_scripted = is_scripted + self.pmda = thispmda # the shared pmda +@@ -659,6 +660,19 @@ class Source(object): + self.pmda.log("instname_labels returning %s" % naming_labels) if self.pmda.dbg else None + return naming_labels + ++ def valid_metric_name(self, name): ++ ''' ++ Check validity of given metric name component: ++ - only contains alphanumerics and/or underscores ++ (colon removed later, allowed through here); ++ - only starts with an alphabetic character. ++ ''' ++ if not name[0].isalpha(): ++ return False ++ if not re.sub('[_.:]', '', name).isalnum(): ++ return False ++ return True ++ + def parse_metric_line(self, line, pcpline, helpline, typeline): + ''' + Parse the sample line, identify/create corresponding metric & instance. +@@ -687,8 +701,10 @@ class Source(object): + # Nb: filter pattern is applied only to the leaf component of the full metric name + if self.check_filter(sp.name, "METRIC") != "INCLUDE": + self.pmda.log("Metric %s excluded by config filters" % fullname) +- return ++ return True + else: ++ if not self.valid_metric_name(sp.name): ++ raise ValueError('invalid metric name: ' + sp.name) + # new metric + metricnum = self.pmids_table.intern_lookup_value(sp.name) + # check if the config specifies metadata for this metric +@@ -708,24 +724,32 @@ class Source(object): + else: + m.store_inst(naming_labels, sp.value) + self.pmda.set_notify_change() ++ except ValueError as e: ++ if not self.parse_error: ++ self.pmda.err("cannot parse name in %s: %s" % (line, e)) ++ self.parse_error = True # one-time-only error diagnostic ++ return False + except Exception as e: ++ if self.pmda.dbg: ++ traceback.print_exc() # traceback can be handy here + self.pmda.err("cannot parse/store %s: %s" % (line, e)) +- traceback.print_exc() # traceback can be handy here +- ++ return False ++ return True + + def parse_lines(self, text): +- '''Refresh all the metric metadata as it is found, including creating +- new metrics. Store away metric values for subsequent +- fetch()es. Parse errors may result in exceptions. +- That's OK, we don't try heroics to parse non-compliant +- data. Return number of metrics extracted. + ''' +- ++ Refresh all the metric metadata as it is found, including creating ++ new metrics. Store away metric values for subsequent fetch()es. ++ Input parse errors result in exceptions and early termination. ++ That's OK, we don't try heroics to parse non-compliant data. ++ Return number of metrics extracted. ++ ''' + num_metrics = 0 + lines = text.splitlines() + pcpline = None + helpline = None + typeline = None ++ badness = False + state = "metadata" + for line in lines: + self.pmda.debug("line: %s state: %s" % (line, state)) if self.pmda.dbg else None +@@ -762,9 +786,15 @@ class Source(object): + # NB: could verify helpline/typeline lp[2] matches, + # but we don't have to go out of our way to support + # non-compliant exporters. +- self.parse_metric_line(l, pcpline, helpline, typeline) ++ if not self.parse_metric_line(l, pcpline, helpline, typeline): ++ badness = True ++ break # bad metric line, skip the remainder of this file + num_metrics += 1 + ++ # clear one-time-only error diagnostic if the situation is now resolved ++ if not badness: ++ self.parse_error = False ++ + # NB: this logic only ever -adds- Metrics to a Source. If a source + # stops supplying some metrics, then a PCP app will see a PM_ERR_INST + # coming back when it tries to fetch them. We could perhaps keep the +@@ -842,7 +872,7 @@ class Source(object): + metadata = line.split(' ')[1:] + self.metadatalist.append(metadata) + else: +- self.pmda.log('Warning: %s ignored unrecognised config entry "%s"' % (self.url, line)) ++ self.pmda.err('%s ignored unrecognised config entry "%s"' % (self.url, line)) + + self.pmda.debug("DEBUG url: %s HEADERS: %s" % (self.url, self.headers)) if self.pmda.dbg else None + self.pmda.debug("DEBUG url: %s FILTERS: %s" % (self.url, self.filterlist)) if self.pmda.dbg else None +@@ -911,8 +941,7 @@ class Source(object): + except Exception as e: + self.pmda.stats_status[self.cluster] = 'failed to fetch URL or execute script %s: %s' % (self.path, e) + self.pmda.stats_status_code[self.cluster] = status_code +- self.pmda.debug('Warning: cannot fetch URL or execute script %s: %s' % (self.path, e)) if self.pmda.dbg else None +- return ++ self.pmda.debug('cannot fetch URL or execute script %s: %s' % (self.path, e)) + + def refresh2(self, timeout): + ''' +@@ -950,7 +979,7 @@ class Source(object): + self.pmda.log("fetch: item=%d inst=%d m.mname=%s" % (item, inst, m.mname)) if self.pmda.dbg else None + return m.fetch_inst(inst) + except Exception as e: +- self.pmda.log("Warning: cannot fetch item %d inst %d: %s" % (item, inst, e)) ++ self.pmda.debug("cannot fetch item %d inst %d: %s" % (item, inst, e)) + return [c_api.PM_ERR_AGAIN, 0] + + class OpenMetricsPMDA(PMDA): +@@ -1468,7 +1497,7 @@ class OpenMetricsPMDA(PMDA): + + def debug(self, s): + if self.dbg: +- super(OpenMetricsPMDA, self).log("debug: " + s) ++ super(OpenMetricsPMDA, self).dbg(s) + + def log(self, message): + PMDA.log(message) +@@ -1526,8 +1555,9 @@ if __name__ == '__main__': + if args.nosort: + sort_conf_list = False + +- pmdadir = os.path.join(os.getenv('PCP_PMDAS_DIR'), args.root) + if not args.config.startswith("/"): ++ pmdadir = os.getenv('PCP_PMDAS_DIR') or '/' ++ pmdadir = os.path.join(pmdadir, args.root) + args.config = os.path.join(pmdadir, args.config) + + # This PMDA starts up in the "notready" state, see the Install script where diff --git a/SOURCES/redhat-bugzilla-2150889-nfsclient-srcport.patch b/SOURCES/redhat-bugzilla-2150889-nfsclient-srcport.patch new file mode 100644 index 0000000..0b14ccb --- /dev/null +++ b/SOURCES/redhat-bugzilla-2150889-nfsclient-srcport.patch @@ -0,0 +1,706 @@ +commit f4e34cd32bc73b53c2fc94c8deb670044d9d18d6 +Author: Nathan Scott +Date: Wed Dec 7 12:17:19 2022 +1100 + + pmdanfsclient: fix srcport handling for rdma and udp mounts + + This bug fix kindly brought to you by the good folk at Penn State Uni. + Regression test update by myself but the real fix came from others. + + Resolves Red Hat BZ #2150889 + +diff --git a/qa/798.out.32 b/qa/798.out.32 +index 8db1b9896..854d667a2 100644 +--- a/qa/798.out.32 ++++ b/qa/798.out.32 +@@ -165,6 +165,110 @@ pmInDom: 62.0 + dbpmda> + Log for pmdanfsclient on HOST ... + Log finished ... ++=== Test case: mountstats-el8.7-rdma.qa ++dbpmda> open pipe $PYTHON pmdanfsclient.python ++Start $PYTHON PMDA: $PYTHON pmdanfsclient.python ++dbpmda> # on some platforms this may take a while ... ++dbpmda> wait 2 ++dbpmda> getdesc on ++dbpmda> desc nfsclient.export ++PMID: 62.0.1 ++ Data Type: string InDom: 62.0 0xf800000 ++ Semantics: instant Units: none ++dbpmda> desc nfsclient.mountpoint ++PMID: 62.0.2 ++ Data Type: string InDom: 62.0 0xf800000 ++ Semantics: instant Units: none ++dbpmda> desc nfsclient.options.string ++PMID: 62.1.1 ++ Data Type: string InDom: 62.0 0xf800000 ++ Semantics: instant Units: none ++dbpmda> desc nfsclient.options.proto ++PMID: 62.1.24 ++ Data Type: string InDom: 62.0 0xf800000 ++ Semantics: instant Units: none ++dbpmda> desc nfsclient.options.vers ++PMID: 62.1.6 ++ Data Type: string InDom: 62.0 0xf800000 ++ Semantics: instant Units: none ++dbpmda> fetch nfsclient.export ++PMID(s): 62.0.1 ++__pmResult ... numpmid: 1 ++ 62.0.1 (): numval: 7 valfmt: 1 vlist[]: ++ inst [N or ???] value "example.com:/group" ++ inst [N or ???] value "example.com:/home" ++ inst [N or ???] value "example.com:/icds" ++ inst [N or ???] value "example.com:/icds-pcp-data" ++ inst [N or ???] value "example.com:/icds-prod" ++ inst [N or ???] value "example.com:/icds-test" ++ inst [N or ???] value "example.com:/work" ++dbpmda> fetch nfsclient.mountpoint ++PMID(s): 62.0.2 ++__pmResult ... numpmid: 1 ++ 62.0.2 (): numval: 7 valfmt: 1 vlist[]: ++ inst [N or ???] value "/storage/group" ++ inst [N or ???] value "/storage/home" ++ inst [N or ???] value "/storage/icds" ++ inst [N or ???] value "/storage/pcp-data" ++ inst [N or ???] value "/storage/prod" ++ inst [N or ???] value "/storage/test" ++ inst [N or ???] value "/storage/work" ++dbpmda> fetch nfsclient.options.string ++PMID(s): 62.1.1 ++__pmResult ... numpmid: 1 ++ 62.1.1 (): numval: 7 valfmt: 1 vlist[]: ++ inst [N or ???] value "rw,vers=3,rsize=65536,wsize=65536,namlen=255,acregmin=3,acregmax=60,acdirmin=30,acdirmax=60,hard,proto=rdma,nconnect=8,port=20049,timeo=600,retrans=2,sec=sys,mountaddr=10.6.159.1,mountvers=3,mountport=0,mountproto=tcp,local_lock=all" ++ inst [N or ???] value "rw,vers=3,rsize=65536,wsize=65536,namlen=255,acregmin=3,acregmax=60,acdirmin=30,acdirmax=60,hard,proto=rdma,nconnect=8,port=20049,timeo=600,retrans=2,sec=sys,mountaddr=10.6.159.18,mountvers=3,mountport=0,mountproto=tcp,local_lock=all" ++ inst [N or ???] value "rw,vers=3,rsize=65536,wsize=65536,namlen=255,acregmin=3,acregmax=60,acdirmin=30,acdirmax=60,hard,proto=rdma,nconnect=8,port=20049,timeo=600,retrans=2,sec=sys,mountaddr=10.6.159.23,mountvers=3,mountport=0,mountproto=tcp,local_lock=all" ++ inst [N or ???] value "rw,vers=3,rsize=65536,wsize=65536,namlen=255,acregmin=3,acregmax=60,acdirmin=30,acdirmax=60,hard,proto=rdma,nconnect=8,port=20049,timeo=600,retrans=2,sec=sys,mountaddr=10.6.159.27,mountvers=3,mountport=0,mountproto=tcp,local_lock=all" ++ inst [N or ???] value "rw,vers=3,rsize=65536,wsize=65536,namlen=255,acregmin=3,acregmax=60,acdirmin=30,acdirmax=60,hard,proto=rdma,nconnect=8,port=20049,timeo=600,retrans=2,sec=sys,mountaddr=10.6.159.29,mountvers=3,mountport=0,mountproto=tcp,local_lock=all" ++ inst [N or ???] value "rw,vers=3,rsize=65536,wsize=65536,namlen=255,acregmin=3,acregmax=60,acdirmin=30,acdirmax=60,hard,proto=rdma,nconnect=8,port=20049,timeo=600,retrans=2,sec=sys,mountaddr=10.6.159.6,mountvers=3,mountport=0,mountproto=tcp,local_lock=all" ++ inst [N or ???] value "rw,vers=3,rsize=65536,wsize=65536,namlen=255,acregmin=3,acregmax=60,acdirmin=30,acdirmax=60,hard,proto=rdma,nconnect=8,port=20049,timeo=600,retrans=2,sec=sys,mountaddr=10.6.159.7,mountvers=3,mountport=0,mountproto=tcp,local_lock=all" ++dbpmda> fetch nfsclient.options.proto ++PMID(s): 62.1.24 ++__pmResult ... numpmid: 1 ++ 62.1.24 (): numval: 7 valfmt: 1 vlist[]: ++ inst [N or ???] value "rdma" ++ inst [N or ???] value "rdma" ++ inst [N or ???] value "rdma" ++ inst [N or ???] value "rdma" ++ inst [N or ???] value "rdma" ++ inst [N or ???] value "rdma" ++ inst [N or ???] value "rdma" ++dbpmda> fetch nfsclient.options.vers ++PMID(s): 62.1.6 ++__pmResult ... numpmid: 1 ++ 62.1.6 (): numval: 7 valfmt: 1 vlist[]: ++ inst [N or ???] value "3" ++ inst [N or ???] value "3" ++ inst [N or ???] value "3" ++ inst [N or ???] value "3" ++ inst [N or ???] value "3" ++ inst [N or ???] value "3" ++ inst [N or ???] value "3" ++dbpmda> fetch nfsclient.options.vers ++PMID(s): 62.1.6 ++__pmResult ... numpmid: 1 ++ 62.1.6 (): numval: 7 valfmt: 1 vlist[]: ++ inst [N or ???] value "3" ++ inst [N or ???] value "3" ++ inst [N or ???] value "3" ++ inst [N or ???] value "3" ++ inst [N or ???] value "3" ++ inst [N or ???] value "3" ++ inst [N or ???] value "3" ++dbpmda> instance 62.0 ++pmInDom: 62.0 ++[ X] inst: X name: "/storage/group" ++[X+1] inst: X+1 name: "/storage/home" ++[X+2] inst: X+2 name: "/storage/icds" ++[X+3] inst: X+3 name: "/storage/pcp-data" ++[X+4] inst: X+4 name: "/storage/prod" ++[X+5] inst: X+5 name: "/storage/test" ++[X+6] inst: X+6 name: "/storage/work" ++dbpmda> ++Log for pmdanfsclient on HOST ... ++Log finished ... + === Test case: mountstats.qa + dbpmda> open pipe $PYTHON pmdanfsclient.python + Start $PYTHON PMDA: $PYTHON pmdanfsclient.python +diff --git a/qa/798.out.64 b/qa/798.out.64 +index 6b6a4b73c..860fa582e 100644 +--- a/qa/798.out.64 ++++ b/qa/798.out.64 +@@ -165,6 +165,110 @@ pmInDom: 62.0 + dbpmda> + Log for pmdanfsclient on HOST ... + Log finished ... ++=== Test case: mountstats-el8.7-rdma.qa ++dbpmda> open pipe $PYTHON pmdanfsclient.python ++Start $PYTHON PMDA: $PYTHON pmdanfsclient.python ++dbpmda> # on some platforms this may take a while ... ++dbpmda> wait 2 ++dbpmda> getdesc on ++dbpmda> desc nfsclient.export ++PMID: 62.0.1 ++ Data Type: string InDom: 62.0 0xf800000 ++ Semantics: instant Units: none ++dbpmda> desc nfsclient.mountpoint ++PMID: 62.0.2 ++ Data Type: string InDom: 62.0 0xf800000 ++ Semantics: instant Units: none ++dbpmda> desc nfsclient.options.string ++PMID: 62.1.1 ++ Data Type: string InDom: 62.0 0xf800000 ++ Semantics: instant Units: none ++dbpmda> desc nfsclient.options.proto ++PMID: 62.1.24 ++ Data Type: string InDom: 62.0 0xf800000 ++ Semantics: instant Units: none ++dbpmda> desc nfsclient.options.vers ++PMID: 62.1.6 ++ Data Type: string InDom: 62.0 0xf800000 ++ Semantics: instant Units: none ++dbpmda> fetch nfsclient.export ++PMID(s): 62.0.1 ++__pmResult ... numpmid: 1 ++ 62.0.1 (): numval: 7 valfmt: 1 vlist[]: ++ inst [N or ???] value "example.com:/group" ++ inst [N or ???] value "example.com:/home" ++ inst [N or ???] value "example.com:/icds" ++ inst [N or ???] value "example.com:/icds-pcp-data" ++ inst [N or ???] value "example.com:/icds-prod" ++ inst [N or ???] value "example.com:/icds-test" ++ inst [N or ???] value "example.com:/work" ++dbpmda> fetch nfsclient.mountpoint ++PMID(s): 62.0.2 ++__pmResult ... numpmid: 1 ++ 62.0.2 (): numval: 7 valfmt: 1 vlist[]: ++ inst [N or ???] value "/storage/group" ++ inst [N or ???] value "/storage/home" ++ inst [N or ???] value "/storage/icds" ++ inst [N or ???] value "/storage/pcp-data" ++ inst [N or ???] value "/storage/prod" ++ inst [N or ???] value "/storage/test" ++ inst [N or ???] value "/storage/work" ++dbpmda> fetch nfsclient.options.string ++PMID(s): 62.1.1 ++__pmResult ... numpmid: 1 ++ 62.1.1 (): numval: 7 valfmt: 1 vlist[]: ++ inst [N or ???] value "rw,vers=3,rsize=65536,wsize=65536,namlen=255,acregmin=3,acregmax=60,acdirmin=30,acdirmax=60,hard,proto=rdma,nconnect=8,port=20049,timeo=600,retrans=2,sec=sys,mountaddr=10.6.159.1,mountvers=3,mountport=0,mountproto=tcp,local_lock=all" ++ inst [N or ???] value "rw,vers=3,rsize=65536,wsize=65536,namlen=255,acregmin=3,acregmax=60,acdirmin=30,acdirmax=60,hard,proto=rdma,nconnect=8,port=20049,timeo=600,retrans=2,sec=sys,mountaddr=10.6.159.18,mountvers=3,mountport=0,mountproto=tcp,local_lock=all" ++ inst [N or ???] value "rw,vers=3,rsize=65536,wsize=65536,namlen=255,acregmin=3,acregmax=60,acdirmin=30,acdirmax=60,hard,proto=rdma,nconnect=8,port=20049,timeo=600,retrans=2,sec=sys,mountaddr=10.6.159.23,mountvers=3,mountport=0,mountproto=tcp,local_lock=all" ++ inst [N or ???] value "rw,vers=3,rsize=65536,wsize=65536,namlen=255,acregmin=3,acregmax=60,acdirmin=30,acdirmax=60,hard,proto=rdma,nconnect=8,port=20049,timeo=600,retrans=2,sec=sys,mountaddr=10.6.159.27,mountvers=3,mountport=0,mountproto=tcp,local_lock=all" ++ inst [N or ???] value "rw,vers=3,rsize=65536,wsize=65536,namlen=255,acregmin=3,acregmax=60,acdirmin=30,acdirmax=60,hard,proto=rdma,nconnect=8,port=20049,timeo=600,retrans=2,sec=sys,mountaddr=10.6.159.29,mountvers=3,mountport=0,mountproto=tcp,local_lock=all" ++ inst [N or ???] value "rw,vers=3,rsize=65536,wsize=65536,namlen=255,acregmin=3,acregmax=60,acdirmin=30,acdirmax=60,hard,proto=rdma,nconnect=8,port=20049,timeo=600,retrans=2,sec=sys,mountaddr=10.6.159.6,mountvers=3,mountport=0,mountproto=tcp,local_lock=all" ++ inst [N or ???] value "rw,vers=3,rsize=65536,wsize=65536,namlen=255,acregmin=3,acregmax=60,acdirmin=30,acdirmax=60,hard,proto=rdma,nconnect=8,port=20049,timeo=600,retrans=2,sec=sys,mountaddr=10.6.159.7,mountvers=3,mountport=0,mountproto=tcp,local_lock=all" ++dbpmda> fetch nfsclient.options.proto ++PMID(s): 62.1.24 ++__pmResult ... numpmid: 1 ++ 62.1.24 (): numval: 7 valfmt: 1 vlist[]: ++ inst [N or ???] value "rdma" ++ inst [N or ???] value "rdma" ++ inst [N or ???] value "rdma" ++ inst [N or ???] value "rdma" ++ inst [N or ???] value "rdma" ++ inst [N or ???] value "rdma" ++ inst [N or ???] value "rdma" ++dbpmda> fetch nfsclient.options.vers ++PMID(s): 62.1.6 ++__pmResult ... numpmid: 1 ++ 62.1.6 (): numval: 7 valfmt: 1 vlist[]: ++ inst [N or ???] value "3" ++ inst [N or ???] value "3" ++ inst [N or ???] value "3" ++ inst [N or ???] value "3" ++ inst [N or ???] value "3" ++ inst [N or ???] value "3" ++ inst [N or ???] value "3" ++dbpmda> fetch nfsclient.options.vers ++PMID(s): 62.1.6 ++__pmResult ... numpmid: 1 ++ 62.1.6 (): numval: 7 valfmt: 1 vlist[]: ++ inst [N or ???] value "3" ++ inst [N or ???] value "3" ++ inst [N or ???] value "3" ++ inst [N or ???] value "3" ++ inst [N or ???] value "3" ++ inst [N or ???] value "3" ++ inst [N or ???] value "3" ++dbpmda> instance 62.0 ++pmInDom: 62.0 ++[ X] inst: X name: "/storage/group" ++[X+1] inst: X+1 name: "/storage/home" ++[X+2] inst: X+2 name: "/storage/icds" ++[X+3] inst: X+3 name: "/storage/pcp-data" ++[X+4] inst: X+4 name: "/storage/prod" ++[X+5] inst: X+5 name: "/storage/test" ++[X+6] inst: X+6 name: "/storage/work" ++dbpmda> ++Log for pmdanfsclient on HOST ... ++Log finished ... + === Test case: mountstats.qa + dbpmda> open pipe $PYTHON pmdanfsclient.python + Start $PYTHON PMDA: $PYTHON pmdanfsclient.python +diff --git a/qa/nfsclient/mountstats-el8.7-rdma.qa b/qa/nfsclient/mountstats-el8.7-rdma.qa +new file mode 100644 +index 000000000..98dd836a1 +--- /dev/null ++++ b/qa/nfsclient/mountstats-el8.7-rdma.qa +@@ -0,0 +1,437 @@ ++device proc mounted on /proc with fstype proc ++device devtmpfs mounted on /dev with fstype devtmpfs ++device securityfs mounted on /sys/kernel/security with fstype securityfs ++device tmpfs mounted on /dev/shm with fstype tmpfs ++device devpts mounted on /dev/pts with fstype devpts ++device tmpfs mounted on /run with fstype tmpfs ++device tmpfs mounted on /sys/fs/cgroup with fstype tmpfs ++device cgroup mounted on /sys/fs/cgroup/systemd with fstype cgroup ++device pstore mounted on /sys/fs/pstore with fstype pstore ++device efivarfs mounted on /sys/firmware/efi/efivars with fstype efivarfs ++device bpf mounted on /sys/fs/bpf with fstype bpf ++device cgroup mounted on /sys/fs/cgroup/hugetlb with fstype cgroup ++device cgroup mounted on /sys/fs/cgroup/cpu,cpuacct with fstype cgroup ++device cgroup mounted on /sys/fs/cgroup/memory with fstype cgroup ++device cgroup mounted on /sys/fs/cgroup/freezer with fstype cgroup ++device cgroup mounted on /sys/fs/cgroup/devices with fstype cgroup ++device cgroup mounted on /sys/fs/cgroup/net_cls,net_prio with fstype cgroup ++device cgroup mounted on /sys/fs/cgroup/perf_event with fstype cgroup ++device cgroup mounted on /sys/fs/cgroup/rdma with fstype cgroup ++device cgroup mounted on /sys/fs/cgroup/cpuset with fstype cgroup ++device cgroup mounted on /sys/fs/cgroup/pids with fstype cgroup ++device cgroup mounted on /sys/fs/cgroup/blkio with fstype cgroup ++device configfs mounted on /sys/kernel/config with fstype configfs ++device rootfs mounted on / with fstype tmpfs ++device rw mounted on /.sllocal/log with fstype tmpfs ++device rpc_pipefs mounted on /var/lib/nfs/rpc_pipefs with fstype rpc_pipefs ++device hugetlbfs mounted on /dev/hugepages with fstype hugetlbfs ++device debugfs mounted on /sys/kernel/debug with fstype debugfs ++device mqueue mounted on /dev/mqueue with fstype mqueue ++device systemd-1 mounted on /proc/sys/fs/binfmt_misc with fstype autofs ++device binfmt_misc mounted on /proc/sys/fs/binfmt_misc with fstype binfmt_misc ++device fusectl mounted on /sys/fs/fuse/connections with fstype fusectl ++device /dev/sda3 mounted on /tmp with fstype xfs ++device /dev/sda1 mounted on /var/log with fstype xfs ++device /dev/sda2 mounted on /var/tmp with fstype xfs ++device tracefs mounted on /sys/kernel/debug/tracing with fstype tracefs ++device example.com:/icds-prod mounted on /storage/prod with fstype nfs statvers=1.1 ++ opts: rw,vers=3,rsize=65536,wsize=65536,namlen=255,acregmin=3,acregmax=60,acdirmin=30,acdirmax=60,hard,proto=rdma,nconnect=8,port=20049,timeo=600,retrans=2,sec=sys,mountaddr=10.6.159.30,mountvers=3,mountport=0,mountproto=tcp,local_lock=all ++ age: 1166720 ++ caps: caps=0x10003fcf,wtmult=4096,dtsize=65536,bsize=0,namlen=255 ++ sec: flavor=1,pseudoflavor=1 ++ events: 3 36 0 0 3 8 44 0 0 3 0 0 0 0 3 0 0 3 0 0 0 0 0 0 0 0 0 ++ bytes: 5086 0 0 0 5086 0 4 0 ++ RPC iostats version: 1.1 p/v: 100003/3 (nfs) ++ xprt: rdma 0 0 1 0 0 5 5 0 5 0 0 0 0 0 0 0 0 0 0 0 0 0 0 11 0 0 0 ++ xprt: rdma 0 0 1 0 0 3 3 0 3 0 0 0 0 0 0 0 0 0 0 0 0 0 0 11 0 0 0 ++ xprt: rdma 0 0 1 0 0 3 3 0 3 0 0 0 0 0 0 0 268 0 0 0 0 0 0 11 0 0 0 ++ xprt: rdma 0 0 1 0 0 3 3 0 3 0 0 0 0 0 0 0 0 0 0 0 0 0 0 11 0 0 0 ++ xprt: rdma 0 0 1 0 0 3 3 0 3 0 0 0 1 4220 0 0 28 0 0 0 0 0 0 11 1 0 0 ++ xprt: rdma 0 0 1 0 0 3 3 0 3 0 0 0 0 0 0 0 0 0 0 0 0 0 0 11 0 0 0 ++ xprt: rdma 0 0 1 0 0 3 3 0 3 0 0 1 0 4746 4746 0 72 0 0 0 0 0 0 11 1 0 0 ++ xprt: rdma 0 0 1 0 0 3 3 0 3 0 0 0 0 0 0 0 0 0 0 0 0 0 0 11 0 0 0 ++ per-op statistics ++ NULL: 1 1 0 40 24 18 0 19 0 ++ GETATTR: 3 3 0 276 336 0 0 0 0 ++ SETATTR: 0 0 0 0 0 0 0 0 0 ++ LOOKUP: 8 8 0 852 1600 16 1 17 1 ++ ACCESS: 7 7 0 672 840 10 1 11 0 ++ READLINK: 1 1 0 92 148 5 0 5 0 ++ READ: 3 3 0 312 5472 0 1 1 0 ++ WRITE: 0 0 0 0 0 0 0 0 0 ++ CREATE: 0 0 0 0 0 0 0 0 0 ++ MKDIR: 0 0 0 0 0 0 0 0 0 ++ SYMLINK: 0 0 0 0 0 0 0 0 0 ++ MKNOD: 0 0 0 0 0 0 0 0 0 ++ REMOVE: 0 0 0 0 0 0 0 0 0 ++ RMDIR: 0 0 0 0 0 0 0 0 0 ++ RENAME: 0 0 0 0 0 0 0 0 0 ++ LINK: 0 0 0 0 0 0 0 0 0 ++ READDIR: 0 0 0 0 0 0 0 0 0 ++ READDIRPLUS: 0 0 0 0 0 0 0 0 0 ++ FSSTAT: 0 0 0 0 0 0 0 0 0 ++ FSINFO: 2 2 0 184 328 6 0 6 0 ++ PATHCONF: 0 0 0 0 0 0 0 0 0 ++ COMMIT: 0 0 0 0 0 0 0 0 0 ++ ++device example.com:/icds-test mounted on /storage/test with fstype nfs statvers=1.1 ++ opts: rw,vers=3,rsize=65536,wsize=65536,namlen=255,acregmin=3,acregmax=60,acdirmin=30,acdirmax=60,hard,proto=rdma,nconnect=8,port=20049,timeo=600,retrans=2,sec=sys,mountaddr=10.6.159.24,mountvers=3,mountport=0,mountproto=tcp,local_lock=all ++ age: 1166720 ++ caps: caps=0x10003fcf,wtmult=4096,dtsize=65536,bsize=0,namlen=255 ++ sec: flavor=1,pseudoflavor=1 ++ events: 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 ++ bytes: 0 0 0 0 0 0 0 0 ++ RPC iostats version: 1.1 p/v: 100003/3 (nfs) ++ xprt: rdma 0 0 1 0 0 3 3 0 3 0 0 0 0 0 0 0 0 0 0 0 0 0 0 11 0 0 0 ++ xprt: rdma 0 0 1 0 0 1 1 0 1 0 0 0 0 0 0 0 0 0 0 0 0 0 0 11 0 0 0 ++ xprt: rdma 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 ++ xprt: rdma 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 ++ xprt: rdma 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 ++ xprt: rdma 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 ++ xprt: rdma 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 ++ xprt: rdma 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 ++ per-op statistics ++ NULL: 1 1 0 40 24 6 0 6 0 ++ GETATTR: 0 0 0 0 0 0 0 0 0 ++ SETATTR: 0 0 0 0 0 0 0 0 0 ++ LOOKUP: 0 0 0 0 0 0 0 0 0 ++ ACCESS: 0 0 0 0 0 0 0 0 0 ++ READLINK: 0 0 0 0 0 0 0 0 0 ++ READ: 0 0 0 0 0 0 0 0 0 ++ WRITE: 0 0 0 0 0 0 0 0 0 ++ CREATE: 0 0 0 0 0 0 0 0 0 ++ MKDIR: 0 0 0 0 0 0 0 0 0 ++ SYMLINK: 0 0 0 0 0 0 0 0 0 ++ MKNOD: 0 0 0 0 0 0 0 0 0 ++ REMOVE: 0 0 0 0 0 0 0 0 0 ++ RMDIR: 0 0 0 0 0 0 0 0 0 ++ RENAME: 0 0 0 0 0 0 0 0 0 ++ LINK: 0 0 0 0 0 0 0 0 0 ++ READDIR: 0 0 0 0 0 0 0 0 0 ++ READDIRPLUS: 0 0 0 0 0 0 0 0 0 ++ FSSTAT: 0 0 0 0 0 0 0 0 0 ++ FSINFO: 2 2 0 184 328 6 0 6 0 ++ PATHCONF: 0 0 0 0 0 0 0 0 0 ++ COMMIT: 0 0 0 0 0 0 0 0 0 ++ ++device example.com:/icds mounted on /storage/icds with fstype nfs statvers=1.1 ++ opts: rw,vers=3,rsize=65536,wsize=65536,namlen=255,acregmin=3,acregmax=60,acdirmin=30,acdirmax=60,hard,proto=rdma,nconnect=8,port=20049,timeo=600,retrans=2,sec=sys,mountaddr=10.6.159.25,mountvers=3,mountport=0,mountproto=tcp,local_lock=all ++ age: 1166720 ++ caps: caps=0x10003fcf,wtmult=4096,dtsize=65536,bsize=0,namlen=255 ++ sec: flavor=1,pseudoflavor=1 ++ events: 2 191 0 0 3 13 207 0 0 30 0 0 0 0 2 0 0 0 0 0 0 0 0 0 0 0 0 ++ bytes: 2452 0 0 0 2555904 0 624 0 ++ RPC iostats version: 1.1 p/v: 100003/3 (nfs) ++ xprt: rdma 0 0 1 0 0 12 12 0 12 0 0 6 0 352256 352256 0 0 0 0 0 0 0 0 11 6 0 0 ++ xprt: rdma 0 0 1 0 0 10 10 0 10 0 0 5 0 184320 184320 0 0 0 0 0 0 0 0 11 5 0 0 ++ xprt: rdma 0 0 2 0 0 10 10 0 10 0 0 4 0 225280 225280 0 0 0 0 0 0 0 0 22 4 0 0 ++ xprt: rdma 0 0 1 0 0 9 9 0 9 0 0 6 0 294912 294912 0 0 0 0 0 0 0 0 11 6 0 0 ++ xprt: rdma 0 0 1 0 0 9 9 0 9 0 0 7 0 339968 339968 0 0 0 0 0 0 0 0 11 7 0 0 ++ xprt: rdma 0 0 1 0 0 9 9 0 9 0 0 7 0 352256 352256 0 0 0 0 0 0 0 0 11 7 0 0 ++ xprt: rdma 0 0 1 0 0 9 9 0 9 0 0 7 0 405504 405504 0 0 0 0 0 0 0 0 11 7 0 0 ++ xprt: rdma 0 0 1 0 0 9 9 0 9 0 0 7 0 401408 401408 0 0 0 0 0 0 0 0 11 7 0 0 ++ per-op statistics ++ NULL: 1 1 0 40 24 5 0 5 0 ++ GETATTR: 2 2 0 188 224 11 0 11 0 ++ SETATTR: 0 0 0 0 0 0 0 0 0 ++ LOOKUP: 13 13 0 1364 1952 15 3 19 5 ++ ACCESS: 9 9 0 864 1080 16 1 18 0 ++ READLINK: 0 0 0 0 0 0 0 0 0 ++ READ: 49 49 0 5096 2562176 0 15 17 0 ++ WRITE: 0 0 0 0 0 0 0 0 0 ++ CREATE: 0 0 0 0 0 0 0 0 0 ++ MKDIR: 0 0 0 0 0 0 0 0 0 ++ SYMLINK: 0 0 0 0 0 0 0 0 0 ++ MKNOD: 0 0 0 0 0 0 0 0 0 ++ REMOVE: 0 0 0 0 0 0 0 0 0 ++ RMDIR: 0 0 0 0 0 0 0 0 0 ++ RENAME: 0 0 0 0 0 0 0 0 0 ++ LINK: 0 0 0 0 0 0 0 0 0 ++ READDIR: 0 0 0 0 0 0 0 0 0 ++ READDIRPLUS: 0 0 0 0 0 0 0 0 0 ++ FSSTAT: 0 0 0 0 0 0 0 0 0 ++ FSINFO: 2 2 0 184 328 5 0 5 0 ++ PATHCONF: 0 0 0 0 0 0 0 0 0 ++ COMMIT: 0 0 0 0 0 0 0 0 0 ++ ++device example.com:/group mounted on /storage/group with fstype nfs statvers=1.1 ++ opts: rw,vers=3,rsize=65536,wsize=65536,namlen=255,acregmin=3,acregmax=60,acdirmin=30,acdirmax=60,hard,proto=rdma,nconnect=8,port=20049,timeo=600,retrans=2,sec=sys,mountaddr=10.6.159.1,mountvers=3,mountport=0,mountproto=tcp,local_lock=all ++ age: 1166720 ++ caps: caps=0x10003fcf,wtmult=4096,dtsize=65536,bsize=0,namlen=255 ++ sec: flavor=1,pseudoflavor=1 ++ events: 359470 11669975 26864 99083 237241 104622 12074173 76552450 12372 24061401 0 184482 64402 15675 250513 4475 60207 205420 0 13655 41997395 0 0 0 0 0 0 ++ bytes: 5990252963873 271047280191 0 0 1730835995062 212213971093 422603768 51862807 ++ RPC iostats version: 1.1 p/v: 100003/3 (nfs) ++ xprt: rdma 0 0 111 1 28 5001450 5001450 0 16468960 244630485 411153 4499604 2419 216488884485 216363618297 705084 2119278 0 610 0 0 0 0 4422 4913176 0 0 ++ xprt: rdma 0 0 110 1 28 5000369 5000369 0 16473884 244723520 411409 4497949 2419 216448785625 216324215309 744372 2220863 0 584 0 0 0 0 4466 4911777 0 0 ++ xprt: rdma 0 0 110 1 12 5001230 5001230 0 16466569 244632808 411122 4499109 2294 216463811427 216346453699 725596 2162884 0 656 0 0 0 0 4642 4912525 0 0 ++ xprt: rdma 0 0 114 1 12 5000643 5000643 0 16476687 244518169 411370 4498397 2400 216437529332 216314007816 754588 2166359 0 638 0 0 0 0 4675 4912167 0 0 ++ xprt: rdma 0 0 113 0 12 5001675 5001675 0 16476119 244839819 411304 4499163 2420 216552802382 216427322070 674092 2054877 0 666 0 0 0 0 4796 4912887 0 0 ++ xprt: rdma 0 0 112 1 62 5001928 5001928 0 16478906 244718584 411345 4499807 2449 216598144747 216470930799 709516 2049733 0 629 0 0 0 0 4664 4913601 0 0 ++ xprt: rdma 0 0 108 0 62 5000620 5000620 0 16457063 244260204 411118 4498680 2347 216410122025 216289485037 742848 2157015 0 618 0 0 0 0 4488 4912145 0 0 ++ xprt: rdma 0 0 105 0 62 5001170 5001170 0 16465684 244462170 411102 4499474 2363 216499718003 216377867059 774804 2218299 0 664 0 0 0 0 4532 4912939 0 0 ++ per-op statistics ++ NULL: 1 1 0 40 24 6 0 6 0 ++ GETATTR: 359470 359470 0 37679148 40260640 6615 68172 77464 0 ++ SETATTR: 21554 21554 0 3038404 775944 123 18549 18914 0 ++ LOOKUP: 116511 116511 0 14261900 8782176 956 41688 44033 90189 ++ ACCESS: 64375 64375 0 7046576 7725000 886 14014 15459 0 ++ READLINK: 736 736 0 79416 104332 2 249 269 0 ++ READ: 36019499 36019499 0 4279070464 1735446589952 406570 26116356 27160896 0 ++ WRITE: 3304938 3304938 0 212624926960 528790080 1195907908 50317800 1246575394 0 ++ CREATE: 77879 77879 0 11891764 19937024 494 299155 303286 0 ++ MKDIR: 2201 2201 0 325500 545856 26 7601 7669 80 ++ SYMLINK: 4 4 0 768 1024 0 13 13 0 ++ MKNOD: 0 0 0 0 0 0 0 0 0 ++ REMOVE: 15039 15039 0 1895992 541404 46 25825 26057 0 ++ RMDIR: 1859 1859 0 223452 66924 4 3096 3127 3 ++ RENAME: 6525 6525 0 1219928 287100 79 13674 13918 0 ++ LINK: 0 0 0 0 0 0 0 0 0 ++ READDIR: 5842 5842 0 747112 19898184 14 7271 7470 0 ++ READDIRPLUS: 9028 9028 0 1164116 74659892 169 17053 17575 0 ++ FSSTAT: 113 113 0 11856 18984 34 126 162 0 ++ FSINFO: 2 2 0 184 328 6 0 6 0 ++ PATHCONF: 0 0 0 0 0 0 0 0 0 ++ COMMIT: 0 0 0 0 0 0 0 0 0 ++ ++device example.com:/home mounted on /storage/home with fstype nfs statvers=1.1 ++ opts: rw,vers=3,rsize=65536,wsize=65536,namlen=255,acregmin=3,acregmax=60,acdirmin=30,acdirmax=60,hard,proto=rdma,nconnect=8,port=20049,timeo=600,retrans=2,sec=sys,mountaddr=10.6.159.7,mountvers=3,mountport=0,mountproto=tcp,local_lock=all ++ age: 1166720 ++ caps: caps=0x10003fcf,wtmult=4096,dtsize=65536,bsize=0,namlen=255 ++ sec: flavor=1,pseudoflavor=1 ++ events: 1431711 44943044 116357 137881 1036341 97602 45725153 741968 1060 314347 7 82823 747343 132339 751136 4247 9781 660494 0 1621 659329 139 0 0 0 0 0 ++ bytes: 31191656856 2345480650 0 0 13941630626 2386575218 3519989 615297 ++ RPC iostats version: 1.1 p/v: 100003/3 (nfs) ++ xprt: rdma 0 0 299 2 19 314383 314383 0 360317 2824 6031 38731 7094 2143783391 1741491091 2076748 3911658 0 48 0 0 0 0 3652 51856 0 0 ++ xprt: rdma 0 0 306 2 19 314284 314284 0 359761 3339 6178 38680 7086 2138653820 1737586668 2122588 3993750 0 53 0 0 0 0 3729 51944 0 0 ++ xprt: rdma 0 0 305 2 19 314534 314534 0 360342 3020 6127 39049 7491 2177604450 1749298514 2178016 4057012 0 51 0 0 0 0 3718 52667 0 0 ++ xprt: rdma 0 0 304 2 19 314570 314568 2 687294 2628 6152 38977 6977 2151798321 1756744897 2311096 3945741 0 45 0 0 0 0 3707 52106 0 0 ++ xprt: rdma 0 0 301 3 19 314546 314545 1 495960 2588 6045 38911 7196 2164869636 1756151804 1995156 3909706 0 48 0 0 0 0 3674 52152 0 0 ++ xprt: rdma 0 0 300 2 19 314597 314597 0 361338 2792 6190 39020 6968 2150734096 1755759176 2129072 3997461 0 47 0 0 0 0 3663 52178 0 0 ++ xprt: rdma 0 0 300 2 19 314532 314532 0 360982 2603 6177 38991 7215 2162670305 1752241557 2106240 4097289 0 45 0 0 0 0 3663 52383 0 0 ++ xprt: rdma 0 0 299 2 19 314696 314696 0 360978 2938 6117 38938 7297 2164527924 1748297356 2332620 3915498 0 51 0 0 0 0 3652 52352 0 0 ++ per-op statistics ++ NULL: 1 1 0 40 24 5 0 5 0 ++ GETATTR: 1431711 1431711 0 152340376 160351352 23887 262978 318138 5 ++ SETATTR: 134526 134526 0 20992092 4842936 592 62889 65101 0 ++ LOOKUP: 105669 105669 0 13828636 11957600 1842 36086 41336 61002 ++ ACCESS: 228039 228039 0 26382944 27364652 2814 47214 53250 7 ++ READLINK: 1627 1627 0 170916 228932 15 337 391 0 ++ READ: 420683 420683 0 50800636 13995575392 7785 255480 270435 0 ++ WRITE: 87100 87100 0 2397805952 13936000 172750 365004 540345 0 ++ CREATE: 29307 29307 0 4889564 7501712 111 49570 50330 4 ++ MKDIR: 7156 7156 0 1067212 1800036 15 11511 11613 145 ++ SYMLINK: 446 446 0 82020 113516 1 682 688 3 ++ MKNOD: 0 0 0 0 0 0 0 0 0 ++ REMOVE: 5852 5852 0 836620 210672 17 13008 13133 0 ++ RMDIR: 2417 2417 0 277660 87012 5 3319 3357 54 ++ RENAME: 2882 2882 0 542372 126808 62 26287 26407 0 ++ LINK: 212 212 0 32072 26288 0 420 425 0 ++ READDIR: 26 26 0 3348 156548 0 81 82 0 ++ READDIRPLUS: 46977 46977 0 6063288 86387612 202 55663 58039 0 ++ FSSTAT: 2784 2784 0 291028 467712 28 3651 4047 0 ++ FSINFO: 2 2 0 184 328 5 0 5 0 ++ PATHCONF: 0 0 0 0 0 0 0 0 0 ++ COMMIT: 0 0 0 0 0 0 0 0 0 ++ ++device example.com:/icds mounted on /storage/icds with fstype nfs statvers=1.1 ++ opts: rw,vers=3,rsize=65536,wsize=65536,namlen=255,acregmin=3,acregmax=60,acdirmin=30,acdirmax=60,hard,proto=rdma,nconnect=8,port=20049,timeo=600,retrans=2,sec=sys,mountaddr=10.6.159.23,mountvers=3,mountport=0,mountproto=tcp,local_lock=all ++ age: 1166720 ++ caps: caps=0x10003fcf,wtmult=4096,dtsize=65536,bsize=0,namlen=255 ++ sec: flavor=1,pseudoflavor=1 ++ events: 1504197 4360204878 3573 11827 998351 46870 4361262723 0 13 86345 0 0 412964 0 810302 0 0 787954 0 0 0 0 0 0 0 0 0 ++ bytes: 2473882007460 0 0 0 3117996888 0 781324 0 ++ RPC iostats version: 1.1 p/v: 100003/3 (nfs) ++ xprt: rdma 0 0 9 0 15 218040 218040 0 220710 0 0 10661 757 426647800 380864004 0 2882858 0 0 0 0 0 0 99 11418 0 0 ++ xprt: rdma 0 0 11 0 20 217911 217911 0 220214 0 0 10742 764 434579499 388755611 0 2968495 0 0 0 0 0 0 121 11506 0 0 ++ xprt: rdma 0 0 11 0 20 218423 218423 0 220758 0 0 10858 830 439053205 388939529 0 2997186 0 0 0 0 0 0 121 11688 0 0 ++ xprt: rdma 0 0 10 0 20 217656 217656 0 220142 0 0 10798 776 439744412 392148264 0 2965342 0 0 0 0 0 0 110 11574 0 0 ++ xprt: rdma 0 0 10 0 20 218280 218280 0 220648 0 0 10783 764 433088272 387536212 0 2916853 0 0 0 0 0 0 110 11547 0 0 ++ xprt: rdma 0 0 10 0 15 217897 217897 0 220528 0 0 10800 744 432500217 387918105 0 3035589 0 0 0 0 0 0 110 11544 0 0 ++ xprt: rdma 0 0 9 0 15 218233 218233 0 220901 0 0 10876 786 437891918 390297186 0 2843850 0 0 0 0 0 0 99 11662 0 0 ++ xprt: rdma 0 0 9 0 15 218287 218287 0 221066 0 0 10817 799 438124667 390593603 0 2900001 0 0 0 0 0 0 99 11616 0 0 ++ per-op statistics ++ NULL: 1 1 0 40 24 5 0 5 0 ++ GETATTR: 1504197 1504197 0 159348668 168470064 5823 250532 346757 0 ++ SETATTR: 0 0 0 0 0 0 0 0 0 ++ LOOKUP: 47852 47852 0 6159548 5950144 134 18038 23896 24837 ++ ACCESS: 79076 79076 0 8750456 9487564 188 14705 18515 389 ++ READLINK: 331 331 0 36048 45912 1 109 121 0 ++ READ: 106156 106156 0 12595328 3131627640 2442 102180 106716 0 ++ WRITE: 0 0 0 0 0 0 0 0 0 ++ CREATE: 972 972 0 159400 34992 4 1815 1875 972 ++ MKDIR: 130 130 0 18948 4680 0 259 260 130 ++ SYMLINK: 0 0 0 0 0 0 0 0 0 ++ MKNOD: 0 0 0 0 0 0 0 0 0 ++ REMOVE: 0 0 0 0 0 0 0 0 0 ++ RMDIR: 0 0 0 0 0 0 0 0 0 ++ RENAME: 0 0 0 0 0 0 0 0 0 ++ LINK: 0 0 0 0 0 0 0 0 0 ++ READDIR: 0 0 0 0 0 0 0 0 0 ++ READDIRPLUS: 5863 5863 0 765784 12555964 21 6882 7339 0 ++ FSSTAT: 120 120 0 13172 20160 0 86 88 0 ++ FSINFO: 2 2 0 184 328 5 0 5 0 ++ PATHCONF: 0 0 0 0 0 0 0 0 0 ++ COMMIT: 0 0 0 0 0 0 0 0 0 ++ ++device example.com:/work mounted on /storage/work with fstype nfs statvers=1.1 ++ opts: rw,vers=3,rsize=65536,wsize=65536,namlen=255,acregmin=3,acregmax=60,acdirmin=30,acdirmax=60,hard,proto=rdma,nconnect=8,port=20049,timeo=600,retrans=2,sec=sys,mountaddr=10.6.159.18,mountvers=3,mountport=0,mountproto=tcp,local_lock=all ++ age: 1166720 ++ caps: caps=0x10003fcf,wtmult=4096,dtsize=65536,bsize=0,namlen=255 ++ sec: flavor=1,pseudoflavor=1 ++ events: 963419 23064824 4957 23437 498893 36863 23579290 20398227 3127 191254 348 187305 207697 6261 405125 1210 0 394647 0 3923 18843784 3 0 0 0 0 0 ++ bytes: 41568180225 14269941286 0 0 23191576124 14729236454 5668359 3684526 ++ RPC iostats version: 1.1 p/v: 100003/3 (nfs) ++ xprt: rdma 0 0 111 1 1 231044 231042 2 873131 366580 33899 45806 1273 2967035298 2901558270 7296576 1169614 0 47 0 0 0 0 1529 80978 0 0 ++ xprt: rdma 0 0 106 0 1 231329 231328 1 717682 361947 33828 45455 1262 2952240311 2887232063 7392372 1221754 0 52 0 0 0 0 1474 80545 0 0 ++ xprt: rdma 0 0 106 1 6 231078 231078 0 485645 363676 33786 45721 1320 2967925402 2899619554 7176800 1189264 0 48 0 0 0 0 1474 80827 0 0 ++ xprt: rdma 0 0 109 0 6 231226 231226 0 486538 362707 33929 45482 1274 2956234946 2890484558 7294244 1190630 0 44 0 0 0 0 1507 80685 0 0 ++ xprt: rdma 0 0 109 0 1 231464 231464 0 486184 367681 33875 45789 1233 2968915191 2906173167 7325356 1151195 0 42 0 0 0 0 1507 80897 0 0 ++ xprt: rdma 0 0 110 0 1 231566 231566 0 485613 362833 33801 45688 1245 2967609016 2903914028 7186764 1166808 0 43 0 0 0 0 1518 80734 0 0 ++ xprt: rdma 0 0 107 0 1 231285 231285 0 485205 365643 33905 45719 1299 2968468178 2901567814 7074060 1190420 0 47 0 0 0 0 1485 80923 0 0 ++ xprt: rdma 0 0 109 1 1 231926 231926 0 487042 369268 33760 45696 1286 2970922269 2904447457 7343684 1204180 0 45 0 0 0 0 1507 80742 0 0 ++ per-op statistics ++ NULL: 1 1 0 40 24 6 0 6 0 ++ GETATTR: 963419 963419 0 101370948 107902704 15461 186184 213618 3 ++ SETATTR: 8104 8104 0 1151604 291744 47 11806 11970 0 ++ LOOKUP: 42796 42796 0 5399836 4859232 622 14626 15832 24620 ++ ACCESS: 45471 45471 0 4949688 5456520 1051 8791 10504 1 ++ READLINK: 14 14 0 1500 1908 0 3 3 0 ++ READ: 371033 371033 0 43379708 23239084264 2450 276265 283119 0 ++ WRITE: 390214 390214 0 14779478604 62433744 3531147 2810230 6351526 4 ++ CREATE: 14579 14579 0 2254256 3731564 70 47115 47475 3 ++ MKDIR: 2317 2317 0 378616 591172 17 9179 9302 9 ++ SYMLINK: 0 0 0 0 0 0 0 0 0 ++ MKNOD: 0 0 0 0 0 0 0 0 0 ++ REMOVE: 903 903 0 121280 32508 7 1340 1364 0 ++ RMDIR: 1 1 0 116 36 0 2 2 1 ++ RENAME: 1691 1691 0 336668 74404 14 2998 3057 0 ++ LINK: 71 71 0 12064 8804 0 91 92 0 ++ READDIR: 4 4 0 512 15012 0 27 28 0 ++ READDIRPLUS: 6573 6573 0 845980 12381640 70 7615 8194 0 ++ FSSTAT: 49 49 0 5256 8232 56 41 99 0 ++ FSINFO: 2 2 0 184 328 5 0 6 0 ++ PATHCONF: 0 0 0 0 0 0 0 0 0 ++ COMMIT: 0 0 0 0 0 0 0 0 0 ++ ++device example.com:/icds-prod mounted on /storage/prod with fstype nfs statvers=1.1 ++ opts: rw,vers=3,rsize=65536,wsize=65536,namlen=255,acregmin=3,acregmax=60,acdirmin=30,acdirmax=60,hard,proto=rdma,nconnect=8,port=20049,timeo=600,retrans=2,sec=sys,mountaddr=10.6.159.27,mountvers=3,mountport=0,mountproto=tcp,local_lock=all ++ age: 1166720 ++ caps: caps=0x10003fcf,wtmult=4096,dtsize=65536,bsize=0,namlen=255 ++ sec: flavor=1,pseudoflavor=1 ++ events: 216327 953839 0 2 94628 41 977594 0 0 32 0 0 7792 0 176250 0 0 90732 0 0 0 0 0 0 0 0 0 ++ bytes: 900620338 0 0 0 317803 0 96 0 ++ RPC iostats version: 1.1 p/v: 100003/3 (nfs) ++ xprt: rdma 0 0 1 0 120 27117 27117 0 27120 0 0 3 0 44329 44329 0 0 0 0 0 0 0 0 11 3 0 0 ++ xprt: rdma 0 0 1 0 120 27117 27117 0 27121 0 0 5 1 110959 82199 0 0 0 0 0 0 0 0 11 6 0 0 ++ xprt: rdma 0 0 1 0 28 27114 27114 0 27115 0 0 1 0 13215 13215 0 4239 0 0 0 0 0 0 11 1 0 0 ++ xprt: rdma 0 0 1 0 28 27113 27113 0 27113 0 0 2 0 61752 61752 0 395 0 0 0 0 0 0 11 2 0 0 ++ xprt: rdma 0 0 1 0 120 27115 27115 0 27118 0 0 2 1 24412 20192 0 3380 0 0 0 0 0 0 11 3 0 0 ++ xprt: rdma 0 0 1 0 120 27113 27113 0 27114 0 0 4 0 24016 24016 0 1518 0 0 0 0 0 0 11 4 0 0 ++ xprt: rdma 0 0 1 0 120 27115 27115 0 27119 0 0 3 1 118321 55597 0 518 0 0 0 0 0 0 11 4 0 0 ++ xprt: rdma 0 0 1 0 120 27113 27113 0 27114 0 0 2 1 75257 10593 0 0 0 0 0 0 0 0 11 3 0 0 ++ per-op statistics ++ NULL: 1 1 0 40 24 5 0 5 0 ++ GETATTR: 216327 216327 0 19911140 24228624 790 44005 48809 0 ++ SETATTR: 0 0 0 0 0 0 0 0 0 ++ LOOKUP: 45 45 0 5016 7200 16 15 32 15 ++ ACCESS: 494 494 0 49128 59280 17 148 169 0 ++ READLINK: 1 1 0 92 148 0 0 0 0 ++ READ: 32 32 0 3392 321932 1 28 30 0 ++ WRITE: 0 0 0 0 0 0 0 0 0 ++ CREATE: 0 0 0 0 0 0 0 0 0 ++ MKDIR: 0 0 0 0 0 0 0 0 0 ++ SYMLINK: 0 0 0 0 0 0 0 0 0 ++ MKNOD: 0 0 0 0 0 0 0 0 0 ++ REMOVE: 0 0 0 0 0 0 0 0 0 ++ RMDIR: 0 0 0 0 0 0 0 0 0 ++ RENAME: 0 0 0 0 0 0 0 0 0 ++ LINK: 0 0 0 0 0 0 0 0 0 ++ READDIR: 0 0 0 0 0 0 0 0 0 ++ READDIRPLUS: 2 2 0 236 3940 0 3 3 0 ++ FSSTAT: 11 11 0 1136 1848 0 8 8 0 ++ FSINFO: 2 2 0 184 328 5 0 6 0 ++ PATHCONF: 0 0 0 0 0 0 0 0 0 ++ COMMIT: 0 0 0 0 0 0 0 0 0 ++ ++device example.com:/icds-test mounted on /storage/test with fstype nfs statvers=1.1 ++ opts: rw,vers=3,rsize=65536,wsize=65536,namlen=255,acregmin=3,acregmax=60,acdirmin=30,acdirmax=60,hard,proto=rdma,nconnect=8,port=20049,timeo=600,retrans=2,sec=sys,mountaddr=10.6.159.29,mountvers=3,mountport=0,mountproto=tcp,local_lock=all ++ age: 1166720 ++ caps: caps=0x10003fcf,wtmult=4096,dtsize=65536,bsize=0,namlen=255 ++ sec: flavor=1,pseudoflavor=1 ++ events: 11 226 0 0 0 6 260 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 ++ bytes: 0 0 0 0 0 0 0 0 ++ RPC iostats version: 1.1 p/v: 100003/3 (nfs) ++ xprt: rdma 0 0 5 0 0 7 7 0 7 0 0 0 0 0 0 0 0 0 0 0 0 0 0 55 0 0 0 ++ xprt: rdma 0 0 6 0 0 6 6 0 6 0 0 0 1 28932 172 0 0 0 0 0 0 0 0 66 1 0 0 ++ xprt: rdma 0 0 5 0 0 5 5 0 5 0 0 0 0 0 0 0 0 0 0 0 0 0 0 55 0 0 0 ++ xprt: rdma 0 0 5 0 0 5 5 0 5 0 0 0 0 0 0 0 0 0 0 0 0 0 0 55 0 0 0 ++ xprt: rdma 0 0 5 0 0 5 5 0 5 0 0 0 0 0 0 0 0 0 0 0 0 0 0 55 0 0 0 ++ xprt: rdma 0 0 4 0 0 4 4 0 4 0 0 0 0 0 0 0 0 0 0 0 0 0 0 44 0 0 0 ++ xprt: rdma 0 0 4 0 0 4 4 0 4 0 0 0 0 0 0 0 0 0 0 0 0 0 0 44 0 0 0 ++ xprt: rdma 0 0 4 0 0 4 4 0 4 0 0 0 0 0 0 0 0 0 0 0 0 0 0 44 0 0 0 ++ per-op statistics ++ NULL: 1 1 0 40 24 6 0 6 0 ++ GETATTR: 11 11 0 1156 1232 174 4 178 0 ++ SETATTR: 0 0 0 0 0 0 0 0 0 ++ LOOKUP: 6 6 0 740 192 35 2 37 6 ++ ACCESS: 6 6 0 652 720 99 2 102 0 ++ READLINK: 0 0 0 0 0 0 0 0 0 ++ READ: 0 0 0 0 0 0 0 0 0 ++ WRITE: 0 0 0 0 0 0 0 0 0 ++ CREATE: 0 0 0 0 0 0 0 0 0 ++ MKDIR: 0 0 0 0 0 0 0 0 0 ++ SYMLINK: 0 0 0 0 0 0 0 0 0 ++ MKNOD: 0 0 0 0 0 0 0 0 0 ++ REMOVE: 0 0 0 0 0 0 0 0 0 ++ RMDIR: 0 0 0 0 0 0 0 0 0 ++ RENAME: 0 0 0 0 0 0 0 0 0 ++ LINK: 0 0 0 0 0 0 0 0 0 ++ READDIR: 0 0 0 0 0 0 0 0 0 ++ READDIRPLUS: 0 0 0 0 0 0 0 0 0 ++ FSSTAT: 12 12 0 1240 2016 109 6 116 0 ++ FSINFO: 2 2 0 184 328 5 0 6 0 ++ PATHCONF: 0 0 0 0 0 0 0 0 0 ++ COMMIT: 0 0 0 0 0 0 0 0 0 ++ ++device example.com:/icds-pcp-data mounted on /storage/pcp-data with fstype nfs statvers=1.1 ++ opts: rw,vers=3,rsize=65536,wsize=65536,namlen=255,acregmin=3,acregmax=60,acdirmin=30,acdirmax=60,hard,proto=rdma,nconnect=8,port=20049,timeo=600,retrans=2,sec=sys,mountaddr=10.6.159.6,mountvers=3,mountport=0,mountproto=tcp,local_lock=all ++ age: 1166720 ++ caps: caps=0x10003fcf,wtmult=4096,dtsize=65536,bsize=0,namlen=255 ++ sec: flavor=1,pseudoflavor=1 ++ events: 6699 70747 708 20599 2762 2253 102622 720956 1 5508 0 37492 1710 163 1296 0 0 1194 0 0 720956 0 0 0 0 0 0 ++ bytes: 1920695637 2068044591 0 0 1319950703 2143394964 322276 542160 ++ RPC iostats version: 1.1 p/v: 100003/3 (nfs) ++ xprt: rdma 0 0 1 0 163 11491 11491 0 11791 0 5235 2523 200 173699455 165053291 995572 0 0 0 0 0 0 0 11 7958 0 0 ++ xprt: rdma 0 0 1 0 163 11490 11490 0 11783 0 5227 2523 206 174106340 165126008 1038128 0 0 0 0 0 0 0 11 7956 0 0 ++ xprt: rdma 0 0 1 0 102 11490 11490 0 11795 0 5201 2531 215 174949903 165380715 1042544 0 0 0 0 0 0 0 11 7947 0 0 ++ xprt: rdma 0 0 1 0 102 11490 11490 0 11803 0 5249 2530 198 173876087 165402039 1006656 2100 0 0 0 0 0 0 11 7977 0 0 ++ xprt: rdma 0 0 1 0 102 11490 11490 0 11809 0 5227 2522 200 173699084 165094476 1049444 1968 0 0 0 0 0 0 11 7949 0 0 ++ xprt: rdma 0 0 1 0 43 11490 11490 0 11796 0 5213 2527 208 174387760 165255764 1003696 1736 0 0 0 0 0 0 11 7948 0 0 ++ xprt: rdma 0 0 1 0 43 11490 11490 0 11812 0 5229 2520 208 174061854 164930038 1022056 0 0 0 0 0 0 0 11 7957 0 0 ++ xprt: rdma 0 0 1 0 43 11490 11490 0 11797 0 5218 2519 216 174643548 164999120 1041756 0 0 0 0 0 0 0 11 7953 0 0 ++ per-op statistics ++ NULL: 1 1 0 40 24 5 0 5 0 ++ GETATTR: 6699 6699 0 658876 750288 34 1459 1700 0 ++ SETATTR: 878 878 0 112776 31608 3 378 409 0 ++ LOOKUP: 1631 1631 0 188400 64096 16 372 413 1569 ++ ACCESS: 952 952 0 95396 114240 20 197 241 0 ++ READLINK: 0 0 0 0 0 0 0 0 0 ++ READ: 20198 20198 0 2204072 1322536072 1049 7547 9069 0 ++ WRITE: 57666 57666 0 2149857320 9226560 1838 115379 120554 0 ++ CREATE: 826 826 0 99672 211456 3 1352 1378 0 ++ MKDIR: 15 15 0 2048 3840 0 32 32 0 ++ SYMLINK: 0 0 0 0 0 0 0 0 0 ++ MKNOD: 0 0 0 0 0 0 0 0 0 ++ REMOVE: 715 715 0 74796 25740 5 967 995 0 ++ RMDIR: 0 0 0 0 0 0 0 0 0 ++ RENAME: 2 2 0 304 88 0 3 3 0 ++ LINK: 0 0 0 0 0 0 0 0 0 ++ READDIR: 0 0 0 0 0 0 0 0 0 ++ READDIRPLUS: 700 700 0 84356 1157640 5 532 602 0 ++ FSSTAT: 684 684 0 65712 114912 8 750 794 0 ++ FSINFO: 2 2 0 184 328 5 0 5 0 ++ PATHCONF: 0 0 0 0 0 0 0 0 0 ++ COMMIT: 0 0 0 0 0 0 0 0 0 ++ ++device scratch mounted on /scratch with fstype gpfs +diff --git a/src/pmdas/nfsclient/pmdanfsclient.python b/src/pmdas/nfsclient/pmdanfsclient.python +index 8e96e9031..786d95845 100644 +--- a/src/pmdas/nfsclient/pmdanfsclient.python ++++ b/src/pmdas/nfsclient/pmdanfsclient.python +@@ -585,7 +585,7 @@ class NFSCLIENTPMDA(PMDA): + client.xprt.sending_u = self.longs(values[12]) + client.xprt.pending_u = self.longs(values[13]) + elif xprt_prot == 'udp': +- client.xprt.srcport = values[1] ++ client.xprt.srcport = self.chars(values[1]) + client.xprt.bind_count = self.longs(values[2]) + client.xprt.sends = self.longs(values[3]) + client.xprt.recvs = self.longs(values[4]) +@@ -596,7 +596,7 @@ class NFSCLIENTPMDA(PMDA): + client.xprt.sending_u = self.longs(values[9]) + client.xprt.pending_u = self.longs(values[10]) + elif xprt_prot == 'rdma': +- client.xprt.srcport = values[1] ++ client.xprt.srcport = self.chars(values[1]) + client.xprt.bind_count = self.longs(values[2]) + client.xprt.connect_count = self.longs(values[3]) + client.xprt.connect_time = self.longs(values[4]) diff --git a/SOURCES/redhat-bugzilla-2159207-pmproxy-rollup-fixes.patch b/SOURCES/redhat-bugzilla-2159207-pmproxy-rollup-fixes.patch new file mode 100644 index 0000000..12520cf --- /dev/null +++ b/SOURCES/redhat-bugzilla-2159207-pmproxy-rollup-fixes.patch @@ -0,0 +1,2711 @@ +diff -Naurp pcp-5.3.7.orig/man/man3/pmwebapi.3 pcp-5.3.7/man/man3/pmwebapi.3 +--- pcp-5.3.7.orig/man/man3/pmwebapi.3 2022-04-05 09:05:43.000000000 +1000 ++++ pcp-5.3.7/man/man3/pmwebapi.3 2023-07-05 13:43:00.404035611 +1000 +@@ -175,6 +175,10 @@ parameter. + The value passed in the request will be sent back in the + response \- all responses will be in JSON object form in + this case, with top level "client" and "result" fields. ++.PP ++REST API clients can optionally submit an URL-encoded query string ++in the body of the HTTP request unless otherwise noted. ++In this case the POST method must be used instead of the GET method. + .SS GET \fI/series/query\fR \- \fBpmSeriesQuery\fR(3) + .TS + box; +diff -Naurp pcp-5.3.7.orig/qa/1604 pcp-5.3.7/qa/1604 +--- pcp-5.3.7.orig/qa/1604 1970-01-01 10:00:00.000000000 +1000 ++++ pcp-5.3.7/qa/1604 2023-07-05 13:42:53.394025688 +1000 +@@ -0,0 +1,114 @@ ++#!/bin/sh ++# PCP QA Test No. 1604 ++# Exercise pmproxy REST API /series/values endpoint using curl(1). ++# ++# Copyright (c) 2022 Red Hat. ++# ++ ++seq=`basename $0` ++echo "QA output created by $seq" ++ ++# get standard environment, filters and checks ++. ./common.product ++. ./common.filter ++. ./common.check ++ ++_check_series ++which jq >/dev/null 2>&1 || _notrun "jq not installed" ++ ++_cleanup() ++{ ++ cd $here ++ [ -n "$pmproxy_pid" ] && $signal -s TERM $pmproxy_pid ++ [ -n "$options" ] && redis-cli $options shutdown ++ if $need_restore ++ then ++ need_restore=false ++ _restore_config $PCP_SYSCONF_DIR/pmproxy ++ fi ++ $sudo rm -rf $tmp $tmp.* ++} ++ ++status=1 # failure is the default! ++signal=$PCP_BINADM_DIR/pmsignal ++ ++userid=`id -u` ++username=`id -u -n` ++hostname=`hostname` ++machineid=`_machine_id` ++domainname=`_domain_name` ++ ++need_restore=false ++$sudo rm -rf $tmp $tmp.* $seq.full ++trap "_cleanup; exit \$status" 0 1 2 3 15 ++ ++_filter_source() ++{ ++ sed \ ++ -e "s,$here,PATH,g" \ ++ -e "s,$hostname,QAHOST,g" \ ++ #end ++} ++ ++_format_timestamps() ++{ ++ jq '.[].timestamp |= ((. / 1000 | strftime("%Y-%m-%d %H:%M:%S")) + "." + (. * 1000 % 1000000 | tostring))' ++} ++ ++# real QA test starts here ++_save_config $PCP_SYSCONF_DIR/pmproxy ++$sudo rm -f $PCP_SYSCONF_DIR/pmproxy/* ++need_restore=true ++ ++echo "Start test Redis server ..." ++redisport=`_find_free_port` ++options="-p $redisport" ++redis-server --port $redisport --save "" > $tmp.redis 2>&1 & ++_check_redis_ping $redisport ++_check_redis_server $redisport ++echo ++ ++_check_redis_server_version $redisport ++ ++# import some well-known test data into Redis ++pmseries $options --load "$here/archives/bozo-disk" | _filter_source ++ ++# start pmproxy ++proxyport=`_find_free_port` ++proxyopts="-p $proxyport -r $redisport -t" # -Dseries,http,af ++pmproxy -f -U $username -x $seq.full -l $tmp.pmproxy.log $proxyopts & ++pmproxy_pid=$! ++ ++# check pmproxy has started and is available for requests ++pmcd_wait -h localhost@localhost:$proxyport -v -t 5sec ++ ++series1=`pmseries $options disk.all.read` ++[ -z "$series1" ] && _fail "Cannot find any timeseries matching disk.all.read" ++echo "Using series $series1 for disk.all.read" ++ ++ ++echo "== no interval" | tee -a $seq.full ++url="http://localhost:$proxyport/series/values?series=$series1&start=1489620673&finish=1489620793" ++echo "$url" >> $seq.full ++curl --get --silent "$url" | tee -a $seq.full | _format_timestamps ++ ++echo "== 10s interval" | tee -a $seq.full ++url="http://localhost:$proxyport/series/values?series=$series1&start=1489620673&finish=1489620793&interval=10" ++echo "$url" >> $seq.full ++curl --get --silent "$url" | tee -a $seq.full | _format_timestamps ++ ++echo "== 20s interval" | tee -a $seq.full ++url="http://localhost:$proxyport/series/values?series=$series1&start=1489620673&finish=1489620793&interval=20" ++echo "$url" >> $seq.full ++curl --get --silent "$url" | tee -a $seq.full | _format_timestamps ++cat $tmp.pmproxy.log >> $seq.full ++ ++echo "== 20s interval, starting 2m before first sample in archive" | tee -a $seq.full ++url="http://localhost:$proxyport/series/values?series=$series1&start=1489620553&finish=1489620793&interval=20" ++echo "$url" >> $seq.full ++curl --get --silent "$url" | tee -a $seq.full | _format_timestamps ++ ++ ++cat $tmp.pmproxy.log >> $seq.full ++status=0 ++exit +diff -Naurp pcp-5.3.7.orig/qa/1604.out pcp-5.3.7/qa/1604.out +--- pcp-5.3.7.orig/qa/1604.out 1970-01-01 10:00:00.000000000 +1000 ++++ pcp-5.3.7/qa/1604.out 2023-07-05 13:42:53.394025688 +1000 +@@ -0,0 +1,204 @@ ++QA output created by 1604 ++Start test Redis server ... ++PING ++PONG ++ ++pmseries: [Info] processed 21 archive records from PATH/archives/bozo-disk ++Using series c61812b8ed3def24fa1df3fbf8d19a96cea3e0f9 for disk.all.read ++== no interval ++[ ++ { ++ "series": "c61812b8ed3def24fa1df3fbf8d19a96cea3e0f9", ++ "timestamp": "2017-03-15 23:31:13.890965", ++ "value": "1537640" ++ }, ++ { ++ "series": "c61812b8ed3def24fa1df3fbf8d19a96cea3e0f9", ++ "timestamp": "2017-03-15 23:31:23.891401", ++ "value": "1538109" ++ }, ++ { ++ "series": "c61812b8ed3def24fa1df3fbf8d19a96cea3e0f9", ++ "timestamp": "2017-03-15 23:31:33.891167", ++ "value": "1538453" ++ }, ++ { ++ "series": "c61812b8ed3def24fa1df3fbf8d19a96cea3e0f9", ++ "timestamp": "2017-03-15 23:31:43.891451", ++ "value": "1538888" ++ }, ++ { ++ "series": "c61812b8ed3def24fa1df3fbf8d19a96cea3e0f9", ++ "timestamp": "2017-03-15 23:31:53.891930", ++ "value": "1546137" ++ }, ++ { ++ "series": "c61812b8ed3def24fa1df3fbf8d19a96cea3e0f9", ++ "timestamp": "2017-03-15 23:32:03.891452", ++ "value": "1552940" ++ }, ++ { ++ "series": "c61812b8ed3def24fa1df3fbf8d19a96cea3e0f9", ++ "timestamp": "2017-03-15 23:32:13.891363", ++ "value": "1563099" ++ }, ++ { ++ "series": "c61812b8ed3def24fa1df3fbf8d19a96cea3e0f9", ++ "timestamp": "2017-03-15 23:32:23.891335", ++ "value": "1572878" ++ }, ++ { ++ "series": "c61812b8ed3def24fa1df3fbf8d19a96cea3e0f9", ++ "timestamp": "2017-03-15 23:32:33.891427", ++ "value": "1581847" ++ }, ++ { ++ "series": "c61812b8ed3def24fa1df3fbf8d19a96cea3e0f9", ++ "timestamp": "2017-03-15 23:32:43.891381", ++ "value": "1592546" ++ }, ++ { ++ "series": "c61812b8ed3def24fa1df3fbf8d19a96cea3e0f9", ++ "timestamp": "2017-03-15 23:32:53.891394", ++ "value": "1598167" ++ }, ++ { ++ "series": "c61812b8ed3def24fa1df3fbf8d19a96cea3e0f9", ++ "timestamp": "2017-03-15 23:33:03.891344", ++ "value": "1598172" ++ } ++] ++== 10s interval ++[ ++ { ++ "series": "c61812b8ed3def24fa1df3fbf8d19a96cea3e0f9", ++ "timestamp": "2017-03-15 23:31:13.890965", ++ "value": "1537640" ++ }, ++ { ++ "series": "c61812b8ed3def24fa1df3fbf8d19a96cea3e0f9", ++ "timestamp": "2017-03-15 23:31:23.891401", ++ "value": "1538109" ++ }, ++ { ++ "series": "c61812b8ed3def24fa1df3fbf8d19a96cea3e0f9", ++ "timestamp": "2017-03-15 23:31:33.891167", ++ "value": "1538453" ++ }, ++ { ++ "series": "c61812b8ed3def24fa1df3fbf8d19a96cea3e0f9", ++ "timestamp": "2017-03-15 23:31:43.891451", ++ "value": "1538888" ++ }, ++ { ++ "series": "c61812b8ed3def24fa1df3fbf8d19a96cea3e0f9", ++ "timestamp": "2017-03-15 23:31:53.891930", ++ "value": "1546137" ++ }, ++ { ++ "series": "c61812b8ed3def24fa1df3fbf8d19a96cea3e0f9", ++ "timestamp": "2017-03-15 23:32:03.891452", ++ "value": "1552940" ++ }, ++ { ++ "series": "c61812b8ed3def24fa1df3fbf8d19a96cea3e0f9", ++ "timestamp": "2017-03-15 23:32:13.891363", ++ "value": "1563099" ++ }, ++ { ++ "series": "c61812b8ed3def24fa1df3fbf8d19a96cea3e0f9", ++ "timestamp": "2017-03-15 23:32:23.891335", ++ "value": "1572878" ++ }, ++ { ++ "series": "c61812b8ed3def24fa1df3fbf8d19a96cea3e0f9", ++ "timestamp": "2017-03-15 23:32:33.891427", ++ "value": "1581847" ++ }, ++ { ++ "series": "c61812b8ed3def24fa1df3fbf8d19a96cea3e0f9", ++ "timestamp": "2017-03-15 23:32:43.891381", ++ "value": "1592546" ++ }, ++ { ++ "series": "c61812b8ed3def24fa1df3fbf8d19a96cea3e0f9", ++ "timestamp": "2017-03-15 23:32:53.891394", ++ "value": "1598167" ++ }, ++ { ++ "series": "c61812b8ed3def24fa1df3fbf8d19a96cea3e0f9", ++ "timestamp": "2017-03-15 23:33:03.891344", ++ "value": "1598172" ++ } ++] ++== 20s interval ++[ ++ { ++ "series": "c61812b8ed3def24fa1df3fbf8d19a96cea3e0f9", ++ "timestamp": "2017-03-15 23:31:13.890965", ++ "value": "1537640" ++ }, ++ { ++ "series": "c61812b8ed3def24fa1df3fbf8d19a96cea3e0f9", ++ "timestamp": "2017-03-15 23:31:23.891401", ++ "value": "1538109" ++ }, ++ { ++ "series": "c61812b8ed3def24fa1df3fbf8d19a96cea3e0f9", ++ "timestamp": "2017-03-15 23:31:43.891451", ++ "value": "1538888" ++ }, ++ { ++ "series": "c61812b8ed3def24fa1df3fbf8d19a96cea3e0f9", ++ "timestamp": "2017-03-15 23:32:03.891452", ++ "value": "1552940" ++ }, ++ { ++ "series": "c61812b8ed3def24fa1df3fbf8d19a96cea3e0f9", ++ "timestamp": "2017-03-15 23:32:23.891335", ++ "value": "1572878" ++ }, ++ { ++ "series": "c61812b8ed3def24fa1df3fbf8d19a96cea3e0f9", ++ "timestamp": "2017-03-15 23:32:43.891381", ++ "value": "1592546" ++ }, ++ { ++ "series": "c61812b8ed3def24fa1df3fbf8d19a96cea3e0f9", ++ "timestamp": "2017-03-15 23:33:03.891344", ++ "value": "1598172" ++ } ++] ++== 20s interval, starting 2m before first sample in archive ++[ ++ { ++ "series": "c61812b8ed3def24fa1df3fbf8d19a96cea3e0f9", ++ "timestamp": "2017-03-15 23:31:13.890965", ++ "value": "1537640" ++ }, ++ { ++ "series": "c61812b8ed3def24fa1df3fbf8d19a96cea3e0f9", ++ "timestamp": "2017-03-15 23:31:33.891167", ++ "value": "1538453" ++ }, ++ { ++ "series": "c61812b8ed3def24fa1df3fbf8d19a96cea3e0f9", ++ "timestamp": "2017-03-15 23:31:53.891930", ++ "value": "1546137" ++ }, ++ { ++ "series": "c61812b8ed3def24fa1df3fbf8d19a96cea3e0f9", ++ "timestamp": "2017-03-15 23:32:23.891335", ++ "value": "1572878" ++ }, ++ { ++ "series": "c61812b8ed3def24fa1df3fbf8d19a96cea3e0f9", ++ "timestamp": "2017-03-15 23:32:43.891381", ++ "value": "1592546" ++ }, ++ { ++ "series": "c61812b8ed3def24fa1df3fbf8d19a96cea3e0f9", ++ "timestamp": "2017-03-15 23:33:03.891344", ++ "value": "1598172" ++ } ++] +diff -Naurp pcp-5.3.7.orig/qa/1626 pcp-5.3.7/qa/1626 +--- pcp-5.3.7.orig/qa/1626 2023-07-05 13:42:25.513986223 +1000 ++++ pcp-5.3.7/qa/1626 2023-07-05 13:42:53.394025688 +1000 +@@ -2,7 +2,7 @@ + # PCP QA Test No. 1626 + # pmproxy metrics + # +-# Copyright (c) 2021 Red Hat. ++# Copyright (c) 2021-2022 Red Hat. + # + + seq=`basename $0` +@@ -71,10 +71,10 @@ val=`_probe_val pmproxy.pid` + pid=`_pmproxy_mainpid` + if [ "$pid" -eq "$val" ]; then :; else echo FAIL pid=$pid val=$val && exit; fi + +-echo === check initial pmproxy.webgroup metrics +-for m in instmap labelsmap namesmap contextmap; do +- [ `_probe_val pmproxy.webgroup.$m.size` -eq 0 ] && continue +- echo FAILED pmproxy.webgroup.$m.size expected to be zero && exit ++echo === check initial pmproxy.map metrics ++for m in instance label metric context; do ++ [ `_probe_val pmproxy.map.$m.size` -eq 0 ] && continue ++ echo FAILED pmproxy.map.$m.size expected to be zero && exit + done + + echo "=== start the metrics timer with a /metrics RESTAPI call" +@@ -85,18 +85,18 @@ val=`curl -Gs 'http://localhost:44322/me + echo "=== wait for the maps to be updated" + count=0 + while true; do +- sz=`_probe_val pmproxy.webgroup.namesmap.size` ++ sz=`_probe_val pmproxy.map.metric.size` + [ "$sz" -gt 0 ] && break + count=`expr $count + 1` + [ $count -gt 20 ] && echo FAILED sz=\"$sz\" after $count iterations && break + sleep 2 + done + +-echo === pmproxy.webgroup map size metrics should now be nonzero +-for m in instmap labelsmap namesmap contextmap; do +- sz=`_probe_val pmproxy.webgroup.$m.size` ++echo === pmproxy.map size metrics should now be nonzero ++for m in instance label metric context; do ++ sz=`_probe_val pmproxy.map.$m.size` + [ "$sz" -gt 0 ] && continue +- echo FAILED pmproxy.webgroup.$m.size is \"$sz\" but expected to be non-zero ++ echo FAILED pmproxy.map.$m.size is \"$sz\" but expected to be non-zero + exit + done + +diff -Naurp pcp-5.3.7.orig/qa/1626.out pcp-5.3.7/qa/1626.out +--- pcp-5.3.7.orig/qa/1626.out 2023-07-05 13:42:25.513986223 +1000 ++++ pcp-5.3.7/qa/1626.out 2023-07-05 13:42:53.394025688 +1000 +@@ -1,10 +1,10 @@ + QA output created by 1626 + == wait for pmproxy server metrics + === check pmproxy.pid +-=== check initial pmproxy.webgroup metrics ++=== check initial pmproxy.map metrics + === start the metrics timer with a /metrics RESTAPI call + === wait for the maps to be updated +-=== pmproxy.webgroup map size metrics should now be nonzero ++=== pmproxy.map size metrics should now be nonzero + === check pmproxy cpu counters + === check for discovery partial metadata reads + === check maxrss and datasz values +diff -Naurp pcp-5.3.7.orig/qa/1689.out pcp-5.3.7/qa/1689.out +--- pcp-5.3.7.orig/qa/1689.out 2023-07-05 13:42:25.513986223 +1000 ++++ pcp-5.3.7/qa/1689.out 2023-07-05 13:42:53.394025688 +1000 +@@ -165,6 +165,30 @@ pmproxy.discover.throttled_changed_callb + Help: + Number of filesystem change callbacks that were ignored due to throttling + ++pmproxy.map.context.size PMID: 4.1.6 [context map dictionary size] ++ Data Type: 32-bit unsigned int InDom: PM_INDOM_NULL 0xffffffff ++ Semantics: instant Units: none ++Help: ++Number of entries in the context map dictionary ++ ++pmproxy.map.instance.size PMID: 4.1.9 [instance names map dictionary size] ++ Data Type: 32-bit unsigned int InDom: PM_INDOM_NULL 0xffffffff ++ Semantics: instant Units: none ++Help: ++Number of entries in the instance name map dictionary ++ ++pmproxy.map.label.size PMID: 4.1.8 [label names map dictionary size] ++ Data Type: 32-bit unsigned int InDom: PM_INDOM_NULL 0xffffffff ++ Semantics: instant Units: none ++Help: ++Number of entries in the labels map dictionary ++ ++pmproxy.map.metric.size PMID: 4.1.7 [metric names map dictionary size] ++ Data Type: 32-bit unsigned int InDom: PM_INDOM_NULL 0xffffffff ++ Semantics: instant Units: none ++Help: ++Number of entries in the metric names map dictionary ++ + pmproxy.mem.datasz PMID: 4.1.5 [virtual data size] + Data Type: 64-bit unsigned int InDom: PM_INDOM_NULL 0xffffffff + Semantics: instant Units: Kbyte +@@ -291,26 +315,14 @@ pmproxy.series.values.calls PMID: 4.6.6 + Help: + total RESTAPI calls to /series/values + +-pmproxy.webgroup.contextmap.size PMID: 4.7.1 [context map dictionary size] +- Data Type: 32-bit unsigned int InDom: PM_INDOM_NULL 0xffffffff +- Semantics: instant Units: none +-Help: +-Number of entries in the context map dictionary +- +-pmproxy.webgroup.instmap.size PMID: 4.7.4 [instance name map dictionary size] +- Data Type: 32-bit unsigned int InDom: PM_INDOM_NULL 0xffffffff +- Semantics: instant Units: none +-Help: +-Number of entries in the instance name map dictionary +- +-pmproxy.webgroup.labelsmap.size PMID: 4.7.3 [labels map dictionary size] ++pmproxy.webgroup.gc.context.drops PMID: 4.7.2 [contexts dropped in last garbage collection] + Data Type: 32-bit unsigned int InDom: PM_INDOM_NULL 0xffffffff + Semantics: instant Units: none + Help: +-Number of entries in the labels map dictionary ++Contexts dropped during most recent webgroup garbage collection + +-pmproxy.webgroup.namesmap.size PMID: 4.7.2 [metric names map dictionary size] ++pmproxy.webgroup.gc.context.scans PMID: 4.7.1 [contexts scanned in last garbage collection] + Data Type: 32-bit unsigned int InDom: PM_INDOM_NULL 0xffffffff + Semantics: instant Units: none + Help: +-Number of entries in the metric names map dictionary ++Contexts scanned during most recent webgroup garbage collection +diff -Naurp pcp-5.3.7.orig/qa/1691 pcp-5.3.7/qa/1691 +--- pcp-5.3.7.orig/qa/1691 1970-01-01 10:00:00.000000000 +1000 ++++ pcp-5.3.7/qa/1691 2023-07-05 13:42:53.404025702 +1000 +@@ -0,0 +1,74 @@ ++#!/bin/sh ++# Exercise pmseries handling of loading archives and ignoring metrics thereof. ++# ++# Copyright (c) 2022 Red Hat. ++# ++ ++seq=`basename $0` ++echo "QA output created by $seq" ++ ++# get standard environment, filters and checks ++. ./common.product ++. ./common.filter ++. ./common.check ++ ++_check_series ++ ++_cleanup() ++{ ++ [ -n "$options" ] && redis-cli $options shutdown ++ cd $here ++ $sudo rm -rf $tmp $tmp.* ++} ++ ++status=1 # failure is the default! ++hostname=`pmhostname` ++redisport=`_find_free_port` ++options="-p $redisport" ++ ++$sudo rm -rf $tmp $tmp.* $seq.full ++trap "_cleanup; exit \$status" 0 1 2 3 15 ++ ++_filter_source() ++{ ++ sed \ ++ -e "s,$here,PATH,g" \ ++ -e "s,$hostname,QAHOST,g" \ ++ #end ++} ++ ++# real QA test starts here ++cat > $tmp.default.conf < $tmp.ignore.conf < $tmp.redis 2>&1 & ++_check_redis_ping $redisport ++_check_redis_server $redisport ++ ++_check_redis_server_version $redisport ++ ++echo && echo "Load archive using default config" ++pmseries $options -c $tmp.default.conf --load "{source.path: \"$here/archives/viewqa1\"}" | _filter_source ++ ++echo && echo "Query series kernel.*" ++pmseries $options -m `pmseries $options 'kernel.*'` ++ ++ ++echo && echo "Clear Redis DB" ++redis-cli $options flushall ++ ++echo && echo "Load archive using: exclude.metrics = kernel.all.cpu.i*, kernel.all.cpu.user" ++pmseries $options -c $tmp.ignore.conf --load "{source.path: \"$here/archives/viewqa1\"}" | _filter_source ++ ++echo && echo "Query series kernel.*" ++pmseries $options -m `pmseries $options 'kernel.*'` ++ ++# success, all done ++status=0 ++exit +diff -Naurp pcp-5.3.7.orig/qa/1691.out pcp-5.3.7/qa/1691.out +--- pcp-5.3.7.orig/qa/1691.out 1970-01-01 10:00:00.000000000 +1000 ++++ pcp-5.3.7/qa/1691.out 2023-07-05 13:42:53.404025702 +1000 +@@ -0,0 +1,44 @@ ++QA output created by 1691 ++Start test Redis server ... ++PING ++PONG ++ ++Load archive using default config ++pmseries: [Info] processed 151 archive records from PATH/archives/viewqa1 ++ ++Query series kernel.* ++ ++8b013dfdb7214e020848d0a4a9952ff2a3f777cc ++ Metric: kernel.all.cpu.idle ++ ++95135e6cb517cb50fd0c9e28985b944d88850332 ++ Metric: kernel.all.cpu.intr ++ ++a046e8429bb493a2b40fc23857157a262649d02d ++ Metric: kernel.all.cpu.wait.total ++ ++d38aff137f65367ce1aec169be675021a3ebb25c ++ Metric: kernel.all.cpu.nice ++ ++f0983eec7e7c01361317266c4259467d35e0ec3e ++ Metric: kernel.all.cpu.sys ++ ++e7aa0bd3dc7afc149badec1808fa4fa5c63a7fa3 ++ Metric: kernel.all.cpu.user ++ ++Clear Redis DB ++OK ++ ++Load archive using: exclude.metrics = kernel.all.cpu.i*, kernel.all.cpu.user ++pmseries: [Info] processed 151 archive records from PATH/archives/viewqa1 ++ ++Query series kernel.* ++ ++a046e8429bb493a2b40fc23857157a262649d02d ++ Metric: kernel.all.cpu.wait.total ++ ++f0983eec7e7c01361317266c4259467d35e0ec3e ++ Metric: kernel.all.cpu.sys ++ ++d38aff137f65367ce1aec169be675021a3ebb25c ++ Metric: kernel.all.cpu.nice +diff -Naurp pcp-5.3.7.orig/qa/1697 pcp-5.3.7/qa/1697 +--- pcp-5.3.7.orig/qa/1697 1970-01-01 10:00:00.000000000 +1000 ++++ pcp-5.3.7/qa/1697 2023-07-05 13:43:00.404035611 +1000 +@@ -0,0 +1,134 @@ ++#!/bin/sh ++# PCP QA Test No. 1697 ++# Valgrind pmproxy REST API testing. ++# Based on 1601 and 1696 ++ ++# Copyright (c) 2022 Red Hat. ++# ++ ++seq=`basename $0` ++echo "QA output created by $seq" ++ ++# get standard environment, filters and checks ++. ./common.product ++. ./common.filter ++. ./common.check ++. ./common.python ++ ++ ++_check_valgrind ++_check_series ++_check_redis_server_version_offline ++ ++_cleanup() ++{ ++ cd $here ++ [ -n "$options" ] && redis-cli $options shutdown ++ $sudo rm -rf $tmp $tmp.* ++} ++ ++status=1 # failure is the default! ++username=`id -u -n` ++$sudo rm -rf $tmp $tmp.* $seq.full ++trap "_cleanup; exit \$status" 0 1 2 3 15 ++ ++# create a pmproxy configuration ++cat < $tmp.conf ++[pmproxy] ++pcp.enabled = true ++http.enabled = true ++[pmseries] ++enabled = true ++EOF ++ ++_filter_source() ++{ ++ sed \ ++ -e "s,$here,PATH,g" \ ++ -e "s,$hostname,QAHOST,g" \ ++ #end ++} ++ ++_filter_proxyport() ++{ ++ sed \ ++ -e "s/ FD $proxyport / FD PORT /g" \ ++ -e '/PORT ipv6 /d' \ ++ # end ++} ++ ++# real QA test starts here ++echo "Start test Redis server ..." ++redisport=`_find_free_port` ++options="-p $redisport" ++redis-server --port $redisport --save "" > $tmp.redis 2>&1 & ++_check_redis_ping $redisport ++_check_redis_server $redisport ++echo ++ ++_check_redis_server_version $redisport ++ ++# import some well-known test data into Redis ++pmseries $options --load "$here/archives/proc" | _filter_source ++ ++# start pmproxy ++mkdir -p $tmp.pmproxy/pmproxy ++export PCP_RUN_DIR=$tmp.pmproxy ++export PCP_TMP_DIR=$tmp.pmproxy ++proxyport=`_find_free_port` ++$_valgrind_clean_assert pmproxy -f -p $proxyport -r $redisport -U $username -l- -c $tmp.conf >$tmp.valout 2>$tmp.valerr & ++pid=$! ++ ++# valgrind takes awhile to fire up ++i=0 ++while [ $i -lt 40 ] ++do ++ $PCP_BINADM_DIR/telnet-probe -c localhost $proxyport && break ++ sleep 1 ++ i=`expr $i + 1` ++done ++if $PCP_BINADM_DIR/telnet-probe -c localhost $proxyport ++then ++ echo "Startup took $i secs" >>$seq.full ++else ++ echo "Arrgh: valgrind failed start pmproxy and get port $proxyport ready after 30 secs" ++ exit ++fi ++ ++series1=`pmseries $options disk.all.read` ++[ -z "$series1" ] && _fail "Cannot find any timeseries matching disk.all.read" ++echo "Using series $series1 for disk.all.read" ++ ++series2=`pmseries $options disk.dev.read` ++[ -z "$series2" ] && _fail "Cannot find any timeseries matching disk.dev.read" ++echo "Using series $series2 for disk.dev.read" ++ ++series3=`pmseries $options kernel.all.uptime` ++[ -z "$series3" ] && _fail "Cannot find any timeseries matching kernel.all.uptime" ++echo "Using series $series3 for kernel.all.uptime" ++ ++ ++echo "== verify metric descs" | tee -a $seq.full ++curl --silent "http://localhost:$proxyport/series/descs" -d "series=$series1,$series2,$series3" | tee -a $seq.full | pmjson ++ ++echo "== verify metric names" | tee -a $seq.full ++curl --silent "http://localhost:$proxyport/series/metrics" -d "series=$series1,$series2,$series3" | tee -a $seq.full | pmjson ++ ++echo "== verify metric labels" | tee -a $seq.full ++curl --silent "http://localhost:$proxyport/series/labels" -d "series=$series1,$series3" | tee -a $seq.full | pmjson ++ ++echo "== verify metric insts" | tee -a $seq.full ++curl --silent "http://localhost:$proxyport/series/instances" -d "series=$series2" | tee -a $seq.full | pmjson ++ ++# valgrind takes awhile to shutdown too ++pmsignal $pid ++pmsleep 3.5 ++echo "=== valgrind stdout ===" | tee -a $seq.full ++cat $tmp.valout | _filter_valgrind ++ ++echo "=== valgrind stderr ===" | tee -a $seq.full ++cat $tmp.valerr | _filter_pmproxy_log | grep -v "Cannot connect to Redis" | _filter_proxyport ++ ++# success, all done ++status=0 ++exit +diff -Naurp pcp-5.3.7.orig/qa/1697.out pcp-5.3.7/qa/1697.out +--- pcp-5.3.7.orig/qa/1697.out 1970-01-01 10:00:00.000000000 +1000 ++++ pcp-5.3.7/qa/1697.out 2023-07-05 13:43:00.414035625 +1000 +@@ -0,0 +1,93 @@ ++QA output created by 1697 ++Start test Redis server ... ++PING ++PONG ++ ++pmseries: [Info] processed 5 archive records from PATH/archives/proc ++Using series 1440b8b8bfe69465340eb934e9086ae8212f3cff for disk.all.read ++Using series 605fc77742cd0317597291329561ac4e50c0dd12 for disk.dev.read ++Using series 01d8bc7fa75aaff98a08aa0b1c0f2394368d5183 for kernel.all.uptime ++== verify metric descs ++[ ++ { ++ "series": "1440b8b8bfe69465340eb934e9086ae8212f3cff", ++ "source": "2cd6a38f9339f2dd1f0b4775bda89a9e7244def6", ++ "pmid": "60.0.24", ++ "indom": "none", ++ "semantics": "counter", ++ "type": "u64", ++ "units": "count" ++ }, ++ { ++ "series": "605fc77742cd0317597291329561ac4e50c0dd12", ++ "source": "2cd6a38f9339f2dd1f0b4775bda89a9e7244def6", ++ "pmid": "60.0.4", ++ "indom": "60.1", ++ "semantics": "counter", ++ "type": "u32", ++ "units": "count" ++ }, ++ { ++ "series": "01d8bc7fa75aaff98a08aa0b1c0f2394368d5183", ++ "source": "2cd6a38f9339f2dd1f0b4775bda89a9e7244def6", ++ "pmid": "60.26.0", ++ "indom": "none", ++ "semantics": "instant", ++ "type": "u32", ++ "units": "sec" ++ } ++] ++== verify metric names ++[ ++ { ++ "series": "1440b8b8bfe69465340eb934e9086ae8212f3cff", ++ "name": "disk.all.read" ++ }, ++ { ++ "series": "605fc77742cd0317597291329561ac4e50c0dd12", ++ "name": "disk.dev.read" ++ }, ++ { ++ "series": "01d8bc7fa75aaff98a08aa0b1c0f2394368d5183", ++ "name": "kernel.all.uptime" ++ } ++] ++== verify metric labels ++[ ++ { ++ "series": "1440b8b8bfe69465340eb934e9086ae8212f3cff", ++ "labels": { ++ "hostname": "bozo-laptop" ++ } ++ }, ++ { ++ "series": "01d8bc7fa75aaff98a08aa0b1c0f2394368d5183", ++ "labels": { ++ "hostname": "bozo-laptop" ++ } ++ } ++] ++== verify metric insts ++[ ++ { ++ "series": "605fc77742cd0317597291329561ac4e50c0dd12", ++ "source": "2cd6a38f9339f2dd1f0b4775bda89a9e7244def6", ++ "instance": "c3795d8b757506a2901c6b08b489ba56cae7f0d4", ++ "id": 0, ++ "name": "sda" ++ } ++] ++=== valgrind stdout === ++=== valgrind stderr === ++Log for pmproxy on HOST started DATE ++ ++pmproxy: PID = PID ++pmproxy request port(s): ++ sts fd port family address ++ === ==== ===== ====== ======= ++ok FD unix UNIX_DOMAIN_SOCKET ++ok FD PORT inet INADDR_ANY ++[DATE] pmproxy(PID) Info: pmproxy caught SIGTERM ++[DATE] pmproxy(PID) Info: pmproxy Shutdown ++ ++Log finished DATE +diff -Naurp pcp-5.3.7.orig/qa/common.check pcp-5.3.7/qa/common.check +--- pcp-5.3.7.orig/qa/common.check 2023-07-05 13:42:25.513986223 +1000 ++++ pcp-5.3.7/qa/common.check 2023-07-05 13:42:53.404025702 +1000 +@@ -1226,7 +1226,7 @@ _wait_for_pmproxy_metrics() + { + _n=0 + while true; do +- pminfo -f pmproxy.pid pmproxy.cpu pmproxy.webgroup.instmap.size >/dev/null 2>&1 && return 0 ++ pminfo -f pmproxy.pid pmproxy.cpu pmproxy.map.instance.size >/dev/null 2>&1 && return 0 + sleep 0.25 + _n=`expr $_n + 1` + [ $_n -lt 20 ] && continue +diff -Naurp pcp-5.3.7.orig/qa/group pcp-5.3.7/qa/group +--- pcp-5.3.7.orig/qa/group 2023-07-05 13:42:25.523986237 +1000 ++++ pcp-5.3.7/qa/group 2023-07-05 13:43:00.414035625 +1000 +@@ -1869,6 +1869,7 @@ x11 + 1601 pmseries pmproxy local + 1602 pmproxy local + 1603 pmproxy local ++1604 pmseries pmproxy local + 1608 pmproxy local + 1613 pmda.linux kernel local + 1622:retired local +@@ -1890,10 +1891,12 @@ x11 + 1688 pmieconf local + 1689 pmproxy libpcp_web local + 1690 pmseries local ++1691 pmseries local + 1692 pmda.pmcd local + 1694 pidstat local python pcp pmlogextract + 1695 pmproxy valgrind local + 1696 pmproxy valgrind local ++1697 pmproxy valgrind local + 1700 pmda.bpftrace local python + 1701 pmda.bpftrace local python + 1702 pmda.bpftrace local python +diff -Naurp pcp-5.3.7.orig/src/libpcp_web/src/load.c pcp-5.3.7/src/libpcp_web/src/load.c +--- pcp-5.3.7.orig/src/libpcp_web/src/load.c 2023-07-05 13:42:25.523986237 +1000 ++++ pcp-5.3.7/src/libpcp_web/src/load.c 2023-07-05 13:42:53.414025716 +1000 +@@ -1,5 +1,5 @@ + /* +- * Copyright (c) 2017-2021 Red Hat. ++ * Copyright (c) 2017-2022 Red Hat. + * + * This library is free software; you can redistribute it and/or modify it + * under the terms of the GNU Lesser General Public License as published +@@ -15,6 +15,7 @@ + #include + #include + #include ++#include + #include "encoding.h" + #include "discover.h" + #include "schema.h" +@@ -93,7 +94,7 @@ load_prepare_metric(const char *name, vo + if ((sts = pmLookupName(1, &name, &pmid)) < 0) { + if (sts == PM_ERR_IPC) + cp->setup = 0; +- infofmt(msg, "failed to lookup metric name (pmid=%s): %s", ++ infofmt(msg, "failed to lookup metric name (name=%s): %s", + name, pmErrStr_r(sts, pmmsg, sizeof(pmmsg))); + batoninfo(baton, PMLOG_WARNING, msg); + } else if ((hname = strdup(name)) == NULL) { +@@ -557,48 +558,91 @@ server_cache_update_done(void *arg) + server_cache_window(baton); + } + +-void +-server_cache_window(void *arg) ++#if defined(HAVE_LIBUV) ++/* this function runs in a worker thread */ ++static void ++fetch_archive(uv_work_t *req) + { +- seriesLoadBaton *baton = (seriesLoadBaton *)arg; ++ seriesLoadBaton *baton = (seriesLoadBaton *)req->data; + seriesGetContext *context = &baton->pmapi; +- struct timeval *finish = &baton->timing.end; ++ context_t *cp = &context->context; + pmResult *result; + int sts; + +- seriesBatonCheckMagic(baton, MAGIC_LOAD, "server_cache_window"); +- seriesBatonCheckCount(context, "server_cache_window"); + assert(context->result == NULL); ++ if ((context->error = sts = pmUseContext(cp->context)) >= 0) ++ if ((context->error = sts = pmFetchArchive(&result)) >= 0) ++ context->result = result; ++} + +- if (pmDebugOptions.series) +- fprintf(stderr, "server_cache_window: fetching next result\n"); +- +- seriesBatonReference(context, "server_cache_window"); +- context->done = server_cache_series_finished; ++/* this function runs in the main thread */ ++static void ++fetch_archive_done(uv_work_t *req, int status) ++{ ++ seriesLoadBaton *baton = (seriesLoadBaton *)req->data; ++ seriesGetContext *context = &baton->pmapi; ++ struct timeval *finish = &baton->timing.end; ++ int sts = context->error; + +- if ((sts = pmFetchArchive(&result)) >= 0) { +- context->result = result; +- if (finish->tv_sec > result->timestamp.tv_sec || +- (finish->tv_sec == result->timestamp.tv_sec && +- finish->tv_usec >= result->timestamp.tv_usec)) { ++ free(req); ++ if (context->loaded) { ++ assert(context->result == NULL); ++ doneSeriesGetContext(context, "fetch_archive_done"); ++ } else if (sts >= 0) { ++ if (finish->tv_sec > context->result->timestamp.tv_sec || ++ (finish->tv_sec == context->result->timestamp.tv_sec && ++ finish->tv_usec >= context->result->timestamp.tv_usec)) { + context->done = server_cache_update_done; +- series_cache_update(baton, NULL); ++ series_cache_update(baton, baton->exclude_pmids); + } + else { + if (pmDebugOptions.series) +- fprintf(stderr, "server_cache_window: end of time window\n"); ++ fprintf(stderr, "%s: time window end\n", "fetch_archive_done"); + sts = PM_ERR_EOL; +- pmFreeResult(result); ++ pmFreeResult(context->result); + context->result = NULL; + } + } + + if (sts < 0) { +- context->error = sts; + if (sts != PM_ERR_EOL) + baton->error = sts; +- doneSeriesGetContext(context, "server_cache_window"); ++ doneSeriesGetContext(context, "fetch_archive_done"); + } ++ ++ doneSeriesLoadBaton(baton, "fetch_archive_done"); ++} ++#endif ++ ++void ++server_cache_window(void *arg) ++{ ++ seriesLoadBaton *baton = (seriesLoadBaton *)arg; ++ seriesGetContext *context = &baton->pmapi; ++ ++ seriesBatonCheckMagic(baton, MAGIC_LOAD, "server_cache_window"); ++ seriesBatonCheckCount(context, "server_cache_window"); ++ assert(context->result == NULL); ++ ++ if (pmDebugOptions.series) ++ fprintf(stderr, "%s: fetching next result\n", "server_cache_window"); ++ ++#if defined(HAVE_LIBUV) ++ seriesBatonReference(baton, "server_cache_window"); ++ seriesBatonReference(context, "server_cache_window"); ++ context->done = server_cache_series_finished; ++ ++ /* ++ * We must perform pmFetchArchive(3) in a worker thread ++ * because it contains blocking (synchronous) I/O calls ++ */ ++ uv_work_t *req = malloc(sizeof(uv_work_t)); ++ req->data = baton; ++ uv_queue_work(uv_default_loop(), req, fetch_archive, fetch_archive_done); ++#else ++ baton->error = -ENOTSUP; ++ server_cache_series_finished(arg); ++#endif + } + + static void +@@ -640,7 +684,35 @@ add_source_metric(seriesLoadBaton *baton + } + + static void +-load_prepare_metrics(seriesLoadBaton *baton) ++load_prepare_exclude_metric(const char *name, void *arg) ++{ ++ seriesLoadBaton *baton = (seriesLoadBaton *)arg; ++ char pmmsg[PM_MAXERRMSGLEN]; ++ sds msg; ++ pmID pmid; ++ int i; ++ int sts; ++ ++ /* ++ * check if this metric name matches any exclude pattern ++ * if it matches, add the pmID of this metric to the exclude list ++ */ ++ for (i = 0; i < baton->exclude_npatterns; i++) { ++ if (fnmatch(baton->exclude_patterns[i], name, 0) == 0) { ++ if ((sts = pmLookupName(1, &name, &pmid)) < 0) { ++ infofmt(msg, "failed to lookup metric name (name=%s): %s", ++ name, pmErrStr_r(sts, pmmsg, sizeof(pmmsg))); ++ batoninfo(baton, PMLOG_WARNING, msg); ++ } else { ++ dictAdd(baton->exclude_pmids, &pmid, NULL); ++ } ++ break; ++ } ++ } ++} ++ ++static void ++load_prepare_included_metrics(seriesLoadBaton *baton) + { + context_t *cp = &baton->pmapi.context; + const char **metrics = baton->metrics; +@@ -659,6 +731,57 @@ load_prepare_metrics(seriesLoadBaton *ba + } + } + ++static void ++load_prepare_excluded_metrics(seriesLoadBaton *baton) ++{ ++ pmSeriesModule *module = (pmSeriesModule *)baton->module; ++ seriesModuleData *data = getSeriesModuleData(module); ++ char pmmsg[PM_MAXERRMSGLEN]; ++ sds msg; ++ int i, sts; ++ sds exclude_metrics_option; ++ sds *patterns = NULL; ++ int npatterns = 0; ++ ++ if (data == NULL) { ++ baton->error = -ENOMEM; ++ return; ++ } ++ ++ if (!(exclude_metrics_option = pmIniFileLookup(data->config, "discover", "exclude.metrics"))) { ++ /* option not set, using default value of no excluded metrics */ ++ return; ++ } ++ ++ if (!(patterns = sdssplitlen(exclude_metrics_option, sdslen(exclude_metrics_option), ",", 1, &npatterns))) { ++ /* empty option, ignore */ ++ return; ++ } ++ ++ /* trim each comma-separated entry */ ++ for (i = 0; i < npatterns; i++) ++ patterns[i] = sdstrim(patterns[i], " "); ++ ++ baton->exclude_patterns = patterns; ++ baton->exclude_npatterns = npatterns; ++ ++ /* ++ * unfortunately we need to traverse the entire PMNS here to match the patterns (e.g. proc.*) ++ * against metrics and gather a list of pmIDs to exclude ++ * ++ * alternatively pattern matching could happen in series_cache_update(), however that will come ++ * with a performance penalty (looping through patterns + fnmatch() each in a hot path vs. ++ * simple pmID lookup in a dict) ++ */ ++ if ((sts = pmTraversePMNS_r("", load_prepare_exclude_metric, baton)) < 0) { ++ infofmt(msg, "PMNS traversal failed: %s", pmErrStr_r(sts, pmmsg, sizeof(pmmsg))); ++ batoninfo(baton, PMLOG_WARNING, msg); ++ } ++ ++ sdsfreesplitres(patterns, npatterns); ++ baton->exclude_patterns = NULL; ++} ++ + static int + load_prepare_timing(seriesLoadBaton *baton) + { +@@ -812,11 +935,12 @@ doneSeriesGetContext(seriesGetContext *c + if (seriesBatonDereference(context, caller) && context->done != NULL) + context->done(baton); + +- if (context->error) { ++ if (context->error && !context->loaded) { + char pmmsg[PM_MAXERRMSGLEN]; + sds msg; + + if (context->error == PM_ERR_EOL) { ++ context->loaded = 1; + infofmt(msg, "processed %llu archive records from %s", + context->count, context->context.name.sds); + batoninfo(baton, PMLOG_INFO, msg); +@@ -893,7 +1017,8 @@ connect_pmapi_source_service(void *arg) + } else if (baton->error == 0) { + /* setup metric and time-based filtering for source load */ + load_prepare_timing(baton); +- load_prepare_metrics(baton); ++ load_prepare_included_metrics(baton); ++ load_prepare_excluded_metrics(baton); + } + series_load_end_phase(baton); + } +@@ -966,6 +1091,7 @@ initSeriesLoadBaton(seriesLoadBaton *bat + + baton->errors = dictCreate(&intKeyDictCallBacks, baton); + baton->wanted = dictCreate(&intKeyDictCallBacks, baton); ++ baton->exclude_pmids = dictCreate(&intKeyDictCallBacks, baton); + } + + void +@@ -979,6 +1105,7 @@ freeSeriesLoadBaton(seriesLoadBaton *bat + freeSeriesGetContext(&baton->pmapi, 0); + dictRelease(baton->errors); + dictRelease(baton->wanted); ++ dictRelease(baton->exclude_pmids); + free(baton->metrics); + + memset(baton, 0, sizeof(*baton)); +diff -Naurp pcp-5.3.7.orig/src/libpcp_web/src/query.c pcp-5.3.7/src/libpcp_web/src/query.c +--- pcp-5.3.7.orig/src/libpcp_web/src/query.c 2023-07-05 13:42:25.523986237 +1000 ++++ pcp-5.3.7/src/libpcp_web/src/query.c 2023-07-05 13:42:53.414025716 +1000 +@@ -49,7 +49,7 @@ typedef struct seriesGetLookup { + } seriesGetLookup; + + typedef struct seriesGetQuery { +- node_t root; ++ node_t *root; + timing_t timing; + } seriesGetQuery; + +@@ -63,16 +63,14 @@ typedef struct seriesQueryBaton { + void *userdata; + redisSlots *slots; + int error; +- union { +- seriesGetLookup lookup; +- seriesGetQuery query; +- } u; ++ seriesGetLookup lookup; ++ seriesGetQuery query; + } seriesQueryBaton; + + static void series_pattern_match(seriesQueryBaton *, node_t *); + static int series_union(series_set_t *, series_set_t *); + static int series_intersect(series_set_t *, series_set_t *); +-static int series_calculate(seriesQueryBaton *, node_t *, int); ++static int series_calculate(node_t *, int, void *); + static void series_redis_hash_expression(seriesQueryBaton *, char *, int); + static void series_node_get_metric_name(seriesQueryBaton *, seriesGetSID *, series_sample_set_t *); + static void series_node_get_desc(seriesQueryBaton *, sds, series_sample_set_t *); +@@ -88,8 +86,8 @@ static void + initSeriesGetQuery(seriesQueryBaton *baton, node_t *root, timing_t *timing) + { + seriesBatonCheckMagic(baton, MAGIC_QUERY, "initSeriesGetQuery"); +- baton->u.query.root = *root; +- baton->u.query.timing = *timing; ++ baton->query.root = root; ++ baton->query.timing = *timing; + } + + static int +@@ -106,18 +104,21 @@ skip_free_value_set(node_t *np) + } + + static void +-freeSeriesQueryNode(node_t *np, int level) ++freeSeriesQueryNode(node_t *np) + { + int n_samples; ++ + if (np == NULL) + return; ++ + if (skip_free_value_set(np) != 0) { + int i, j, k; ++ + for (i = 0; i < np->value_set.num_series; i++) { + n_samples = np->value_set.series_values[i].num_samples; + if (n_samples < 0) n_samples = -n_samples; + for (j = 0; j < n_samples; j++) { +- for (k=0; k < np->value_set.series_values[i].series_sample[j].num_instances; k++) { ++ for (k = 0; k < np->value_set.series_values[i].series_sample[j].num_instances; k++) { + sdsfree(np->value_set.series_values[i].series_sample[j].series_instance[k].timestamp); + sdsfree(np->value_set.series_values[i].series_sample[j].series_instance[k].series); + sdsfree(np->value_set.series_values[i].series_sample[j].series_instance[k].data); +@@ -127,13 +128,22 @@ freeSeriesQueryNode(node_t *np, int leve + sdsfree(np->value_set.series_values[i].sid->name); + free(np->value_set.series_values[i].sid); + free(np->value_set.series_values[i].series_sample); ++ sdsfree(np->value_set.series_values[i].series_desc.indom); ++ sdsfree(np->value_set.series_values[i].series_desc.pmid); ++ sdsfree(np->value_set.series_values[i].series_desc.semantics); ++ sdsfree(np->value_set.series_values[i].series_desc.source); ++ sdsfree(np->value_set.series_values[i].series_desc.type); ++ sdsfree(np->value_set.series_values[i].series_desc.units); + } + free(np->value_set.series_values); + } +- freeSeriesQueryNode(np->left, level+1); +- freeSeriesQueryNode(np->right, level+1); +- if (level != 0) +- free(np); ++ freeSeriesQueryNode(np->left); ++ freeSeriesQueryNode(np->right); ++ if (np->result.nseries) ++ free(np->result.series); ++ sdsfree(np->value); ++ sdsfree(np->key); ++ free(np); + } + + static void +@@ -141,9 +151,7 @@ freeSeriesGetQuery(seriesQueryBaton *bat + { + seriesBatonCheckMagic(baton, MAGIC_QUERY, "freeSeriesGetQuery"); + seriesBatonCheckCount(baton, "freeSeriesGetQuery"); +- if (baton->error == 0) { +- freeSeriesQueryNode(&baton->u.query.root, 0); +- } ++ freeSeriesQueryNode(baton->query.root); + memset(baton, 0, sizeof(seriesQueryBaton)); + free(baton); + } +@@ -226,15 +234,15 @@ initSeriesGetLookup(seriesQueryBaton *ba + + /* pattern matching parameter, optional */ + if (nseries == 0 && series != NULL) +- baton->u.lookup.pattern = *series; ++ baton->lookup.pattern = *series; + /* else lookup array of individual sids */ + for (i = 0; i < nseries; i++) { +- sid = &baton->u.lookup.series[i]; ++ sid = &baton->lookup.series[i]; + initSeriesGetSID(sid, series[i], 0, baton); + } +- baton->u.lookup.nseries = nseries; +- baton->u.lookup.func = func; +- baton->u.lookup.map = map; ++ baton->lookup.nseries = nseries; ++ baton->lookup.func = func; ++ baton->lookup.map = map; + } + + static void +@@ -247,9 +255,9 @@ freeSeriesGetLookup(seriesQueryBaton *ba + seriesBatonCheckMagic(baton, MAGIC_QUERY, "freeSeriesGetLookup"); + seriesBatonCheckCount(baton, "freeSeriesGetLookup"); + +- nseries = baton->u.lookup.nseries; ++ nseries = baton->lookup.nseries; + for (i = 0; i < nseries; i++) { +- sid = &baton->u.lookup.series[i]; ++ sid = &baton->lookup.series[i]; + sdsfree(sid->name); + } + bytes = sizeof(seriesQueryBaton) + (nseries * sizeof(seriesGetSID)); +@@ -400,7 +408,7 @@ extract_mapping(seriesQueryBaton *baton, + + if (reply->type == REDIS_REPLY_STRING) { + key = sdsnewlen(reply->str, reply->len); +- entry = redisMapLookup(baton->u.lookup.map, key); ++ entry = redisMapLookup(baton->lookup.map, key); + sdsfree(key); + if (entry != NULL) { + key = redisMapValue(entry); +@@ -553,7 +561,14 @@ static int + use_next_sample(seriesSampling *sampling) + { + /* if the next timestamp is past our goal, use the current value */ +- if (pmTimespec_cmp(&sampling->next_timespec, &sampling->goal) > 0) { ++ double goal_delta = pmTimespec_delta(&sampling->next_timespec, &sampling->goal); ++ if (goal_delta > 0) { ++ /* if the goal significantly lags behind, reset it */ ++ /* this can happen when start < first sample or when there are gaps in the series */ ++ if (goal_delta > 2 * sampling->delta.tv_sec) { ++ sampling->goal = sampling->next_timespec; ++ } ++ + /* selected a value for this interval so move the goal posts */ + pmTimespec_add(&sampling->goal, &sampling->delta); + return 0; +@@ -567,7 +582,7 @@ series_values_reply(seriesQueryBaton *ba + { + seriesSampling sampling = {0}; + redisReply *reply, *sample, **elements; +- timing_t *tp = &baton->u.query.timing; ++ timing_t *tp = &baton->query.timing; + int n, sts, next, nelements; + sds msg, save_timestamp; + +@@ -1360,14 +1375,14 @@ static void + on_series_solve_setup(void *arg) + { + if (pmDebugOptions.query) +- fprintf(stderr, "on_series_solve_setup\n"); ++ fprintf(stderr, "%s\n", "on_series_solve_setup"); + } + + static void + on_series_solve_log(pmLogLevel level, sds message, void *arg) + { + if (pmDebugOptions.query) +- fprintf(stderr, "on_series_solve_log: %s\n", message); ++ fprintf(stderr, "%s: %s\n", "on_series_solve_log", message); + } + + static void +@@ -1377,7 +1392,7 @@ on_series_solve_done(int status, void *a + + seriesBatonCheckMagic(baton, MAGIC_QUERY, "on_series_solve_done"); + if (pmDebugOptions.query && pmDebugOptions.desperate) +- fprintf(stderr, "on_series_solve_done: arg=%p status=%d\n", arg, status); ++ fprintf(stderr, "%s: arg=%p status=%d\n", "on_series_solve_done", arg, status); + baton->callbacks->on_done(status, baton->userdata); + } + +@@ -1388,7 +1403,7 @@ on_series_solve_value(pmSID sid, pmSerie + + seriesBatonCheckMagic(baton, MAGIC_QUERY, "on_series_solve_value"); + if (pmDebugOptions.query && pmDebugOptions.desperate) +- fprintf(stderr, "on_series_solve_value: arg=%p %s %s %s\n", ++ fprintf(stderr, "%s: arg=%p %s %s %s\n", "on_series_solve_value", + arg, value->timestamp, value->data, value->series); + return baton->callbacks->on_value(sid, value, baton->userdata); + } +@@ -1399,9 +1414,9 @@ on_series_solve_inst_done(int status, vo + { + seriesQueryBaton *baton = arg; + +- seriesBatonCheckMagic(baton, MAGIC_QUERY, "on_series_solve_done"); ++ seriesBatonCheckMagic(baton, MAGIC_QUERY, "on_series_solve_inst_done"); + if (pmDebugOptions.query && pmDebugOptions.desperate) +- fprintf(stderr, "on_series_solve_done: arg=%p status=%d\n", arg, status); ++ fprintf(stderr, "%s: arg=%p status=%d\n", "on_series_solve_done", arg, status); + /* on_done is called by series_query_finished */ + seriesBatonDereference(baton, "on_series_solve_inst_done"); + } +@@ -1419,7 +1434,7 @@ on_series_solve_inst_value(pmSID sid, pm + + seriesBatonCheckMagic(baton, MAGIC_QUERY, "on_series_solve_inst_value"); + if (pmDebugOptions.query) { +- fprintf(stderr, "on_series_solve_inst_value: arg=%p %s %s %s\n", ++ fprintf(stderr, "%s: arg=%p %s %s %s\n", "on_series_solve_inst_value", + arg, value->timestamp, value->data, value->series); + } + +@@ -1478,11 +1493,11 @@ series_solve_sid_expr(pmSeriesSettings * + seriesBatonCheckMagic(sid, MAGIC_SID, "series_query_expr_reply"); + seriesBatonCheckMagic(baton, MAGIC_QUERY, "series_query_expr_reply"); + +- if (pmDebugOptions.query) { +- fprintf(stderr, "series_solve_sid_expr: SID %s, " +- "seriesQueryBaton=%p, pmSeriesBaton=userdata=%p expr=\"%s\"\n", +- sid->name, baton, baton->userdata, expr->query); +- } ++ if (pmDebugOptions.query) ++ fprintf(stderr, "%s: SID %s, seriesQueryBaton=%p, " ++ "pmSeriesBaton=userdata=%p expr=\"%s\"\n", ++ "series_solve_sid_expr", ++ sid->name, baton, baton->userdata, expr->query); + + /* ref baton until on_series_solve_done */ + seriesBatonReference(baton, "series_solve_sid_expr"); +@@ -1491,7 +1506,7 @@ series_solve_sid_expr(pmSeriesSettings * + pmSeriesSetSlots(&settings->module, baton->slots); + settings->module = *baton->module; /* struct cpy */ + +- sts = series_solve(settings, sp.expr, &baton->u.query.timing, ++ sts = series_solve(settings, sp.expr, &baton->query.timing, + PM_SERIES_FLAG_NONE, baton); + } + +@@ -1587,7 +1602,7 @@ series_value_count_only(timing_t *tp) + static void + series_prepare_time(seriesQueryBaton *baton, series_set_t *result) + { +- timing_t *tp = &baton->u.query.timing; ++ timing_t *tp = &baton->query.timing; + unsigned char *series = result->series; + seriesGetSID *sid; + char buffer[64], revbuf[64]; +@@ -1730,11 +1745,16 @@ series_query_report_matches(void *arg) + + seriesBatonReference(baton, "series_query_report_matches"); + +- has_function = series_calculate(baton, &baton->u.query.root, 0); +- ++ has_function = series_calculate(baton->query.root, 0, arg); ++ ++ /* ++ * Store the canonical query to Redis if this query statement has ++ * function operation. ++ */ + if (has_function != 0) + series_redis_hash_expression(baton, hashbuf, sizeof(hashbuf)); +- series_report_set(baton, &baton->u.query.root); ++ ++ series_report_set(baton, baton->query.root); + series_query_end_phase(baton); + } + +@@ -1747,7 +1767,7 @@ series_query_maps(void *arg) + seriesBatonCheckCount(baton, "series_query_maps"); + + seriesBatonReference(baton, "series_query_maps"); +- series_prepare_maps(baton, &baton->u.query.root, 0); ++ series_prepare_maps(baton, baton->query.root, 0); + series_query_end_phase(baton); + } + +@@ -1760,7 +1780,7 @@ series_query_eval(void *arg) + seriesBatonCheckCount(baton, "series_query_eval"); + + seriesBatonReference(baton, "series_query_eval"); +- series_prepare_eval(baton, &baton->u.query.root, 0); ++ series_prepare_eval(baton, baton->query.root, 0); + series_query_end_phase(baton); + } + +@@ -1773,7 +1793,7 @@ series_query_expr(void *arg) + seriesBatonCheckCount(baton, "series_query_expr"); + + seriesBatonReference(baton, "series_query_expr"); +- series_prepare_expr(baton, &baton->u.query.root, 0); ++ series_prepare_expr(baton, baton->query.root, 0); + series_query_end_phase(baton); + } + +@@ -1827,7 +1847,7 @@ series_values_store_to_node(seriesQueryB + { + seriesSampling sampling = {0}; + redisReply *reply, *sample, **elements; +- timing_t *tp = &baton->u.query.timing; ++ timing_t *tp = &baton->query.timing; + int i, sts, next, nelements; + int idx_series = np->value_set.num_series; + int idx_sample = 0; +@@ -2540,9 +2560,9 @@ series_rate_check(pmSeriesDesc desc) + * The number of samples in result is one less than the original samples. + */ + static void +-series_calculate_rate(node_t *np) ++series_calculate_rate(node_t *np, void *arg) + { +- seriesQueryBaton *baton = (seriesQueryBaton *)np->baton; ++ seriesQueryBaton *baton = (seriesQueryBaton *)arg; + pmSeriesValue s_pmval, t_pmval; + unsigned int n_instances, n_samples, i, j, k; + double s_data, t_data, mult; +@@ -2627,9 +2647,9 @@ series_calculate_rate(node_t *np) + * Compare and pick the maximal instance value(s) among samples for each metric. + */ + static void +-series_calculate_max(node_t *np) ++series_calculate_max(node_t *np, void *arg) + { +- seriesQueryBaton *baton = (seriesQueryBaton *)np->baton; ++ seriesQueryBaton *baton = (seriesQueryBaton *)arg; + unsigned int n_series, n_samples, n_instances, i, j, k; + double max_data, data; + int max_pointer; +@@ -2686,9 +2706,9 @@ series_calculate_max(node_t *np) + * Compare and pick the minimal instance value(s) among samples for each metric. + */ + static void +-series_calculate_min(node_t *np) ++series_calculate_min(node_t *np, void *arg) + { +- seriesQueryBaton *baton = (seriesQueryBaton *)np->baton; ++ seriesQueryBaton *baton = (seriesQueryBaton *)arg; + unsigned int n_series, n_samples, n_instances, i, j, k; + double min_data, data; + int min_pointer; +@@ -2838,9 +2858,9 @@ series_pmAtomValue_conv_str(int type, ch + * metrics to be modified. + */ + static void +-series_calculate_rescale(node_t *np) ++series_calculate_rescale(node_t *np, void *arg) + { +- seriesQueryBaton *baton = (seriesQueryBaton *)np->baton; ++ seriesQueryBaton *baton = (seriesQueryBaton *)arg; + double mult; + pmUnits iunit; + char *errmsg, str_val[256]; +@@ -2933,9 +2953,9 @@ series_abs_pmAtomValue(int type, pmAtomV + } + + static void +-series_calculate_abs(node_t *np) ++series_calculate_abs(node_t *np, void *arg) + { +- seriesQueryBaton *baton = (seriesQueryBaton *)np->baton; ++ seriesQueryBaton *baton = (seriesQueryBaton *)arg; + pmAtomValue val; + int type, sts, str_len, i, j, k; + char str_val[256]; +@@ -2976,9 +2996,10 @@ series_calculate_abs(node_t *np) + * calculate sum or avg series per-instance over time samples + */ + static void +-series_calculate_statistical(node_t *np, nodetype_t func) ++series_calculate_statistical(node_t *np, void *arg) + { +- seriesQueryBaton *baton = (seriesQueryBaton *)np->baton; ++ seriesQueryBaton *baton = (seriesQueryBaton *)arg; ++ nodetype_t func = np->type; + unsigned int n_series, n_samples, n_instances, i, j, k; + double sum_data, data; + char sum_data_str[64]; +@@ -3072,9 +3093,9 @@ series_floor_pmAtomValue(int type, pmAto + } + + static void +-series_calculate_floor(node_t *np) ++series_calculate_floor(node_t *np, void *arg) + { +- seriesQueryBaton *baton = (seriesQueryBaton *)np->baton; ++ seriesQueryBaton *baton = (seriesQueryBaton *)arg; + pmAtomValue val; + int type, sts, str_len, i, j, k; + char str_val[256]; +@@ -3157,9 +3178,9 @@ series_log_pmAtomValue(int itype, int *o + * Return the logarithm of x to base b (log_b^x). + */ + static void +-series_calculate_log(node_t *np) ++series_calculate_log(node_t *np, void *arg) + { +- seriesQueryBaton *baton = (seriesQueryBaton *)np->baton; ++ seriesQueryBaton *baton = (seriesQueryBaton *)arg; + double base; + pmAtomValue val; + int i, j, k, itype, otype=PM_TYPE_UNKNOWN; +@@ -3251,9 +3272,9 @@ series_sqrt_pmAtomValue(int itype, int * + } + + static void +-series_calculate_sqrt(node_t *np) ++series_calculate_sqrt(node_t *np, void *arg) + { +- seriesQueryBaton *baton = (seriesQueryBaton *)np->baton; ++ seriesQueryBaton *baton = (seriesQueryBaton *)arg; + pmAtomValue val; + int i, j, k, itype, otype=PM_TYPE_UNKNOWN; + int sts, str_len; +@@ -3320,9 +3341,9 @@ series_round_pmAtomValue(int type, pmAto + } + + static void +-series_calculate_round(node_t *np) ++series_calculate_round(node_t *np, void *arg) + { +- seriesQueryBaton *baton = (seriesQueryBaton *)np->baton; ++ seriesQueryBaton *baton = (seriesQueryBaton *)arg; + pmAtomValue val; + int i, j, k, type, sts, str_len; + char str_val[256]; +@@ -3393,6 +3414,7 @@ series_calculate_binary_check(int ope_ty + baton->error = -EPROTO; + return -1; + } ++ + /* + * For addition and subtraction all dimensions for each of + * the operands and result are identical. +@@ -3711,9 +3733,9 @@ series_binary_meta_update(node_t *left, + } + + static void +-series_calculate_plus(node_t *np) ++series_calculate_plus(node_t *np, void *arg) + { +- seriesQueryBaton *baton = (seriesQueryBaton *)np->baton; ++ seriesQueryBaton *baton = (seriesQueryBaton *)arg; + node_t *left = np->left, *right = np->right; + int l_type, r_type, otype=PM_TYPE_UNKNOWN; + int l_sem, r_sem, j, k; +@@ -3763,9 +3785,9 @@ series_calculate_plus(node_t *np) + } + + static void +-series_calculate_minus(node_t *np) ++series_calculate_minus(node_t *np, void *arg) + { +- seriesQueryBaton *baton = (seriesQueryBaton *)np->baton; ++ seriesQueryBaton *baton = (seriesQueryBaton *)arg; + node_t *left = np->left, *right = np->right; + unsigned int num_samples, num_instances, j, k; + pmAtomValue l_val, r_val; +@@ -3815,9 +3837,9 @@ series_calculate_minus(node_t *np) + } + + static void +-series_calculate_star(node_t *np) ++series_calculate_star(node_t *np, void *arg) + { +- seriesQueryBaton *baton = (seriesQueryBaton *)np->baton; ++ seriesQueryBaton *baton = (seriesQueryBaton *)arg; + node_t *left = np->left, *right = np->right; + unsigned int num_samples, num_instances, j, k; + pmAtomValue l_val, r_val; +@@ -3867,9 +3889,9 @@ series_calculate_star(node_t *np) + } + + static void +-series_calculate_slash(node_t *np) ++series_calculate_slash(node_t *np, void *arg) + { +- seriesQueryBaton *baton = (seriesQueryBaton *)np->baton; ++ seriesQueryBaton *baton = (seriesQueryBaton *)arg; + node_t *left = np->left, *right = np->right; + unsigned int num_samples, num_instances, j, k; + pmAtomValue l_val, r_val; +@@ -3878,12 +3900,16 @@ series_calculate_slash(node_t *np) + int l_sem, r_sem; + sds msg; + +- if (left->value_set.num_series == 0 || right->value_set.num_series == 0) return; +- if (series_calculate_binary_check(N_SLASH, baton, left, right, &l_type, &r_type, &l_sem, &r_sem, +- &l_units, &r_units, &large_units, left->value_set.series_values[0].series_desc.indom, +- right->value_set.series_values[0].series_desc.indom) != 0) { ++ if (left->value_set.num_series == 0 || right->value_set.num_series == 0) + return; +- } ++ ++ if (series_calculate_binary_check(N_SLASH, baton, left, right, ++ &l_type, &r_type, &l_sem, &r_sem, ++ &l_units, &r_units, &large_units, ++ left->value_set.series_values[0].series_desc.indom, ++ right->value_set.series_values[0].series_desc.indom) != 0) ++ return; ++ + num_samples = left->value_set.series_values[0].num_samples; + + for (j = 0; j < num_samples; j++) { +@@ -3912,7 +3938,6 @@ series_calculate_slash(node_t *np) + + series_binary_meta_update(left, &large_units, &l_sem, &r_sem, &otype); + np->value_set = left->value_set; +- + } + + /* +@@ -3924,81 +3949,65 @@ series_calculate_slash(node_t *np) + * store them into this node. + */ + static int +-series_calculate(seriesQueryBaton *baton, node_t *np, int level) ++series_calculate(node_t *np, int level, void *arg) + { + int sts; + + if (np == NULL) + return 0; +- if ((sts = series_calculate(baton, np->left, level+1)) < 0) ++ if ((sts = series_calculate(np->left, level+1, arg)) < 0) + return sts; +- if ((sts = series_calculate(baton, np->right, level+1)) < 0) ++ if ((sts = series_calculate(np->right, level+1, arg)) < 0) + return sts; + +- np->baton = baton; +- switch (np->type) { +- case N_RATE: +- series_calculate_rate(np); +- sts = N_RATE; +- break; +- case N_MAX: +- series_calculate_max(np); +- sts = N_MAX; +- break; +- case N_MIN: +- series_calculate_min(np); +- sts = N_MIN; +- break; +- case N_RESCALE: +- series_calculate_rescale(np); +- sts = N_RESCALE; +- break; +- case N_ABS: +- series_calculate_abs(np); +- sts = N_ABS; +- break; +- case N_FLOOR: +- series_calculate_floor(np); +- sts = N_FLOOR; +- break; +- case N_LOG: +- series_calculate_log(np); +- sts = N_LOG; +- break; +- case N_SQRT: +- series_calculate_sqrt(np); +- sts = N_SQRT; +- break; +- case N_ROUND: +- series_calculate_round(np); +- sts = N_ROUND; +- break; +- case N_PLUS: +- series_calculate_plus(np); +- sts = N_PLUS; +- break; +- case N_MINUS: +- series_calculate_minus(np); +- sts = N_MINUS; +- break; +- case N_STAR: +- series_calculate_star(np); +- sts = N_STAR; +- break; +- case N_SLASH: +- series_calculate_slash(np); +- sts = N_SLASH; +- break; +- case N_AVG: +- series_calculate_statistical(np, N_AVG); +- sts = N_AVG; +- break; +- case N_SUM: +- series_calculate_statistical(np, N_SUM); +- sts = N_SUM; +- break; +- default: +- break; ++ switch ((sts = np->type)) { ++ case N_RATE: ++ series_calculate_rate(np, arg); ++ break; ++ case N_MAX: ++ series_calculate_max(np, arg); ++ break; ++ case N_MIN: ++ series_calculate_min(np, arg); ++ break; ++ case N_RESCALE: ++ series_calculate_rescale(np, arg); ++ sts = N_RESCALE; ++ break; ++ case N_ABS: ++ series_calculate_abs(np, arg); ++ break; ++ case N_FLOOR: ++ series_calculate_floor(np, arg); ++ break; ++ case N_LOG: ++ series_calculate_log(np, arg); ++ break; ++ case N_SQRT: ++ series_calculate_sqrt(np, arg); ++ break; ++ case N_ROUND: ++ series_calculate_round(np, arg); ++ break; ++ case N_PLUS: ++ series_calculate_plus(np, arg); ++ break; ++ case N_MINUS: ++ series_calculate_minus(np, arg); ++ break; ++ case N_STAR: ++ series_calculate_star(np, arg); ++ break; ++ case N_SLASH: ++ series_calculate_slash(np, arg); ++ break; ++ case N_AVG: ++ case N_SUM: ++ series_calculate_statistical(np, arg); ++ break; ++ default: ++ sts = 0; /* no function */ ++ break; + } + return sts; + } +@@ -4006,9 +4015,9 @@ series_calculate(seriesQueryBaton *baton + static int + check_compatibility(pmUnits *units_a, pmUnits *units_b) + { +- if (compare_pmUnits_dim(units_a, units_b) == 0) { ++ if (compare_pmUnits_dim(units_a, units_b) == 0) + return 0; +- } else return -1; ++ return -1; + } + + static void +@@ -4072,7 +4081,7 @@ series_redis_hash_expression(seriesQuery + unsigned char hash[20]; + sds key, msg; + char *errmsg; +- node_t *np = &baton->u.query.root; ++ node_t *np = baton->query.root; + int i, j, num_series = np->value_set.num_series; + pmUnits units0, units1, large_units; + double mult; +@@ -4088,17 +4097,18 @@ series_redis_hash_expression(seriesQuery + + for (i = 0; i < num_series; i++) { + if (!np->value_set.series_values[i].compatibility) { +- infofmt(msg, "Descriptors for metric '%s' do not satisfy compatibility between different hosts/sources.\n", ++ infofmt(msg, "Descriptors for metric '%s' between different sources are incompatible.\n", + np->value_set.series_values[i].metric_name); + batoninfo(baton, PMLOG_ERROR, msg); + baton->error = -EPROTO; +- continue; ++ break; + } + + if (strncmp(np->value_set.series_values[i].series_desc.units, "none", 4) == 0) + memset(&units0, 0, sizeof(units0)); +- else if (pmParseUnitsStr(np->value_set.series_values[i].series_desc.units, +- &units0, &mult, &errmsg) != 0) { ++ else if (pmParseUnitsStr( ++ np->value_set.series_values[i].series_desc.units, ++ &units0, &mult, &errmsg) != 0) { + np->value_set.series_values[i].compatibility = 0; + infofmt(msg, "Invalid units string: %s\n", + np->value_set.series_values[i].series_desc.units); +@@ -4127,7 +4137,12 @@ series_redis_hash_expression(seriesQuery + + if (check_compatibility(&units0, &units1) != 0) { + np->value_set.series_values[j].compatibility = 0; +- infofmt(msg, "Incompatible units between operand metrics or expressions\n"); ++ infofmt(msg, "Incompatible units between operand metrics or expressions\n" ++ "'%s' (%s)\nvs\'%s' (%s)\n", ++ np->value_set.series_values[i].metric_name, ++ np->value_set.series_values[i].series_desc.units, ++ np->value_set.series_values[j].metric_name, ++ np->value_set.series_values[j].series_desc.units); + batoninfo(baton, PMLOG_ERROR, msg); + baton->error = -EPROTO; + break; +@@ -4175,7 +4190,7 @@ series_query_report_values(void *arg) + seriesBatonCheckCount(baton, "series_query_report_values"); + + seriesBatonReference(baton, "series_query_report_values"); +- series_prepare_time(baton, &baton->u.query.root.result); ++ series_prepare_time(baton, &baton->query.root->result); + series_query_end_phase(baton); + } + +@@ -4192,7 +4207,7 @@ series_query_funcs_report_values(void *a + seriesBatonReference(baton, "series_query_funcs_report_values"); + + /* For function-type nodes, calculate actual values */ +- has_function = series_calculate(baton, &baton->u.query.root, 0); ++ has_function = series_calculate(baton->query.root, 0, baton->userdata); + + /* + * Store the canonical query to Redis if this query statement has +@@ -4202,7 +4217,7 @@ series_query_funcs_report_values(void *a + series_redis_hash_expression(baton, hashbuf, sizeof(hashbuf)); + + /* time series values saved in root node so report them directly. */ +- series_node_values_report(baton, &baton->u.query.root); ++ series_node_values_report(baton, baton->query.root); + + series_query_end_phase(baton); + } +@@ -4217,7 +4232,7 @@ series_query_funcs(void *arg) + + seriesBatonReference(baton, "series_query_funcs"); + /* Process function-type node */ +- series_process_func(baton, &baton->u.query.root, 0); ++ series_process_func(baton, baton->query.root, 0); + series_query_end_phase(baton); + } + +@@ -4230,7 +4245,7 @@ series_query_desc(void *arg) + seriesBatonCheckCount(baton, "series_query_desc"); + + seriesBatonReference(baton, "series_query_desc"); +- series_expr_node_desc(baton, &baton->u.query.root); ++ series_expr_node_desc(baton, baton->query.root); + series_query_end_phase(baton); + } + +@@ -4380,7 +4395,7 @@ series_map_lookup_expr_reply(redisCluste + baton->error = sts; + } else { + /* call the on_metric callback, whatever it's set to by the caller */ +- baton->u.lookup.func(sid->name, query, baton->userdata); ++ baton->lookup.func(sid->name, query, baton->userdata); + } + } + series_query_end_phase(baton); +@@ -4434,8 +4449,8 @@ series_map_reply(seriesQueryBaton *baton + if (reply->type == REDIS_REPLY_STRING) { + sdsclear(key); + key = sdscatlen(key, reply->str, reply->len); +- if ((entry = redisMapLookup(baton->u.lookup.map, key)) != NULL) +- baton->u.lookup.func(series, redisMapValue(entry), baton->userdata); ++ if ((entry = redisMapLookup(baton->lookup.map, key)) != NULL) ++ baton->lookup.func(series, redisMapValue(entry), baton->userdata); + else { + infofmt(msg, "%s - timeseries string map", series); + batoninfo(baton, PMLOG_CORRUPT, msg); +@@ -4470,11 +4485,11 @@ series_map_keys_callback( + for (i = 0; i < reply->elements; i++) { + child = reply->element[i]; + if (child->type == REDIS_REPLY_STRING) { +- if (baton->u.lookup.pattern != NULL && +- fnmatch(baton->u.lookup.pattern, child->str, 0) != 0) ++ if (baton->lookup.pattern != NULL && ++ fnmatch(baton->lookup.pattern, child->str, 0) != 0) + continue; + val = sdscpylen(val, child->str, child->len); +- baton->u.lookup.func(NULL, val, baton->userdata); ++ baton->lookup.func(NULL, val, baton->userdata); + } else { + infofmt(msg, "bad response for string map %s (%s)", + HVALS, redis_reply_type(child)); +@@ -4599,7 +4614,7 @@ series_label_reply(seriesQueryBaton *bat + sdsclear(vmapID); + vmapID = sdscatlen(vmapID, elements[index+1]->str, elements[index+1]->len); + +- if ((entry = redisMapLookup(baton->u.lookup.map, nmapID)) != NULL) { ++ if ((entry = redisMapLookup(baton->lookup.map, nmapID)) != NULL) { + pmwebapi_hash_str((unsigned char *)nmapID, hashbuf, sizeof(hashbuf)); + vkey = sdscatfmt(sdsempty(), "label.%s.value", hashbuf); + vmap = redisMapCreate(vkey); +@@ -4680,9 +4695,9 @@ series_lookup_labels(void *arg) + seriesBatonCheckCount(baton, "series_lookup_labels"); + + /* unpack - iterate over series and extract labels names and values */ +- for (i = 0; i < baton->u.lookup.nseries; i++) { ++ for (i = 0; i < baton->lookup.nseries; i++) { + seriesBatonReference(baton, "series_lookup_labels"); +- sid = &baton->u.lookup.series[i]; ++ sid = &baton->lookup.series[i]; + key = sdscatfmt(sdsempty(), "pcp:labelvalue:series:%S", sid->name); + cmd = redis_command(2); + cmd = redis_param_str(cmd, HGETALL, HGETALL_LEN); +@@ -4712,7 +4727,7 @@ pmSeriesLabels(pmSeriesSettings *setting + initSeriesGetLookup(baton, nseries, series, settings->callbacks.on_label, labelsmap); + + if (nseries == 0) +- return series_map_keys(baton, redisMapName(baton->u.lookup.map)); ++ return series_map_keys(baton, redisMapName(baton->lookup.map)); + + baton->current = &baton->phases[0]; + baton->phases[i++].func = series_lookup_services; +@@ -4780,8 +4795,8 @@ series_lookup_labelvalues(void *arg) + seriesBatonCheckMagic(baton, MAGIC_QUERY, "series_lookup_labelvalues"); + seriesBatonCheckCount(baton, "series_lookup_labelvalues"); + +- for (i = 0; i < baton->u.lookup.nseries; i++) { +- sid = &baton->u.lookup.series[i]; ++ for (i = 0; i < baton->lookup.nseries; i++) { ++ sid = &baton->lookup.series[i]; + seriesBatonReference(baton, "series_lookup_labelvalues"); + + pmwebapi_string_hash(hash, sid->name, sdslen(sid->name)); +@@ -4874,8 +4889,8 @@ series_lookup_desc(void *arg) + seriesBatonCheckMagic(baton, MAGIC_QUERY, "series_lookup_desc"); + seriesBatonCheckCount(baton, "series_lookup_desc"); + +- for (i = 0; i < baton->u.lookup.nseries; i++) { +- sid = &baton->u.lookup.series[i]; ++ for (i = 0; i < baton->lookup.nseries; i++) { ++ sid = &baton->lookup.series[i]; + seriesBatonReference(baton, "series_lookup_desc"); + + key = sdscatfmt(sdsempty(), "pcp:desc:series:%S", sid->name); +@@ -5054,7 +5069,7 @@ series_inst_expr_reply(redisClusterAsync + baton->error = sts; + } else { + /* Parse the expr (with timing) and series solve the resulting expr tree */ +- baton->u.query.timing.count = 1; ++ baton->query.timing.count = 1; + baton->error = series_solve_sid_expr(&series_solve_inst_settings, &expr, arg); + } + } +@@ -5121,9 +5136,9 @@ series_lookup_instances(void *arg) + seriesBatonCheckMagic(baton, MAGIC_QUERY, "series_lookup_instances_callback"); + seriesBatonCheckCount(baton, "series_lookup_instances_callback"); + +- for (i = 0; i < baton->u.lookup.nseries; i++) { ++ for (i = 0; i < baton->lookup.nseries; i++) { + seriesBatonReference(baton, "series_lookup_instances_callback"); +- sid = &baton->u.lookup.series[i]; ++ sid = &baton->lookup.series[i]; + key = sdscatfmt(sdsempty(), "pcp:instances:series:%S", sid->name); + cmd = redis_command(2); + cmd = redis_param_str(cmd, SMEMBERS, SMEMBERS_LEN); +@@ -5153,7 +5168,7 @@ pmSeriesInstances(pmSeriesSettings *sett + initSeriesGetLookup(baton, nseries, series, settings->callbacks.on_instance, instmap); + + if (nseries == 0) +- return series_map_keys(baton, redisMapName(baton->u.lookup.map)); ++ return series_map_keys(baton, redisMapName(baton->lookup.map)); + + baton->current = &baton->phases[0]; + baton->phases[i++].func = series_lookup_services; +@@ -5177,7 +5192,7 @@ redis_lookup_mapping_callback( + + /* unpack - produce reverse map of ids-to-names for each context */ + if (LIKELY(reply && reply->type == REDIS_REPLY_ARRAY)) { +- reverse_map(baton, baton->u.lookup.map, reply->elements, reply->element); ++ reverse_map(baton, baton->lookup.map, reply->elements, reply->element); + } else { + infofmt(msg, "expected array from %s %s (type=%s)", + HGETALL, "pcp:map:context.name", redis_reply_type(reply)); +@@ -5198,7 +5213,7 @@ series_lookup_mapping(void *arg) + seriesBatonCheckCount(baton, "series_lookup_mapping"); + seriesBatonReference(baton, "series_lookup_mapping"); + +- key = sdscatfmt(sdsempty(), "pcp:map:%s", redisMapName(baton->u.lookup.map)); ++ key = sdscatfmt(sdsempty(), "pcp:map:%s", redisMapName(baton->lookup.map)); + cmd = redis_command(2); + cmd = redis_param_str(cmd, HGETALL, HGETALL_LEN); + cmd = redis_param_sds(cmd, key); +@@ -5328,9 +5343,9 @@ series_lookup_sources(void *arg) + seriesBatonCheckCount(baton, "series_lookup_sources"); + seriesBatonReference(baton, "series_lookup_sources"); + +- for (i = 0; i < baton->u.lookup.nseries; i++) { ++ for (i = 0; i < baton->lookup.nseries; i++) { + seriesBatonReference(baton, "series_lookup_sources"); +- sid = &baton->u.lookup.series[i]; ++ sid = &baton->lookup.series[i]; + key = sdscatfmt(sdsempty(), "pcp:context.name:source:%S", sid->name); + cmd = redis_command(2); + cmd = redis_param_str(cmd, SMEMBERS, SMEMBERS_LEN); +@@ -5361,7 +5376,7 @@ pmSeriesSources(pmSeriesSettings *settin + initSeriesGetLookup(baton, nsources, sources, settings->callbacks.on_context, contextmap); + + if (nsources == 0) +- return series_map_keys(baton, redisMapName(baton->u.lookup.map)); ++ return series_map_keys(baton, redisMapName(baton->lookup.map)); + + baton->current = &baton->phases[0]; + baton->phases[i++].func = series_lookup_services; +@@ -5384,9 +5399,9 @@ series_lookup_metrics(void *arg) + seriesBatonCheckMagic(baton, MAGIC_QUERY, "series_lookup_metrics"); + seriesBatonCheckCount(baton, "series_lookup_metrics"); + +- for (i = 0; i < baton->u.lookup.nseries; i++) { ++ for (i = 0; i < baton->lookup.nseries; i++) { + seriesBatonReference(baton, "series_lookup_metrics"); +- sid = &baton->u.lookup.series[i]; ++ sid = &baton->lookup.series[i]; + key = sdscatfmt(sdsempty(), "pcp:metric.name:series:%S", sid->name); + cmd = redis_command(2); + cmd = redis_param_str(cmd, SMEMBERS, SMEMBERS_LEN); +@@ -5416,7 +5431,7 @@ pmSeriesMetrics(pmSeriesSettings *settin + initSeriesGetLookup(baton, nseries, series, settings->callbacks.on_metric, namesmap); + + if (nseries == 0) +- return series_map_keys(baton, redisMapName(baton->u.lookup.map)); ++ return series_map_keys(baton, redisMapName(baton->lookup.map)); + + baton->current = &baton->phases[0]; + baton->phases[i++].func = series_lookup_services; +@@ -5517,55 +5532,66 @@ parseseries(seriesQueryBaton *baton, sds + } + + static void +-initSeriesGetValues(seriesQueryBaton *baton, int nseries, sds *series, ++initSeriesGetValues(seriesQueryBaton *baton, int nseries, sds *inseries, + pmSeriesTimeWindow *window) + { +- struct series_set *result = &baton->u.query.root.result; +- struct timing *timing = &baton->u.query.timing; ++ struct node *node = NULL; ++ struct timing timing = {0}; ++ unsigned char *series = NULL; ++ struct series_set *result; + struct timeval offset; + int i; +- +- /* validate and convert 40-byte (ASCII) SIDs to internal 20-byte form */ +- result->nseries = nseries; +- if ((result->series = calloc(nseries, 20)) == NULL) { ++ ++ /* allocate a local parse node, timing and result SIDs */ ++ if (!(node = (node_t *)calloc(1, sizeof(node_t)))) ++ baton->error = -ENOMEM; ++ else if (!(series = calloc(nseries, 20))) /* 20 byte SIDs */ + baton->error = -ENOMEM; +- } else { +- for (i = 0; i < nseries; i++) +- parseseries(baton, series[i], result->series + (i * 20)); +- } + if (baton->error) { +- if (result->series) +- free(result->series); ++ if (series) free(series); ++ if (node) free(node); + return; +- } ++ } ++ ++ /* track this memory in the baton for async free later */ ++ result = &node->result; ++ result->series = series; ++ result->nseries = nseries; ++ ++ /* validate and convert 40-byte (ASCII) SIDs to internal 20-byte form */ ++ for (i = 0; i < nseries; i++) ++ parseseries(baton, inseries[i], result->series + (i * 20)); + + /* validate and convert time window specification to internal struct */ +- timing->window = *window; ++ timing.window = *window; + if (window->delta) +- parsedelta(baton, window->delta, &timing->delta, "delta"); ++ parsedelta(baton, window->delta, &timing.delta, "delta"); + if (window->align) +- parsetime(baton, window->align, &timing->align, "align"); ++ parsetime(baton, window->align, &timing.align, "align"); + if (window->start) +- parsetime(baton, window->start, &timing->start, "start"); ++ parsetime(baton, window->start, &timing.start, "start"); + if (window->end) +- parsetime(baton, window->end, &timing->end, "end"); ++ parsetime(baton, window->end, &timing.end, "end"); + if (window->range) { +- parsedelta(baton, window->range, &timing->start, "range"); ++ parsedelta(baton, window->range, &timing.start, "range"); + gettimeofday(&offset, NULL); +- tsub(&offset, &timing->start); +- timing->start = offset; +- timing->end.tv_sec = INT_MAX; ++ tsub(&offset, &timing.start); ++ timing.start = offset; ++ timing.end.tv_sec = INT_MAX; + } + if (window->count) +- parseuint(baton, window->count, &timing->count, "count"); ++ parseuint(baton, window->count, &timing.count, "count"); + if (window->offset) +- parseuint(baton, window->offset, &timing->offset, "offset"); ++ parseuint(baton, window->offset, &timing.offset, "offset"); + if (window->zone) +- parsezone(baton, window->zone, &timing->zone, "timezone"); ++ parsezone(baton, window->zone, &timing.zone, "timezone"); + + /* if no time window parameters passed, default to latest value */ +- if (!series_time_window(timing)) +- timing->count = 1; ++ if (!series_time_window(&timing)) ++ timing.count = 1; ++ ++ baton->query.timing = timing; /* struct copy */ ++ baton->query.root = node; + } + + int +diff -Naurp pcp-5.3.7.orig/src/libpcp_web/src/schema.h pcp-5.3.7/src/libpcp_web/src/schema.h +--- pcp-5.3.7.orig/src/libpcp_web/src/schema.h 2023-07-05 13:42:25.533986251 +1000 ++++ pcp-5.3.7/src/libpcp_web/src/schema.h 2023-07-05 13:42:53.414025716 +1000 +@@ -1,5 +1,5 @@ + /* +- * Copyright (c) 2017-2021 Red Hat. ++ * Copyright (c) 2017-2022 Red Hat. + * + * This library is free software; you can redistribute it and/or modify it + * under the terms of the GNU Lesser General Public License as published +@@ -126,6 +126,7 @@ typedef struct seriesGetContext { + context_t context; + unsigned long long count; /* number of samples processed */ + pmResult *result; /* currently active sample data */ ++ int loaded; /* end of archive data reached */ + int error; /* PMAPI error code from fetch */ + + redisDoneCallBack done; +@@ -152,6 +153,9 @@ typedef struct seriesLoadBaton { + const char **metrics; /* metric specification strings */ + dict *errors; /* PMIDs where errors observed */ + dict *wanted; /* allowed metrics list PMIDs */ ++ sds *exclude_patterns; /* list of exclude metric patterns (e.g. proc.*) */ ++ unsigned int exclude_npatterns; /* number of exclude metric patterns */ ++ dict *exclude_pmids; /* dict of excluded pmIDs (pmID: NULL) */ + + int error; + void *arg; +diff -Naurp pcp-5.3.7.orig/src/libpcp_web/src/slots.c pcp-5.3.7/src/libpcp_web/src/slots.c +--- pcp-5.3.7.orig/src/libpcp_web/src/slots.c 2023-07-05 13:42:25.533986251 +1000 ++++ pcp-5.3.7/src/libpcp_web/src/slots.c 2023-07-05 13:42:53.414025716 +1000 +@@ -633,8 +633,10 @@ redisSlotsProxyConnect(redisSlots *slots + redisReader *reader = *readerp; + redisReply *reply = NULL; + dictEntry *entry; +- long long position, offset, length; +- sds cmd, key, msg; ++ size_t replyStartPosition; ++ long long position; ++ sds cmd, msg; ++ int hasKey; + int sts; + + if (!reader && +@@ -644,33 +646,45 @@ redisSlotsProxyConnect(redisSlots *slots + return -ENOMEM; + } + +- offset = reader->pos; +- length = sdslen(reader->buf); +- if (redisReaderFeed(reader, buffer, nread) != REDIS_OK || +- redisReaderGetReply(reader, (void **)&reply) != REDIS_OK) { ++ if (redisReaderFeed(reader, buffer, nread) != REDIS_OK) { + infofmt(msg, "failed to parse Redis protocol request"); + info(PMLOG_REQUEST, msg, arg), sdsfree(msg); + return -EPROTO; + } + +- if (reply != NULL) { /* client request is complete */ +- key = cmd = NULL; ++ /* parse all Redis requests contained in buffer (Redis pipelining) */ ++ while (1) { ++ replyStartPosition = reader->pos; ++ sts = redisReaderGetReply(reader, (void **)&reply); ++ if (sts != REDIS_OK) { ++ infofmt(msg, "failed to parse Redis protocol request"); ++ info(PMLOG_REQUEST, msg, arg), sdsfree(msg); ++ return -EPROTO; ++ } ++ if (reply == NULL) { ++ break; ++ } ++ ++ cmd = NULL; ++ hasKey = 0; + if (reply->type == REDIS_REPLY_ARRAY || + reply->type == REDIS_REPLY_MAP || + reply->type == REDIS_REPLY_SET) + cmd = sdsnew(reply->element[0]->str); + if (cmd && (entry = dictFind(slots->keymap, cmd)) != NULL) { + position = dictGetSignedIntegerVal(entry); +- if (position < reply->elements) +- key = sdsnew(reply->element[position]->str); ++ if (position > 0 && position < reply->elements) ++ hasKey = 1; + } + sdsfree(cmd); +- cmd = sdsnewlen(reader->buf + offset, sdslen(reader->buf) - length); +- if (key != NULL && position > 0) ++ ++ cmd = sdsnewlen(reader->buf + replyStartPosition, reader->pos - replyStartPosition); ++ if (hasKey) + sts = redisSlotsRequest(slots, cmd, callback, arg); + else + sts = redisSlotsRequestFirstNode(slots, cmd, callback, arg); +- sdsfree(key); ++ sdsfree(cmd); ++ + if (sts != REDIS_OK) { + redisReply *errorReply = calloc(1, sizeof(redisReply)); + errorReply->type = REDIS_REPLY_ERROR; +diff -Naurp pcp-5.3.7.orig/src/libpcp_web/src/timer.c pcp-5.3.7/src/libpcp_web/src/timer.c +--- pcp-5.3.7.orig/src/libpcp_web/src/timer.c 2023-07-05 13:42:25.533986251 +1000 ++++ pcp-5.3.7/src/libpcp_web/src/timer.c 2023-07-05 13:42:53.414025716 +1000 +@@ -13,6 +13,7 @@ + */ + #include + #include "load.h" ++#include "maps.h" + #include "libpcp.h" + #include "mmv_stats.h" + +@@ -23,6 +24,10 @@ typedef enum server_metric { + SERVER_CPU_TOT, + SERVER_MEM_MAXRSS, + SERVER_MEM_DATASZ, ++ SERVER_MAP_CONTEXT_SIZE, ++ SERVER_MAP_METRIC_SIZE, ++ SERVER_MAP_LABEL_SIZE, ++ SERVER_MAP_INST_SIZE, + NUM_SERVER_METRIC + } server_metric_t; + +@@ -156,6 +161,7 @@ server_metrics_refresh(void *map) + { + double usr, sys; + unsigned long long datasz = 0; ++ unsigned int value; + struct rusage usage = {0}; + + __pmProcessDataSize((unsigned long*) &datasz); +@@ -173,6 +179,16 @@ server_metrics_refresh(void *map) + + /* exported as uint64 but manipulated as ulong/ulong long */ + mmv_set(map, server.metrics[SERVER_MEM_DATASZ], &datasz); ++ ++ /* update global maps size metrics */ ++ value = contextmap? dictSize(contextmap) : 0; ++ mmv_set(map, server.metrics[SERVER_MAP_CONTEXT_SIZE], &value); ++ value = namesmap? dictSize(namesmap) : 0; ++ mmv_set(map, server.metrics[SERVER_MAP_METRIC_SIZE], &value); ++ value = labelsmap? dictSize(labelsmap) : 0; ++ mmv_set(map, server.metrics[SERVER_MAP_LABEL_SIZE], &value); ++ value = instmap? dictSize(instmap) : 0; ++ mmv_set(map, server.metrics[SERVER_MAP_INST_SIZE], &value); + } + + /* +@@ -181,6 +197,7 @@ server_metrics_refresh(void *map) + int + pmWebTimerSetMetricRegistry(struct mmv_registry *registry) + { ++ pmAtomValue **ap; + pmUnits nounits = MMV_UNITS(0,0,0,0,0,0); + pmUnits units_kbytes = MMV_UNITS(1, 0, 0, PM_SPACE_KBYTE, 0, 0); + pmUnits units_msec = MMV_UNITS(0, 1, 0, 0, PM_TIME_MSEC, 0); +@@ -228,18 +245,46 @@ pmWebTimerSetMetricRegistry(struct mmv_r + "virtual data size", + "Process memory virtual data size from sbrk(2)"); + ++ /* ++ * Reverse mapping dict metrics ++ */ ++ mmv_stats_add_metric(registry, "map.context.size", SERVER_MAP_CONTEXT_SIZE, ++ MMV_TYPE_U32, MMV_SEM_INSTANT, nounits, MMV_INDOM_NULL, ++ "context map dictionary size", ++ "Number of entries in the context map dictionary"); ++ ++ mmv_stats_add_metric(registry, "map.metric.size", SERVER_MAP_METRIC_SIZE, ++ MMV_TYPE_U32, MMV_SEM_INSTANT, nounits, MMV_INDOM_NULL, ++ "metric names map dictionary size", ++ "Number of entries in the metric names map dictionary"); ++ ++ mmv_stats_add_metric(registry, "map.label.size", SERVER_MAP_LABEL_SIZE, ++ MMV_TYPE_U32, MMV_SEM_INSTANT, nounits, MMV_INDOM_NULL, ++ "label names map dictionary size", ++ "Number of entries in the labels map dictionary"); ++ ++ mmv_stats_add_metric(registry, "map.instance.size", SERVER_MAP_INST_SIZE, ++ MMV_TYPE_U32, MMV_SEM_INSTANT, nounits, MMV_INDOM_NULL, ++ "instance names map dictionary size", ++ "Number of entries in the instance name map dictionary"); ++ + if ((map = mmv_stats_start(registry)) == NULL) { + pmNotifyErr(LOG_ERR, "%s: server instrumentation disabled", + "pmWebTimerSetMetricRegistry"); + return -EINVAL; + } + +- server.metrics[SERVER_PID] = mmv_lookup_value_desc(map, "pid", NULL); +- server.metrics[SERVER_CPU_USR] = mmv_lookup_value_desc(map, "cpu.user", NULL); +- server.metrics[SERVER_CPU_SYS] = mmv_lookup_value_desc(map, "cpu.sys", NULL); +- server.metrics[SERVER_CPU_TOT] = mmv_lookup_value_desc(map, "cpu.total", NULL); +- server.metrics[SERVER_MEM_MAXRSS] = mmv_lookup_value_desc(map, "mem.maxrss", NULL); +- server.metrics[SERVER_MEM_DATASZ] = mmv_lookup_value_desc(map, "mem.datasz", NULL); ++ ap = server.metrics; ++ ap[SERVER_PID] = mmv_lookup_value_desc(map, "pid", NULL); ++ ap[SERVER_CPU_USR] = mmv_lookup_value_desc(map, "cpu.user", NULL); ++ ap[SERVER_CPU_SYS] = mmv_lookup_value_desc(map, "cpu.sys", NULL); ++ ap[SERVER_CPU_TOT] = mmv_lookup_value_desc(map, "cpu.total", NULL); ++ ap[SERVER_MEM_MAXRSS] = mmv_lookup_value_desc(map, "mem.maxrss", NULL); ++ ap[SERVER_MEM_DATASZ] = mmv_lookup_value_desc(map, "mem.datasz", NULL); ++ ap[SERVER_MAP_CONTEXT_SIZE] = mmv_lookup_value_desc(map, "map.context.size", NULL); ++ ap[SERVER_MAP_METRIC_SIZE] = mmv_lookup_value_desc(map, "map.metric.size", NULL); ++ ap[SERVER_MAP_LABEL_SIZE] = mmv_lookup_value_desc(map, "map.label.size", NULL); ++ ap[SERVER_MAP_INST_SIZE] = mmv_lookup_value_desc(map, "map.instance.size", NULL); + + /* PID doesn't change, set it once */ + mmv_set(map, server.metrics[SERVER_PID], &pid); +diff -Naurp pcp-5.3.7.orig/src/libpcp_web/src/webgroup.c pcp-5.3.7/src/libpcp_web/src/webgroup.c +--- pcp-5.3.7.orig/src/libpcp_web/src/webgroup.c 2023-07-05 13:42:25.533986251 +1000 ++++ pcp-5.3.7/src/libpcp_web/src/webgroup.c 2023-07-05 13:42:53.414025716 +1000 +@@ -44,10 +44,8 @@ enum matches { MATCH_EXACT, MATCH_GLOB, + enum profile { PROFILE_ADD, PROFILE_DEL }; + + enum webgroup_metric { +- CONTEXT_MAP_SIZE, +- NAMES_MAP_SIZE, +- LABELS_MAP_SIZE, +- INST_MAP_SIZE, ++ WEBGROUP_GC_COUNT, ++ WEBGROUP_GC_DROPS, + NUM_WEBGROUP_METRIC + }; + +@@ -64,7 +62,6 @@ typedef struct webgroups { + uv_mutex_t mutex; + + unsigned int active; +- int timerid; + } webgroups; + + static struct webgroups * +@@ -76,7 +73,6 @@ webgroups_lookup(pmWebGroupModule *modul + module->privdata = calloc(1, sizeof(struct webgroups)); + groups = (struct webgroups *)module->privdata; + uv_mutex_init(&groups->mutex); +- groups->timerid = -1; + } + return groups; + } +@@ -292,8 +288,6 @@ webgroup_timers_stop(struct webgroups *g + if (groups->active) { + uv_timer_stop(&groups->timer); + uv_close((uv_handle_t *)&groups->timer, NULL); +- pmWebTimerRelease(groups->timerid); +- groups->timerid = -1; + groups->active = 0; + } + } +@@ -336,28 +330,15 @@ webgroup_garbage_collect(struct webgroup + uv_mutex_unlock(&groups->mutex); + } + ++ mmv_set(groups->map, groups->metrics[WEBGROUP_GC_DROPS], &drops); ++ mmv_set(groups->map, groups->metrics[WEBGROUP_GC_COUNT], &count); ++ + if (pmDebugOptions.http || pmDebugOptions.libweb) + fprintf(stderr, "%s: finished [%u drops from %u entries]\n", + "webgroup_garbage_collect", drops, count); + } + + static void +-refresh_maps_metrics(void *data) +-{ +- struct webgroups *groups = (struct webgroups *)data; +- unsigned int value; +- +- value = contextmap? dictSize(contextmap) : 0; +- mmv_set(groups->map, groups->metrics[CONTEXT_MAP_SIZE], &value); +- value = namesmap? dictSize(namesmap) : 0; +- mmv_set(groups->map, groups->metrics[NAMES_MAP_SIZE], &value); +- value = labelsmap? dictSize(labelsmap) : 0; +- mmv_set(groups->map, groups->metrics[LABELS_MAP_SIZE], &value); +- value = instmap? dictSize(instmap) : 0; +- mmv_set(groups->map, groups->metrics[INST_MAP_SIZE], &value); +-} +- +-static void + webgroup_worker(uv_timer_t *arg) + { + uv_handle_t *handle = (uv_handle_t *)arg; +@@ -425,8 +406,6 @@ webgroup_lookup_context(pmWebGroupSettin + groups->timer.data = (void *)groups; + uv_timer_start(&groups->timer, webgroup_worker, + default_worker, default_worker); +- /* timer for map stats refresh */ +- groups->timerid = pmWebTimerRegister(refresh_maps_metrics, groups); + } + + if (*id == NULL) { +@@ -2360,36 +2339,21 @@ pmWebGroupSetupMetrics(pmWebGroupModule + if (groups == NULL || groups->registry == NULL) + return; /* no metric registry has been set up */ + +- /* +- * Reverse mapping dict metrics +- */ +- mmv_stats_add_metric(groups->registry, "contextmap.size", 1, +- MMV_TYPE_U32, MMV_SEM_INSTANT, nounits, MMV_INDOM_NULL, +- "context map dictionary size", +- "Number of entries in the context map dictionary"); +- +- mmv_stats_add_metric(groups->registry, "namesmap.size", 2, +- MMV_TYPE_U32, MMV_SEM_INSTANT, nounits, MMV_INDOM_NULL, +- "metric names map dictionary size", +- "Number of entries in the metric names map dictionary"); +- +- mmv_stats_add_metric(groups->registry, "labelsmap.size", 3, ++ mmv_stats_add_metric(groups->registry, "gc.context.scans", 1, + MMV_TYPE_U32, MMV_SEM_INSTANT, nounits, MMV_INDOM_NULL, +- "labels map dictionary size", +- "Number of entries in the labels map dictionary"); ++ "contexts scanned in last garbage collection", ++ "Contexts scanned during most recent webgroup garbage collection"); + +- mmv_stats_add_metric(groups->registry, "instmap.size", 4, ++ mmv_stats_add_metric(groups->registry, "gc.context.drops", 2, + MMV_TYPE_U32, MMV_SEM_INSTANT, nounits, MMV_INDOM_NULL, +- "instance name map dictionary size", +- "Number of entries in the instance name map dictionary"); ++ "contexts dropped in last garbage collection", ++ "Contexts dropped during most recent webgroup garbage collection"); + + groups->map = map = mmv_stats_start(groups->registry); + + ap = groups->metrics; +- ap[CONTEXT_MAP_SIZE] = mmv_lookup_value_desc(map, "contextmap.size", NULL); +- ap[NAMES_MAP_SIZE] = mmv_lookup_value_desc(map, "namesmap.size", NULL); +- ap[LABELS_MAP_SIZE] = mmv_lookup_value_desc(map, "labelsmap.size", NULL); +- ap[INST_MAP_SIZE] = mmv_lookup_value_desc(map, "instmap.size", NULL); ++ ap[WEBGROUP_GC_DROPS] = mmv_lookup_value_desc(map, "gc.context.scans", NULL); ++ ap[WEBGROUP_GC_COUNT] = mmv_lookup_value_desc(map, "gc.context.drops", NULL); + } + + +diff -Naurp pcp-5.3.7.orig/src/pmproxy/src/http.c pcp-5.3.7/src/pmproxy/src/http.c +--- pcp-5.3.7.orig/src/pmproxy/src/http.c 2022-04-05 09:05:43.000000000 +1000 ++++ pcp-5.3.7/src/pmproxy/src/http.c 2023-07-05 13:43:00.414035625 +1000 +@@ -581,7 +581,7 @@ http_add_parameter(dict *parameters, + return 0; + } + +-static int ++int + http_parameters(const char *url, size_t length, dict **parameters) + { + const char *end = url + length; +diff -Naurp pcp-5.3.7.orig/src/pmproxy/src/http.h pcp-5.3.7/src/pmproxy/src/http.h +--- pcp-5.3.7.orig/src/pmproxy/src/http.h 2022-04-05 09:05:43.000000000 +1000 ++++ pcp-5.3.7/src/pmproxy/src/http.h 2023-07-05 13:43:00.414035625 +1000 +@@ -64,6 +64,7 @@ extern void http_transfer(struct client + extern void http_reply(struct client *, sds, http_code_t, http_flags_t, http_options_t); + extern void http_error(struct client *, http_code_t, const char *); + ++extern int http_parameters(const char *, size_t, dict **); + extern int http_decode(const char *, size_t, sds); + extern const char *http_status_mapping(http_code_t); + extern const char *http_content_type(http_flags_t); +diff -Naurp pcp-5.3.7.orig/src/pmproxy/src/search.c pcp-5.3.7/src/pmproxy/src/search.c +--- pcp-5.3.7.orig/src/pmproxy/src/search.c 2023-07-05 13:42:25.533986251 +1000 ++++ pcp-5.3.7/src/pmproxy/src/search.c 2023-07-05 13:42:53.414025716 +1000 +@@ -1,5 +1,5 @@ + /* +- * Copyright (c) 2020 Red Hat. ++ * Copyright (c) 2020,2022 Red Hat. + * + * This program is free software; you can redistribute it and/or modify it + * under the terms of the GNU Lesser General Public License as published +@@ -253,6 +253,9 @@ on_pmsearch_done(int status, void *arg) + flags |= HTTP_FLAG_JSON; + } + http_reply(client, msg, code, flags, options); ++ ++ /* release lock of pmsearch_request_done */ ++ client_put(client); + } + + static void +@@ -481,6 +484,9 @@ pmsearch_request_done(struct client *cli + pmSearchBaton *baton = (pmSearchBaton *)client->u.http.data; + int sts; + ++ /* reference to prevent freeing while waiting for a Redis reply callback */ ++ client_get(client); ++ + if (client->u.http.parser.status_code) { + on_pmsearch_done(-EINVAL, baton); + return 1; +diff -Naurp pcp-5.3.7.orig/src/pmproxy/src/series.c pcp-5.3.7/src/pmproxy/src/series.c +--- pcp-5.3.7.orig/src/pmproxy/src/series.c 2023-07-05 13:42:25.533986251 +1000 ++++ pcp-5.3.7/src/pmproxy/src/series.c 2023-07-05 13:43:00.414035625 +1000 +@@ -1,5 +1,5 @@ + /* +- * Copyright (c) 2019-2020 Red Hat. ++ * Copyright (c) 2019-2020,2022 Red Hat. + * + * This program is free software; you can redistribute it and/or modify it + * under the terms of the GNU Lesser General Public License as published +@@ -12,6 +12,7 @@ + * License for more details. + */ + #include "server.h" ++#include "util.h" + #include + + typedef enum pmSeriesRestKey { +@@ -592,6 +593,9 @@ on_pmseries_done(int status, void *arg) + flags |= HTTP_FLAG_JSON; + } + http_reply(client, msg, code, flags, options); ++ ++ /* release lock of pmseries_request_done */ ++ client_put(client); + } + + static void +@@ -824,30 +828,26 @@ static int + pmseries_request_body(struct client *client, const char *content, size_t length) + { + pmSeriesBaton *baton = (pmSeriesBaton *)client->u.http.data; +- sds series; + + if (pmDebugOptions.http) + fprintf(stderr, "series servlet body (client=%p)\n", client); + +- if (client->u.http.parser.method != HTTP_POST) ++ if (client->u.http.parser.method != HTTP_POST || client->u.http.parameters != NULL) + return 0; + + switch (baton->restkey) { + case RESTKEY_LOAD: + case RESTKEY_QUERY: +- sdsfree(baton->query); +- baton->query = sdsnewlen(content, length); +- break; +- + case RESTKEY_DESC: + case RESTKEY_INSTS: + case RESTKEY_LABELS: + case RESTKEY_METRIC: + case RESTKEY_SOURCE: + case RESTKEY_VALUES: +- series = sdsnewlen(content, length); +- baton->sids = sdssplitlen(series, length, "\n", 1, &baton->nsids); +- sdsfree(series); ++ /* parse URL encoded parameters in the request body */ ++ /* in the same way as the URL query string */ ++ http_parameters(content, length, &client->u.http.parameters); ++ pmseries_setup_request_parameters(client, baton, client->u.http.parameters); + break; + + default: +@@ -890,10 +890,16 @@ pmseries_request_load(struct client *cli + message = sdsnewlen(failed, sizeof(failed) - 1); + http_reply(client, message, HTTP_STATUS_BAD_REQUEST, + HTTP_FLAG_JSON, baton->options); ++ ++ /* release lock of pmseries_request_done */ ++ client_put(client); + } else if (baton->working) { + message = sdsnewlen(loading, sizeof(loading) - 1); + http_reply(client, message, HTTP_STATUS_CONFLICT, + HTTP_FLAG_JSON, baton->options); ++ ++ /* release lock of pmseries_request_done */ ++ client_put(client); + } else { + baton->loading.data = baton; + uv_queue_work(client->proxy->events, &baton->loading, +@@ -907,6 +913,9 @@ pmseries_request_done(struct client *cli + pmSeriesBaton *baton = (pmSeriesBaton *)client->u.http.data; + int sts; + ++ /* reference to prevent freeing while waiting for a Redis reply callback */ ++ client_get(client); ++ + if (client->u.http.parser.status_code) { + on_pmseries_done(-EINVAL, baton); + return 1; +diff -Naurp pcp-5.3.7.orig/src/pmproxy/src/server.c pcp-5.3.7/src/pmproxy/src/server.c +--- pcp-5.3.7.orig/src/pmproxy/src/server.c 2023-07-05 13:42:25.533986251 +1000 ++++ pcp-5.3.7/src/pmproxy/src/server.c 2023-07-05 13:42:53.424025730 +1000 +@@ -180,7 +180,9 @@ signal_init(struct proxy *proxy) + { + uv_loop_t *loop = proxy->events; + ++#if defined(HAVE_SIGPIPE) + signal(SIGPIPE, SIG_IGN); ++#endif + + uv_signal_init(loop, &sighup); + uv_signal_init(loop, &sigint); +@@ -310,6 +312,20 @@ on_write_callback(uv_callback_t *handle, + struct client *client = (struct client *)request->writer.data; + int sts; + ++ /* ++ * client_write() checks if the client is opened, and calls ++ * uv_callback_fire(&proxy->write_callbacks, ...). ++ * In a later loop iteration, on_write_callback() is called and tries ++ * to write to the client. However, the client can be closed between ++ * the call to uv_callback_fire() and the actual on_write_callback() ++ * callback. Therefore we need to check this condition again. ++ */ ++ if (client_is_closed(client)) { ++ /* release lock of client_write */ ++ client_put(client); ++ return 0; ++ } ++ + (void)handle; + if (pmDebugOptions.af) + fprintf(stderr, "%s: client=%p\n", "on_write_callback", client); +@@ -325,6 +341,9 @@ on_write_callback(uv_callback_t *handle, + } + } else + secure_client_write(client, request); ++ ++ /* release lock of client_write */ ++ client_put(client); + return 0; + } + +@@ -353,6 +372,8 @@ client_write(struct client *client, sds + request->writer.data = client; + request->callback = on_client_write; + ++ /* client must not get freed while waiting for the write callback to fire */ ++ client_get(client); + uv_callback_fire(&proxy->write_callbacks, request, NULL); + } else { + client_close(client); diff --git a/SOURCES/redhat-bugzilla-2211263-pmcd-conf-rewrite.patch b/SOURCES/redhat-bugzilla-2211263-pmcd-conf-rewrite.patch new file mode 100644 index 0000000..4881c52 --- /dev/null +++ b/SOURCES/redhat-bugzilla-2211263-pmcd-conf-rewrite.patch @@ -0,0 +1,94 @@ +diff -Naurp pcp-5.3.7.orig/src/pmdas/bcc/Upgrade pcp-5.3.7/src/pmdas/bcc/Upgrade +--- pcp-5.3.7.orig/src/pmdas/bcc/Upgrade 2021-08-16 14:12:25.000000000 +1000 ++++ pcp-5.3.7/src/pmdas/bcc/Upgrade 2023-07-10 16:25:31.904767032 +1000 +@@ -26,8 +26,11 @@ then + else + sed -i -e "s,^\(bcc.*binary\),\1 notready,g" $PCP_PMCDCONF_PATH + fi +- sed -i \ +- -e "s,python $PCP_PMDAS_DIR/bcc/,$PCP_PYTHON_PROG $PCP_PMDAS_DIR/bcc/,g" \ +- $PCP_PMCDCONF_PATH 2>/dev/null ++ if grep -q '^bcc.*python ' "$PCP_PMCDCONF_PATH" 2>/dev/null ++ then ++ sed -i \ ++ -e "s,python $PCP_PMDAS_DIR/bcc/,$PCP_PYTHON_PROG $PCP_PMDAS_DIR/bcc/,g" \ ++ $PCP_PMCDCONF_PATH 2>/dev/null ++ fi + fi + exit 0 +diff -Naurp pcp-5.3.7.orig/src/pmdas/kvm/Upgrade pcp-5.3.7/src/pmdas/kvm/Upgrade +--- pcp-5.3.7.orig/src/pmdas/kvm/Upgrade 2019-02-06 17:16:29.000000000 +1100 ++++ pcp-5.3.7/src/pmdas/kvm/Upgrade 2023-07-10 16:25:31.904767032 +1000 +@@ -17,7 +17,7 @@ + + . $PCP_DIR/etc/pcp.env + +-if grep -q ^kvm "$PCP_PMCDCONF_PATH" 2>/dev/null ++if grep -q '^kvm.*perl.*' "$PCP_PMCDCONF_PATH" 2>/dev/null + then + sed -i -e "s,perl $PCP_PMDAS_DIR/kvm/pmdakvm.pl,$PCP_PMDAS_DIR/kvm/pmdakvm -d 95,g" $PCP_PMCDCONF_PATH 2>/dev/null + fi +diff -Naurp pcp-5.3.7.orig/src/pmdas/mssql/Upgrade pcp-5.3.7/src/pmdas/mssql/Upgrade +--- pcp-5.3.7.orig/src/pmdas/mssql/Upgrade 2021-08-16 14:12:25.000000000 +1000 ++++ pcp-5.3.7/src/pmdas/mssql/Upgrade 2023-07-10 16:25:31.904767032 +1000 +@@ -17,14 +17,20 @@ + + . $PCP_DIR/etc/pcp.env + +-if grep -q ^mssql "$PCP_PMCDCONF_PATH" 2>/dev/null ++if grep -q '^mssql.*perl ' "$PCP_PMCDCONF_PATH" 2>/dev/null + then + sed -i \ +- -e "s,python $PCP_PMDAS_DIR/mssql/,$PCP_PYTHON_PROG $PCP_PMDAS_DIR/mssql/,g" \ + -e "s,perl $PCP_PMDAS_DIR/mssql/pmdamssql.pl,$PCP_PYTHON_PROG $PCP_PMDAS_DIR/mssql/pmdamssql.python,g" \ + $PCP_PMCDCONF_PATH 2>/dev/null + fi + ++if grep -q '^mssql.*python ' "$PCP_PMCDCONF_PATH" 2>/dev/null ++then ++ sed -i \ ++ -e "s,python $PCP_PMDAS_DIR/mssql/,$PCP_PYTHON_PROG $PCP_PMDAS_DIR/mssql/,g" \ ++ $PCP_PMCDCONF_PATH 2>/dev/null ++fi ++ + perlpath=`which $PCP_PERL_PROG` + original="$PCP_PMDAS_DIR/mssql/mssql.conf" + upgraded="$PCP_PMDAS_DIR/mssql/mssql.conf.tmp" +diff -Naurp pcp-5.3.7.orig/src/pmdas/openmetrics/Upgrade pcp-5.3.7/src/pmdas/openmetrics/Upgrade +--- pcp-5.3.7.orig/src/pmdas/openmetrics/Upgrade 2021-08-16 14:12:25.000000000 +1000 ++++ pcp-5.3.7/src/pmdas/openmetrics/Upgrade 2023-07-10 16:25:31.904767032 +1000 +@@ -36,7 +36,7 @@ then + rm -f "$PCP_VAR_DIR/pmns/prometheus" 2>/dev/null + fi + +-if grep -q ^openmetrics "$PCP_PMCDCONF_PATH" 2>/dev/null ++if grep -q '^openmetrics.*python ' "$PCP_PMCDCONF_PATH" 2>/dev/null + then + sed -i -e "s,python $PCP_PMDAS_DIR/openmetrics/,$PCP_PYTHON_PROG $PCP_PMDAS_DIR/openmetrics/,g" $PCP_PMCDCONF_PATH 2>/dev/null + fi +diff -Naurp pcp-5.3.7.orig/src/pmdas/postgresql/Upgrade pcp-5.3.7/src/pmdas/postgresql/Upgrade +--- pcp-5.3.7.orig/src/pmdas/postgresql/Upgrade 2021-08-16 14:12:25.000000000 +1000 ++++ pcp-5.3.7/src/pmdas/postgresql/Upgrade 2023-07-10 16:25:31.914767070 +1000 +@@ -17,14 +17,20 @@ + + . $PCP_DIR/etc/pcp.env + +-if grep -q ^postgresql "$PCP_PMCDCONF_PATH" 2>/dev/null ++if grep -q '^postgresql.*perl ' "$PCP_PMCDCONF_PATH" 2>/dev/null + then + sed -i \ +- -e "s,python $PCP_PMDAS_DIR/postgresql/,$PCP_PYTHON_PROG $PCP_PMDAS_DIR/postgresql/,g" \ + -e "s,perl $PCP_PMDAS_DIR/postgresql/pmdapostgresql.pl,$PCP_PYTHON_PROG $PCP_PMDAS_DIR/postgresql/pmdapostgresql.python,g" \ + $PCP_PMCDCONF_PATH 2>/dev/null + fi + ++if grep -q '^postgresql.*python ' "$PCP_PMCDCONF_PATH" 2>/dev/null ++then ++ sed -i \ ++ -e "s,python $PCP_PMDAS_DIR/postgresql/,$PCP_PYTHON_PROG $PCP_PMDAS_DIR/postgresql/,g" \ ++ $PCP_PMCDCONF_PATH 2>/dev/null ++fi ++ + perlpath=`which $PCP_PERL_PROG` + original="$PCP_PMDAS_DIR/postgresql/postgresql.conf" + upgraded="$PCP_PMDAS_DIR/postgresql/pmdapostgresql.conf" diff --git a/SPECS/pcp.spec b/SPECS/pcp.spec index 14b0924..986ba4f 100644 --- a/SPECS/pcp.spec +++ b/SPECS/pcp.spec @@ -1,6 +1,6 @@ Name: pcp Version: 5.3.7 -Release: 16%{?dist} +Release: 18%{?dist} Summary: System-level performance monitoring and performance management License: GPLv2+ and LGPLv2+ and CC-BY URL: https://pcp.io @@ -18,7 +18,13 @@ Patch7: redhat-bugzilla-2093751-sudoers-docs.patch Patch8: redhat-bugzilla-2101574-farm-config.patch Patch9: redhat-bugzilla-2135314-pmfind-fix.patch Patch10: redhat-bugzilla-2139012-pmdasnmp-config.patch -Patch11: redhat-build-jsonsl.patch +Patch11: redhat-bugzilla-1981886-pcp-ss-fetchgroup.patch +Patch12: redhat-bugzilla-2159207-pmproxy-rollup-fixes.patch +Patch13: redhat-bugzilla-2139325-openmetrics-in-grafana.patch +Patch14: redhat-bugzilla-2150889-nfsclient-srcport.patch +Patch15: redhat-bugzilla-2219731-hacluster-metrics.patch +Patch16: redhat-bugzilla-2211263-pmcd-conf-rewrite.patch +Patch17: redhat-build-jsonsl.patch # The additional linker flags break out-of-tree PMDAs. # https://bugzilla.redhat.com/show_bug.cgi?id=2043092 @@ -2284,7 +2290,6 @@ interface rules, type enforcement and file context adjustments for an updated policy package. %endif - %prep %setup -q %patch0 -p1 @@ -2299,6 +2304,12 @@ updated policy package. %patch9 -p1 %patch10 -p1 %patch11 -p1 +%patch12 -p1 +%patch13 -p1 +%patch14 -p1 +%patch15 -p1 +%patch16 -p1 +%patch17 -p1 %build # the buildsubdir macro gets defined in %setup and is apparently only available in the next step (i.e. the %build step) @@ -3370,6 +3381,15 @@ fi %files zeroconf -f pcp-zeroconf-files.rpm %changelog +* Wed Jul 05 2023 Nathan Scott - 5.3.7-18 +- Improve pmproxy handling large HTTP requests (BZ 2159207) +- Fix hacluster metrics with current Pacemaker (BZ 2219731) +- Ensure pmcd.conf not needlessly over written (BZ 2211263) + +* Thu Mar 09 2023 Nathan Scott - 5.3.7-17 +- Harden pmdaopenmetrics metric name validator (BZ 2139325) +- Fix issues in pmdanfsclient srcport handling (BZ 2150889) + * Thu Nov 17 2022 Nathan Scott - 5.3.7-16 - Ensure SNMP metrics config symlink installed (BZ 2139012)