rpms/pcp
7
0
Fork 0
Code Issues Pull Requests Releases Activity
eaeb816b4a
pcp/SOURCES/redhat-bugzilla-2159207-pmproxy-rollup-fixes.patch

2712 lines
92 KiB
Diff
Raw Normal View History

import CS pcp-5.3.7-20.el8
2024-05-22 13:47:50 +00:00
diff -Naurp pcp-5.3.7.orig/man/man3/pmwebapi.3 pcp-5.3.7/man/man3/pmwebapi.3
--- pcp-5.3.7.orig/man/man3/pmwebapi.3 2022-04-05 09:05:43.000000000 +1000
+++ pcp-5.3.7/man/man3/pmwebapi.3 2023-07-05 13:43:00.404035611 +1000
@@ -175,6 +175,10 @@ parameter.
The value passed in the request will be sent back in the
response \- all responses will be in JSON object form in
this case, with top level "client" and "result" fields.
+.PP
+REST API clients can optionally submit an URL-encoded query string
+in the body of the HTTP request unless otherwise noted.
+In this case the POST method must be used instead of the GET method.
.SS GET \fI/series/query\fR \- \fBpmSeriesQuery\fR(3)
.TS
box;
diff -Naurp pcp-5.3.7.orig/qa/1604 pcp-5.3.7/qa/1604
--- pcp-5.3.7.orig/qa/1604 1970-01-01 10:00:00.000000000 +1000
+++ pcp-5.3.7/qa/1604 2023-07-05 13:42:53.394025688 +1000
@@ -0,0 +1,114 @@
+#!/bin/sh
+# PCP QA Test No. 1604
+# Exercise pmproxy REST API /series/values endpoint using curl(1).
+#
+# Copyright (c) 2022 Red Hat.
+#
+
+seq=`basename $0`
+echo "QA output created by $seq"
+
+# get standard environment, filters and checks
+. ./common.product
+. ./common.filter
+. ./common.check
+
+_check_series
+which jq >/dev/null 2>&1 || _notrun "jq not installed"
+
+_cleanup()
+{
+ cd $here
+ [ -n "$pmproxy_pid" ] && $signal -s TERM $pmproxy_pid
+ [ -n "$options" ] && redis-cli $options shutdown
+ if $need_restore
+ then
+ need_restore=false
+ _restore_config $PCP_SYSCONF_DIR/pmproxy
+ fi
+ $sudo rm -rf $tmp $tmp.*
+}
+
+status=1 # failure is the default!
+signal=$PCP_BINADM_DIR/pmsignal
+
+userid=`id -u`
+username=`id -u -n`
+hostname=`hostname`
+machineid=`_machine_id`
+domainname=`_domain_name`
+
+need_restore=false
+$sudo rm -rf $tmp $tmp.* $seq.full
+trap "_cleanup; exit \$status" 0 1 2 3 15
+
+_filter_source()
+{
+ sed \
+ -e "s,$here,PATH,g" \
+ -e "s,$hostname,QAHOST,g" \
+ #end
+}
+
+_format_timestamps()
+{
+ jq '.[].timestamp |= ((. / 1000 | strftime("%Y-%m-%d %H:%M:%S")) + "." + (. * 1000 % 1000000 | tostring))'
+}
+
+# real QA test starts here
+_save_config $PCP_SYSCONF_DIR/pmproxy
+$sudo rm -f $PCP_SYSCONF_DIR/pmproxy/*
+need_restore=true
+
+echo "Start test Redis server ..."
+redisport=`_find_free_port`
+options="-p $redisport"
+redis-server --port $redisport --save "" > $tmp.redis 2>&1 &
+_check_redis_ping $redisport
+_check_redis_server $redisport
+echo
+
+_check_redis_server_version $redisport
+
+# import some well-known test data into Redis
+pmseries $options --load "$here/archives/bozo-disk" | _filter_source
+
+# start pmproxy
+proxyport=`_find_free_port`
+proxyopts="-p $proxyport -r $redisport -t" # -Dseries,http,af
+pmproxy -f -U $username -x $seq.full -l $tmp.pmproxy.log $proxyopts &
+pmproxy_pid=$!
+
+# check pmproxy has started and is available for requests
+pmcd_wait -h localhost@localhost:$proxyport -v -t 5sec
+
+series1=`pmseries $options disk.all.read`
+[ -z "$series1" ] && _fail "Cannot find any timeseries matching disk.all.read"
+echo "Using series $series1 for disk.all.read"
+
+
+echo "== no interval" | tee -a $seq.full
+url="http://localhost:$proxyport/series/values?series=$series1&start=1489620673&finish=1489620793"
+echo "$url" >> $seq.full
+curl --get --silent "$url" | tee -a $seq.full | _format_timestamps
+
+echo "== 10s interval" | tee -a $seq.full
+url="http://localhost:$proxyport/series/values?series=$series1&start=1489620673&finish=1489620793&interval=10"
+echo "$url" >> $seq.full
+curl --get --silent "$url" | tee -a $seq.full | _format_timestamps
+
+echo "== 20s interval" | tee -a $seq.full
+url="http://localhost:$proxyport/series/values?series=$series1&start=1489620673&finish=1489620793&interval=20"
+echo "$url" >> $seq.full
+curl --get --silent "$url" | tee -a $seq.full | _format_timestamps
+cat $tmp.pmproxy.log >> $seq.full
+
+echo "== 20s interval, starting 2m before first sample in archive" | tee -a $seq.full
+url="http://localhost:$proxyport/series/values?series=$series1&start=1489620553&finish=1489620793&interval=20"
+echo "$url" >> $seq.full
+curl --get --silent "$url" | tee -a $seq.full | _format_timestamps
+
+
+cat $tmp.pmproxy.log >> $seq.full
+status=0
+exit
diff -Naurp pcp-5.3.7.orig/qa/1604.out pcp-5.3.7/qa/1604.out
--- pcp-5.3.7.orig/qa/1604.out 1970-01-01 10:00:00.000000000 +1000
+++ pcp-5.3.7/qa/1604.out 2023-07-05 13:42:53.394025688 +1000
@@ -0,0 +1,204 @@
+QA output created by 1604
+Start test Redis server ...
+PING
+PONG
+
+pmseries: [Info] processed 21 archive records from PATH/archives/bozo-disk
+Using series c61812b8ed3def24fa1df3fbf8d19a96cea3e0f9 for disk.all.read
+== no interval
+[
+ {
+ "series": "c61812b8ed3def24fa1df3fbf8d19a96cea3e0f9",
+ "timestamp": "2017-03-15 23:31:13.890965",
+ "value": "1537640"
+ },
+ {
+ "series": "c61812b8ed3def24fa1df3fbf8d19a96cea3e0f9",
+ "timestamp": "2017-03-15 23:31:23.891401",
+ "value": "1538109"
+ },
+ {
+ "series": "c61812b8ed3def24fa1df3fbf8d19a96cea3e0f9",
+ "timestamp": "2017-03-15 23:31:33.891167",
+ "value": "1538453"
+ },
+ {
+ "series": "c61812b8ed3def24fa1df3fbf8d19a96cea3e0f9",
+ "timestamp": "2017-03-15 23:31:43.891451",
+ "value": "1538888"
+ },
+ {
+ "series": "c61812b8ed3def24fa1df3fbf8d19a96cea3e0f9",
+ "timestamp": "2017-03-15 23:31:53.891930",
+ "value": "1546137"
+ },
+ {
+ "series": "c61812b8ed3def24fa1df3fbf8d19a96cea3e0f9",
+ "timestamp": "2017-03-15 23:32:03.891452",
+ "value": "1552940"
+ },
+ {
+ "series": "c61812b8ed3def24fa1df3fbf8d19a96cea3e0f9",
+ "timestamp": "2017-03-15 23:32:13.891363",
+ "value": "1563099"
+ },
+ {
+ "series": "c61812b8ed3def24fa1df3fbf8d19a96cea3e0f9",
+ "timestamp": "2017-03-15 23:32:23.891335",
+ "value": "1572878"
+ },
+ {
+ "series": "c61812b8ed3def24fa1df3fbf8d19a96cea3e0f9",
+ "timestamp": "2017-03-15 23:32:33.891427",
+ "value": "1581847"
+ },
+ {
+ "series": "c61812b8ed3def24fa1df3fbf8d19a96cea3e0f9",
+ "timestamp": "2017-03-15 23:32:43.891381",
+ "value": "1592546"
+ },
+ {
+ "series": "c61812b8ed3def24fa1df3fbf8d19a96cea3e0f9",
+ "timestamp": "2017-03-15 23:32:53.891394",
+ "value": "1598167"
+ },
+ {
+ "series": "c61812b8ed3def24fa1df3fbf8d19a96cea3e0f9",
+ "timestamp": "2017-03-15 23:33:03.891344",
+ "value": "1598172"
+ }
+]
+== 10s interval
+[
+ {
+ "series": "c61812b8ed3def24fa1df3fbf8d19a96cea3e0f9",
+ "timestamp": "2017-03-15 23:31:13.890965",
+ "value": "1537640"
+ },
+ {
+ "series": "c61812b8ed3def24fa1df3fbf8d19a96cea3e0f9",
+ "timestamp": "2017-03-15 23:31:23.891401",
+ "value": "1538109"
+ },
+ {
+ "series": "c61812b8ed3def24fa1df3fbf8d19a96cea3e0f9",
+ "timestamp": "2017-03-15 23:31:33.891167",
+ "value": "1538453"
+ },
+ {
+ "series": "c61812b8ed3def24fa1df3fbf8d19a96cea3e0f9",
+ "timestamp": "2017-03-15 23:31:43.891451",
+ "value": "1538888"
+ },
+ {
+ "series": "c61812b8ed3def24fa1df3fbf8d19a96cea3e0f9",
+ "timestamp": "2017-03-15 23:31:53.891930",
+ "value": "1546137"
+ },
+ {
+ "series": "c61812b8ed3def24fa1df3fbf8d19a96cea3e0f9",
+ "timestamp": "2017-03-15 23:32:03.891452",
+ "value": "1552940"
+ },
+ {
+ "series": "c61812b8ed3def24fa1df3fbf8d19a96cea3e0f9",
+ "timestamp": "2017-03-15 23:32:13.891363",
+ "value": "1563099"
+ },
+ {
+ "series": "c61812b8ed3def24fa1df3fbf8d19a96cea3e0f9",
+ "timestamp": "2017-03-15 23:32:23.891335",
+ "value": "1572878"
+ },
+ {
+ "series": "c61812b8ed3def24fa1df3fbf8d19a96cea3e0f9",
+ "timestamp": "2017-03-15 23:32:33.891427",
+ "value": "1581847"
+ },
+ {
+ "series": "c61812b8ed3def24fa1df3fbf8d19a96cea3e0f9",
+ "timestamp": "2017-03-15 23:32:43.891381",
+ "value": "1592546"
+ },
+ {
+ "series": "c61812b8ed3def24fa1df3fbf8d19a96cea3e0f9",
+ "timestamp": "2017-03-15 23:32:53.891394",
+ "value": "1598167"
+ },
+ {
+ "series": "c61812b8ed3def24fa1df3fbf8d19a96cea3e0f9",
+ "timestamp": "2017-03-15 23:33:03.891344",
+ "value": "1598172"
+ }
+]
+== 20s interval
+[
+ {
+ "series": "c61812b8ed3def24fa1df3fbf8d19a96cea3e0f9",
+ "timestamp": "2017-03-15 23:31:13.890965",
+ "value": "1537640"
+ },
+ {
+ "series": "c61812b8ed3def24fa1df3fbf8d19a96cea3e0f9",
+ "timestamp": "2017-03-15 23:31:23.891401",
+ "value": "1538109"
+ },
+ {
+ "series": "c61812b8ed3def24fa1df3fbf8d19a96cea3e0f9",
+ "timestamp": "2017-03-15 23:31:43.891451",
+ "value": "1538888"
+ },
+ {
+ "series": "c61812b8ed3def24fa1df3fbf8d19a96cea3e0f9",
+ "timestamp": "2017-03-15 23:32:03.891452",
+ "value": "1552940"
+ },
+ {
+ "series": "c61812b8ed3def24fa1df3fbf8d19a96cea3e0f9",
+ "timestamp": "2017-03-15 23:32:23.891335",
+ "value": "1572878"
+ },
+ {
+ "series": "c61812b8ed3def24fa1df3fbf8d19a96cea3e0f9",
+ "timestamp": "2017-03-15 23:32:43.891381",
+ "value": "1592546"
+ },
+ {
+ "series": "c61812b8ed3def24fa1df3fbf8d19a96cea3e0f9",
+ "timestamp": "2017-03-15 23:33:03.891344",
+ "value": "1598172"
+ }
+]
+== 20s interval, starting 2m before first sample in archive
+[
+ {
+ "series": "c61812b8ed3def24fa1df3fbf8d19a96cea3e0f9",
+ "timestamp": "2017-03-15 23:31:13.890965",
+ "value": "1537640"
+ },
+ {
+ "series": "c61812b8ed3def24fa1df3fbf8d19a96cea3e0f9",
+ "timestamp": "2017-03-15 23:31:33.891167",
+ "value": "1538453"
+ },
+ {
+ "series": "c61812b8ed3def24fa1df3fbf8d19a96cea3e0f9",
+ "timestamp": "2017-03-15 23:31:53.891930",
+ "value": "1546137"
+ },
+ {
+ "series": "c61812b8ed3def24fa1df3fbf8d19a96cea3e0f9",
+ "timestamp": "2017-03-15 23:32:23.891335",
+ "value": "1572878"
+ },
+ {
+ "series": "c61812b8ed3def24fa1df3fbf8d19a96cea3e0f9",
+ "timestamp": "2017-03-15 23:32:43.891381",
+ "value": "1592546"
+ },
+ {
+ "series": "c61812b8ed3def24fa1df3fbf8d19a96cea3e0f9",
+ "timestamp": "2017-03-15 23:33:03.891344",
+ "value": "1598172"
+ }
+]
diff -Naurp pcp-5.3.7.orig/qa/1626 pcp-5.3.7/qa/1626
--- pcp-5.3.7.orig/qa/1626 2023-07-05 13:42:25.513986223 +1000
+++ pcp-5.3.7/qa/1626 2023-07-05 13:42:53.394025688 +1000
@@ -2,7 +2,7 @@
# PCP QA Test No. 1626
# pmproxy metrics
#
-# Copyright (c) 2021 Red Hat.
+# Copyright (c) 2021-2022 Red Hat.
#
seq=`basename $0`
@@ -71,10 +71,10 @@ val=`_probe_val pmproxy.pid`
pid=`_pmproxy_mainpid`
if [ "$pid" -eq "$val" ]; then :; else echo FAIL pid=$pid val=$val && exit; fi
-echo === check initial pmproxy.webgroup metrics
-for m in instmap labelsmap namesmap contextmap; do
- [ `_probe_val pmproxy.webgroup.$m.size` -eq 0 ] && continue
- echo FAILED pmproxy.webgroup.$m.size expected to be zero && exit
+echo === check initial pmproxy.map metrics
+for m in instance label metric context; do
+ [ `_probe_val pmproxy.map.$m.size` -eq 0 ] && continue
+ echo FAILED pmproxy.map.$m.size expected to be zero && exit
done
echo "=== start the metrics timer with a /metrics RESTAPI call"
@@ -85,18 +85,18 @@ val=`curl -Gs 'http://localhost:44322/me
echo "=== wait for the maps to be updated"
count=0
while true; do
- sz=`_probe_val pmproxy.webgroup.namesmap.size`
+ sz=`_probe_val pmproxy.map.metric.size`
[ "$sz" -gt 0 ] && break
count=`expr $count + 1`
[ $count -gt 20 ] && echo FAILED sz=\"$sz\" after $count iterations && break
sleep 2
done
-echo === pmproxy.webgroup map size metrics should now be nonzero
-for m in instmap labelsmap namesmap contextmap; do
- sz=`_probe_val pmproxy.webgroup.$m.size`
+echo === pmproxy.map size metrics should now be nonzero
+for m in instance label metric context; do
+ sz=`_probe_val pmproxy.map.$m.size`
[ "$sz" -gt 0 ] && continue
- echo FAILED pmproxy.webgroup.$m.size is \"$sz\" but expected to be non-zero
+ echo FAILED pmproxy.map.$m.size is \"$sz\" but expected to be non-zero
exit
done
diff -Naurp pcp-5.3.7.orig/qa/1626.out pcp-5.3.7/qa/1626.out
--- pcp-5.3.7.orig/qa/1626.out 2023-07-05 13:42:25.513986223 +1000
+++ pcp-5.3.7/qa/1626.out 2023-07-05 13:42:53.394025688 +1000
@@ -1,10 +1,10 @@
QA output created by 1626
== wait for pmproxy server metrics
=== check pmproxy.pid
-=== check initial pmproxy.webgroup metrics
+=== check initial pmproxy.map metrics
=== start the metrics timer with a /metrics RESTAPI call
=== wait for the maps to be updated
-=== pmproxy.webgroup map size metrics should now be nonzero
+=== pmproxy.map size metrics should now be nonzero
=== check pmproxy cpu counters
=== check for discovery partial metadata reads
=== check maxrss and datasz values
diff -Naurp pcp-5.3.7.orig/qa/1689.out pcp-5.3.7/qa/1689.out
--- pcp-5.3.7.orig/qa/1689.out 2023-07-05 13:42:25.513986223 +1000
+++ pcp-5.3.7/qa/1689.out 2023-07-05 13:42:53.394025688 +1000
@@ -165,6 +165,30 @@ pmproxy.discover.throttled_changed_callb
Help:
Number of filesystem change callbacks that were ignored due to throttling
+pmproxy.map.context.size PMID: 4.1.6 [context map dictionary size]
+ Data Type: 32-bit unsigned int InDom: PM_INDOM_NULL 0xffffffff
+ Semantics: instant Units: none
+Help:
+Number of entries in the context map dictionary
+
+pmproxy.map.instance.size PMID: 4.1.9 [instance names map dictionary size]
+ Data Type: 32-bit unsigned int InDom: PM_INDOM_NULL 0xffffffff
+ Semantics: instant Units: none
+Help:
+Number of entries in the instance name map dictionary
+
+pmproxy.map.label.size PMID: 4.1.8 [label names map dictionary size]
+ Data Type: 32-bit unsigned int InDom: PM_INDOM_NULL 0xffffffff
+ Semantics: instant Units: none
+Help:
+Number of entries in the labels map dictionary
+
+pmproxy.map.metric.size PMID: 4.1.7 [metric names map dictionary size]
+ Data Type: 32-bit unsigned int InDom: PM_INDOM_NULL 0xffffffff
+ Semantics: instant Units: none
+Help:
+Number of entries in the metric names map dictionary
+
pmproxy.mem.datasz PMID: 4.1.5 [virtual data size]
Data Type: 64-bit unsigned int InDom: PM_INDOM_NULL 0xffffffff
Semantics: instant Units: Kbyte
@@ -291,26 +315,14 @@ pmproxy.series.values.calls PMID: 4.6.6
Help:
total RESTAPI calls to /series/values
-pmproxy.webgroup.contextmap.size PMID: 4.7.1 [context map dictionary size]
- Data Type: 32-bit unsigned int InDom: PM_INDOM_NULL 0xffffffff
- Semantics: instant Units: none
-Help:
-Number of entries in the context map dictionary
-
-pmproxy.webgroup.instmap.size PMID: 4.7.4 [instance name map dictionary size]
- Data Type: 32-bit unsigned int InDom: PM_INDOM_NULL 0xffffffff
- Semantics: instant Units: none
-Help:
-Number of entries in the instance name map dictionary
-
-pmproxy.webgroup.labelsmap.size PMID: 4.7.3 [labels map dictionary size]
+pmproxy.webgroup.gc.context.drops PMID: 4.7.2 [contexts dropped in last garbage collection]
Data Type: 32-bit unsigned int InDom: PM_INDOM_NULL 0xffffffff
Semantics: instant Units: none
Help:
-Number of entries in the labels map dictionary
+Contexts dropped during most recent webgroup garbage collection
-pmproxy.webgroup.namesmap.size PMID: 4.7.2 [metric names map dictionary size]
+pmproxy.webgroup.gc.context.scans PMID: 4.7.1 [contexts scanned in last garbage collection]
Data Type: 32-bit unsigned int InDom: PM_INDOM_NULL 0xffffffff
Semantics: instant Units: none
Help:
-Number of entries in the metric names map dictionary
+Contexts scanned during most recent webgroup garbage collection
diff -Naurp pcp-5.3.7.orig/qa/1691 pcp-5.3.7/qa/1691
--- pcp-5.3.7.orig/qa/1691 1970-01-01 10:00:00.000000000 +1000
+++ pcp-5.3.7/qa/1691 2023-07-05 13:42:53.404025702 +1000
@@ -0,0 +1,74 @@
+#!/bin/sh
+# Exercise pmseries handling of loading archives and ignoring metrics thereof.
+#
+# Copyright (c) 2022 Red Hat.
+#
+
+seq=`basename $0`
+echo "QA output created by $seq"
+
+# get standard environment, filters and checks
+. ./common.product
+. ./common.filter
+. ./common.check
+
+_check_series
+
+_cleanup()
+{
+ [ -n "$options" ] && redis-cli $options shutdown
+ cd $here
+ $sudo rm -rf $tmp $tmp.*
+}
+
+status=1 # failure is the default!
+hostname=`pmhostname`
+redisport=`_find_free_port`
+options="-p $redisport"
+
+$sudo rm -rf $tmp $tmp.* $seq.full
+trap "_cleanup; exit \$status" 0 1 2 3 15
+
+_filter_source()
+{
+ sed \
+ -e "s,$here,PATH,g" \
+ -e "s,$hostname,QAHOST,g" \
+ #end
+}
+
+# real QA test starts here
+cat > $tmp.default.conf <<EOF
+EOF
+
+cat > $tmp.ignore.conf <<EOF
+[discover]
+exclude.metrics = kernel.all.cpu.i*, kernel.all.cpu.user
+EOF
+
+echo "Start test Redis server ..."
+redis-server --port $redisport --save "" > $tmp.redis 2>&1 &
+_check_redis_ping $redisport
+_check_redis_server $redisport
+
+_check_redis_server_version $redisport
+
+echo && echo "Load archive using default config"
+pmseries $options -c $tmp.default.conf --load "{source.path: \"$here/archives/viewqa1\"}" | _filter_source
+
+echo && echo "Query series kernel.*"
+pmseries $options -m `pmseries $options 'kernel.*'`
+
+
+echo && echo "Clear Redis DB"
+redis-cli $options flushall
+
+echo && echo "Load archive using: exclude.metrics = kernel.all.cpu.i*, kernel.all.cpu.user"
+pmseries $options -c $tmp.ignore.conf --load "{source.path: \"$here/archives/viewqa1\"}" | _filter_source
+
+echo && echo "Query series kernel.*"
+pmseries $options -m `pmseries $options 'kernel.*'`
+
+# success, all done
+status=0
+exit
diff -Naurp pcp-5.3.7.orig/qa/1691.out pcp-5.3.7/qa/1691.out
--- pcp-5.3.7.orig/qa/1691.out 1970-01-01 10:00:00.000000000 +1000
+++ pcp-5.3.7/qa/1691.out 2023-07-05 13:42:53.404025702 +1000
@@ -0,0 +1,44 @@
+QA output created by 1691
+Start test Redis server ...
+PING
+PONG
+
+Load archive using default config
+pmseries: [Info] processed 151 archive records from PATH/archives/viewqa1
+
+Query series kernel.*
+
+8b013dfdb7214e020848d0a4a9952ff2a3f777cc
+ Metric: kernel.all.cpu.idle
+
+95135e6cb517cb50fd0c9e28985b944d88850332
+ Metric: kernel.all.cpu.intr
+
+a046e8429bb493a2b40fc23857157a262649d02d
+ Metric: kernel.all.cpu.wait.total
+
+d38aff137f65367ce1aec169be675021a3ebb25c
+ Metric: kernel.all.cpu.nice
+
+f0983eec7e7c01361317266c4259467d35e0ec3e
+ Metric: kernel.all.cpu.sys
+
+e7aa0bd3dc7afc149badec1808fa4fa5c63a7fa3
+ Metric: kernel.all.cpu.user
+
+Clear Redis DB
+OK
+
+Load archive using: exclude.metrics = kernel.all.cpu.i*, kernel.all.cpu.user
+pmseries: [Info] processed 151 archive records from PATH/archives/viewqa1
+
+Query series kernel.*
+
+a046e8429bb493a2b40fc23857157a262649d02d
+ Metric: kernel.all.cpu.wait.total
+
+f0983eec7e7c01361317266c4259467d35e0ec3e
+ Metric: kernel.all.cpu.sys
+
+d38aff137f65367ce1aec169be675021a3ebb25c
+ Metric: kernel.all.cpu.nice
diff -Naurp pcp-5.3.7.orig/qa/1697 pcp-5.3.7/qa/1697
--- pcp-5.3.7.orig/qa/1697 1970-01-01 10:00:00.000000000 +1000
+++ pcp-5.3.7/qa/1697 2023-07-05 13:43:00.404035611 +1000
@@ -0,0 +1,134 @@
+#!/bin/sh
+# PCP QA Test No. 1697
+# Valgrind pmproxy REST API testing.
+# Based on 1601 and 1696
+
+# Copyright (c) 2022 Red Hat.
+#
+
+seq=`basename $0`
+echo "QA output created by $seq"
+
+# get standard environment, filters and checks
+. ./common.product
+. ./common.filter
+. ./common.check
+. ./common.python
+
+
+_check_valgrind
+_check_series
+_check_redis_server_version_offline
+
+_cleanup()
+{
+ cd $here
+ [ -n "$options" ] && redis-cli $options shutdown
+ $sudo rm -rf $tmp $tmp.*
+}
+
+status=1 # failure is the default!
+username=`id -u -n`
+$sudo rm -rf $tmp $tmp.* $seq.full
+trap "_cleanup; exit \$status" 0 1 2 3 15
+
+# create a pmproxy configuration
+cat <<EOF > $tmp.conf
+[pmproxy]
+pcp.enabled = true
+http.enabled = true
+[pmseries]
+enabled = true
+EOF
+
+_filter_source()
+{
+ sed \
+ -e "s,$here,PATH,g" \
+ -e "s,$hostname,QAHOST,g" \
+ #end
+}
+
+_filter_proxyport()
+{
+ sed \
+ -e "s/ FD $proxyport / FD PORT /g" \
+ -e '/PORT ipv6 /d' \
+ # end
+}
+
+# real QA test starts here
+echo "Start test Redis server ..."
+redisport=`_find_free_port`
+options="-p $redisport"
+redis-server --port $redisport --save "" > $tmp.redis 2>&1 &
+_check_redis_ping $redisport
+_check_redis_server $redisport
+echo
+
+_check_redis_server_version $redisport
+
+# import some well-known test data into Redis
+pmseries $options --load "$here/archives/proc" | _filter_source
+
+# start pmproxy
+mkdir -p $tmp.pmproxy/pmproxy
+export PCP_RUN_DIR=$tmp.pmproxy
+export PCP_TMP_DIR=$tmp.pmproxy
+proxyport=`_find_free_port`
+$_valgrind_clean_assert pmproxy -f -p $proxyport -r $redisport -U $username -l- -c $tmp.conf >$tmp.valout 2>$tmp.valerr &
+pid=$!
+
+# valgrind takes awhile to fire up
+i=0
+while [ $i -lt 40 ]
+do
+ $PCP_BINADM_DIR/telnet-probe -c localhost $proxyport && break
+ sleep 1
+ i=`expr $i + 1`
+done
+if $PCP_BINADM_DIR/telnet-probe -c localhost $proxyport
+then
+ echo "Startup took $i secs" >>$seq.full
+else
+ echo "Arrgh: valgrind failed start pmproxy and get port $proxyport ready after 30 secs"
+ exit
+fi
+
+series1=`pmseries $options disk.all.read`
+[ -z "$series1" ] && _fail "Cannot find any timeseries matching disk.all.read"
+echo "Using series $series1 for disk.all.read"
+
+series2=`pmseries $options disk.dev.read`
+[ -z "$series2" ] && _fail "Cannot find any timeseries matching disk.dev.read"
+echo "Using series $series2 for disk.dev.read"
+
+series3=`pmseries $options kernel.all.uptime`
+[ -z "$series3" ] && _fail "Cannot find any timeseries matching kernel.all.uptime"
+echo "Using series $series3 for kernel.all.uptime"
+
+
+echo "== verify metric descs" | tee -a $seq.full
+curl --silent "http://localhost:$proxyport/series/descs" -d "series=$series1,$series2,$series3" | tee -a $seq.full | pmjson
+
+echo "== verify metric names" | tee -a $seq.full
+curl --silent "http://localhost:$proxyport/series/metrics" -d "series=$series1,$series2,$series3" | tee -a $seq.full | pmjson
+
+echo "== verify metric labels" | tee -a $seq.full
+curl --silent "http://localhost:$proxyport/series/labels" -d "series=$series1,$series3" | tee -a $seq.full | pmjson
+
+echo "== verify metric insts" | tee -a $seq.full
+curl --silent "http://localhost:$proxyport/series/instances" -d "series=$series2" | tee -a $seq.full | pmjson
+
+# valgrind takes awhile to shutdown too
+pmsignal $pid
+pmsleep 3.5
+echo "=== valgrind stdout ===" | tee -a $seq.full
+cat $tmp.valout | _filter_valgrind
+
+echo "=== valgrind stderr ===" | tee -a $seq.full
+cat $tmp.valerr | _filter_pmproxy_log | grep -v "Cannot connect to Redis" | _filter_proxyport
+
+# success, all done
+status=0
+exit
diff -Naurp pcp-5.3.7.orig/qa/1697.out pcp-5.3.7/qa/1697.out
--- pcp-5.3.7.orig/qa/1697.out 1970-01-01 10:00:00.000000000 +1000
+++ pcp-5.3.7/qa/1697.out 2023-07-05 13:43:00.414035625 +1000
@@ -0,0 +1,93 @@
+QA output created by 1697
+Start test Redis server ...
+PING
+PONG
+
+pmseries: [Info] processed 5 archive records from PATH/archives/proc
+Using series 1440b8b8bfe69465340eb934e9086ae8212f3cff for disk.all.read
+Using series 605fc77742cd0317597291329561ac4e50c0dd12 for disk.dev.read
+Using series 01d8bc7fa75aaff98a08aa0b1c0f2394368d5183 for kernel.all.uptime
+== verify metric descs
+[
+ {
+ "series": "1440b8b8bfe69465340eb934e9086ae8212f3cff",
+ "source": "2cd6a38f9339f2dd1f0b4775bda89a9e7244def6",
+ "pmid": "60.0.24",
+ "indom": "none",
+ "semantics": "counter",
+ "type": "u64",
+ "units": "count"
+ },
+ {
+ "series": "605fc77742cd0317597291329561ac4e50c0dd12",
+ "source": "2cd6a38f9339f2dd1f0b4775bda89a9e7244def6",
+ "pmid": "60.0.4",
+ "indom": "60.1",
+ "semantics": "counter",
+ "type": "u32",
+ "units": "count"
+ },
+ {
+ "series": "01d8bc7fa75aaff98a08aa0b1c0f2394368d5183",
+ "source": "2cd6a38f9339f2dd1f0b4775bda89a9e7244def6",
+ "pmid": "60.26.0",
+ "indom": "none",
+ "semantics": "instant",
+ "type": "u32",
+ "units": "sec"
+ }
+]
+== verify metric names
+[
+ {
+ "series": "1440b8b8bfe69465340eb934e9086ae8212f3cff",
+ "name": "disk.all.read"
+ },
+ {
+ "series": "605fc77742cd0317597291329561ac4e50c0dd12",
+ "name": "disk.dev.read"
+ },
+ {
+ "series": "01d8bc7fa75aaff98a08aa0b1c0f2394368d5183",
+ "name": "kernel.all.uptime"
+ }
+]
+== verify metric labels
+[
+ {
+ "series": "1440b8b8bfe69465340eb934e9086ae8212f3cff",
+ "labels": {
+ "hostname": "bozo-laptop"
+ }
+ },
+ {
+ "series": "01d8bc7fa75aaff98a08aa0b1c0f2394368d5183",
+ "labels": {
+ "hostname": "bozo-laptop"
+ }
+ }
+]
+== verify metric insts
+[
+ {
+ "series": "605fc77742cd0317597291329561ac4e50c0dd12",
+ "source": "2cd6a38f9339f2dd1f0b4775bda89a9e7244def6",
+ "instance": "c3795d8b757506a2901c6b08b489ba56cae7f0d4",
+ "id": 0,
+ "name": "sda"
+ }
+]
+=== valgrind stdout ===
+=== valgrind stderr ===
+Log for pmproxy on HOST started DATE
+
+pmproxy: PID = PID
+pmproxy request port(s):
+ sts fd port family address
+ === ==== ===== ====== =======
+ok FD unix UNIX_DOMAIN_SOCKET
+ok FD PORT inet INADDR_ANY
+[DATE] pmproxy(PID) Info: pmproxy caught SIGTERM
+[DATE] pmproxy(PID) Info: pmproxy Shutdown
+
+Log finished DATE
diff -Naurp pcp-5.3.7.orig/qa/common.check pcp-5.3.7/qa/common.check
--- pcp-5.3.7.orig/qa/common.check 2023-07-05 13:42:25.513986223 +1000
+++ pcp-5.3.7/qa/common.check 2023-07-05 13:42:53.404025702 +1000
@@ -1226,7 +1226,7 @@ _wait_for_pmproxy_metrics()
{
_n=0
while true; do
- pminfo -f pmproxy.pid pmproxy.cpu pmproxy.webgroup.instmap.size >/dev/null 2>&1 && return 0
+ pminfo -f pmproxy.pid pmproxy.cpu pmproxy.map.instance.size >/dev/null 2>&1 && return 0
sleep 0.25
_n=`expr $_n + 1`
[ $_n -lt 20 ] && continue
diff -Naurp pcp-5.3.7.orig/qa/group pcp-5.3.7/qa/group
--- pcp-5.3.7.orig/qa/group 2023-07-05 13:42:25.523986237 +1000
+++ pcp-5.3.7/qa/group 2023-07-05 13:43:00.414035625 +1000
@@ -1869,6 +1869,7 @@ x11
1601 pmseries pmproxy local
1602 pmproxy local
1603 pmproxy local
+1604 pmseries pmproxy local
1608 pmproxy local
1613 pmda.linux kernel local
1622:retired local
@@ -1890,10 +1891,12 @@ x11
1688 pmieconf local
1689 pmproxy libpcp_web local
1690 pmseries local
+1691 pmseries local
1692 pmda.pmcd local
1694 pidstat local python pcp pmlogextract
1695 pmproxy valgrind local
1696 pmproxy valgrind local
+1697 pmproxy valgrind local
1700 pmda.bpftrace local python
1701 pmda.bpftrace local python
1702 pmda.bpftrace local python
diff -Naurp pcp-5.3.7.orig/src/libpcp_web/src/load.c pcp-5.3.7/src/libpcp_web/src/load.c
--- pcp-5.3.7.orig/src/libpcp_web/src/load.c 2023-07-05 13:42:25.523986237 +1000
+++ pcp-5.3.7/src/libpcp_web/src/load.c 2023-07-05 13:42:53.414025716 +1000
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 2017-2021 Red Hat.
+ * Copyright (c) 2017-2022 Red Hat.
*
* This library is free software; you can redistribute it and/or modify it
* under the terms of the GNU Lesser General Public License as published
@@ -15,6 +15,7 @@
#include <limits.h>
#include <assert.h>
#include <ctype.h>
+#include <fnmatch.h>
#include "encoding.h"
#include "discover.h"
#include "schema.h"
@@ -93,7 +94,7 @@ load_prepare_metric(const char *name, vo
if ((sts = pmLookupName(1, &name, &pmid)) < 0) {
if (sts == PM_ERR_IPC)
cp->setup = 0;
- infofmt(msg, "failed to lookup metric name (pmid=%s): %s",
+ infofmt(msg, "failed to lookup metric name (name=%s): %s",
name, pmErrStr_r(sts, pmmsg, sizeof(pmmsg)));
batoninfo(baton, PMLOG_WARNING, msg);
} else if ((hname = strdup(name)) == NULL) {
@@ -557,48 +558,91 @@ server_cache_update_done(void *arg)
server_cache_window(baton);
}
-void
-server_cache_window(void *arg)
+#if defined(HAVE_LIBUV)
+/* this function runs in a worker thread */
+static void
+fetch_archive(uv_work_t *req)
{
- seriesLoadBaton *baton = (seriesLoadBaton *)arg;
+ seriesLoadBaton *baton = (seriesLoadBaton *)req->data;
seriesGetContext *context = &baton->pmapi;
- struct timeval *finish = &baton->timing.end;
+ context_t *cp = &context->context;
pmResult *result;
int sts;
- seriesBatonCheckMagic(baton, MAGIC_LOAD, "server_cache_window");
- seriesBatonCheckCount(context, "server_cache_window");
assert(context->result == NULL);
+ if ((context->error = sts = pmUseContext(cp->context)) >= 0)
+ if ((context->error = sts = pmFetchArchive(&result)) >= 0)
+ context->result = result;
+}
- if (pmDebugOptions.series)
- fprintf(stderr, "server_cache_window: fetching next result\n");
-
- seriesBatonReference(context, "server_cache_window");
- context->done = server_cache_series_finished;
+/* this function runs in the main thread */
+static void
+fetch_archive_done(uv_work_t *req, int status)
+{
+ seriesLoadBaton *baton = (seriesLoadBaton *)req->data;
+ seriesGetContext *context = &baton->pmapi;
+ struct timeval *finish = &baton->timing.end;
+ int sts = context->error;
- if ((sts = pmFetchArchive(&result)) >= 0) {
- context->result = result;
- if (finish->tv_sec > result->timestamp.tv_sec ||
- (finish->tv_sec == result->timestamp.tv_sec &&
- finish->tv_usec >= result->timestamp.tv_usec)) {
+ free(req);
+ if (context->loaded) {
+ assert(context->result == NULL);
+ doneSeriesGetContext(context, "fetch_archive_done");
+ } else if (sts >= 0) {
+ if (finish->tv_sec > context->result->timestamp.tv_sec ||
+ (finish->tv_sec == context->result->timestamp.tv_sec &&
+ finish->tv_usec >= context->result->timestamp.tv_usec)) {
context->done = server_cache_update_done;
- series_cache_update(baton, NULL);
+ series_cache_update(baton, baton->exclude_pmids);
}
else {
if (pmDebugOptions.series)
- fprintf(stderr, "server_cache_window: end of time window\n");
+ fprintf(stderr, "%s: time window end\n", "fetch_archive_done");
sts = PM_ERR_EOL;
- pmFreeResult(result);
+ pmFreeResult(context->result);
context->result = NULL;
}
}
if (sts < 0) {
- context->error = sts;
if (sts != PM_ERR_EOL)
baton->error = sts;
- doneSeriesGetContext(context, "server_cache_window");
+ doneSeriesGetContext(context, "fetch_archive_done");
}
+
+ doneSeriesLoadBaton(baton, "fetch_archive_done");
+}
+#endif
+
+void
+server_cache_window(void *arg)
+{
+ seriesLoadBaton *baton = (seriesLoadBaton *)arg;
+ seriesGetContext *context = &baton->pmapi;
+
+ seriesBatonCheckMagic(baton, MAGIC_LOAD, "server_cache_window");
+ seriesBatonCheckCount(context, "server_cache_window");
+ assert(context->result == NULL);
+
+ if (pmDebugOptions.series)
+ fprintf(stderr, "%s: fetching next result\n", "server_cache_window");
+
+#if defined(HAVE_LIBUV)
+ seriesBatonReference(baton, "server_cache_window");
+ seriesBatonReference(context, "server_cache_window");
+ context->done = server_cache_series_finished;
+
+ /*
+ * We must perform pmFetchArchive(3) in a worker thread
+ * because it contains blocking (synchronous) I/O calls
+ */
+ uv_work_t *req = malloc(sizeof(uv_work_t));
+ req->data = baton;
+ uv_queue_work(uv_default_loop(), req, fetch_archive, fetch_archive_done);
+#else
+ baton->error = -ENOTSUP;
+ server_cache_series_finished(arg);
+#endif
}
static void
@@ -640,7 +684,35 @@ add_source_metric(seriesLoadBaton *baton
}
static void
-load_prepare_metrics(seriesLoadBaton *baton)
+load_prepare_exclude_metric(const char *name, void *arg)
+{
+ seriesLoadBaton *baton = (seriesLoadBaton *)arg;
+ char pmmsg[PM_MAXERRMSGLEN];
+ sds msg;
+ pmID pmid;
+ int i;
+ int sts;
+
+ /*
+ * check if this metric name matches any exclude pattern
+ * if it matches, add the pmID of this metric to the exclude list
+ */
+ for (i = 0; i < baton->exclude_npatterns; i++) {
+ if (fnmatch(baton->exclude_patterns[i], name, 0) == 0) {
+ if ((sts = pmLookupName(1, &name, &pmid)) < 0) {
+ infofmt(msg, "failed to lookup metric name (name=%s): %s",
+ name, pmErrStr_r(sts, pmmsg, sizeof(pmmsg)));
+ batoninfo(baton, PMLOG_WARNING, msg);
+ } else {
+ dictAdd(baton->exclude_pmids, &pmid, NULL);
+ }
+ break;
+ }
+ }
+}
+
+static void
+load_prepare_included_metrics(seriesLoadBaton *baton)
{
context_t *cp = &baton->pmapi.context;
const char **metrics = baton->metrics;
@@ -659,6 +731,57 @@ load_prepare_metrics(seriesLoadBaton *ba
}
}
+static void
+load_prepare_excluded_metrics(seriesLoadBaton *baton)
+{
+ pmSeriesModule *module = (pmSeriesModule *)baton->module;
+ seriesModuleData *data = getSeriesModuleData(module);
+ char pmmsg[PM_MAXERRMSGLEN];
+ sds msg;
+ int i, sts;
+ sds exclude_metrics_option;
+ sds *patterns = NULL;
+ int npatterns = 0;
+
+ if (data == NULL) {
+ baton->error = -ENOMEM;
+ return;
+ }
+
+ if (!(exclude_metrics_option = pmIniFileLookup(data->config, "discover", "exclude.metrics"))) {
+ /* option not set, using default value of no excluded metrics */
+ return;
+ }
+
+ if (!(patterns = sdssplitlen(exclude_metrics_option, sdslen(exclude_metrics_option), ",", 1, &npatterns))) {
+ /* empty option, ignore */
+ return;
+ }
+
+ /* trim each comma-separated entry */
+ for (i = 0; i < npatterns; i++)
+ patterns[i] = sdstrim(patterns[i], " ");
+
+ baton->exclude_patterns = patterns;
+ baton->exclude_npatterns = npatterns;
+
+ /*
+ * unfortunately we need to traverse the entire PMNS here to match the patterns (e.g. proc.*)
+ * against metrics and gather a list of pmIDs to exclude
+ *
+ * alternatively pattern matching could happen in series_cache_update(), however that will come
+ * with a performance penalty (looping through patterns + fnmatch() each in a hot path vs.
+ * simple pmID lookup in a dict)
+ */
+ if ((sts = pmTraversePMNS_r("", load_prepare_exclude_metric, baton)) < 0) {
+ infofmt(msg, "PMNS traversal failed: %s", pmErrStr_r(sts, pmmsg, sizeof(pmmsg)));
+ batoninfo(baton, PMLOG_WARNING, msg);
+ }
+
+ sdsfreesplitres(patterns, npatterns);
+ baton->exclude_patterns = NULL;
+}
+
static int
load_prepare_timing(seriesLoadBaton *baton)
{
@@ -812,11 +935,12 @@ doneSeriesGetContext(seriesGetContext *c
if (seriesBatonDereference(context, caller) && context->done != NULL)
context->done(baton);
- if (context->error) {
+ if (context->error && !context->loaded) {
char pmmsg[PM_MAXERRMSGLEN];
sds msg;
if (context->error == PM_ERR_EOL) {
+ context->loaded = 1;
infofmt(msg, "processed %llu archive records from %s",
context->count, context->context.name.sds);
batoninfo(baton, PMLOG_INFO, msg);
@@ -893,7 +1017,8 @@ connect_pmapi_source_service(void *arg)
} else if (baton->error == 0) {
/* setup metric and time-based filtering for source load */
load_prepare_timing(baton);
- load_prepare_metrics(baton);
+ load_prepare_included_metrics(baton);
+ load_prepare_excluded_metrics(baton);
}
series_load_end_phase(baton);
}
@@ -966,6 +1091,7 @@ initSeriesLoadBaton(seriesLoadBaton *bat
baton->errors = dictCreate(&intKeyDictCallBacks, baton);
baton->wanted = dictCreate(&intKeyDictCallBacks, baton);
+ baton->exclude_pmids = dictCreate(&intKeyDictCallBacks, baton);
}
void
@@ -979,6 +1105,7 @@ freeSeriesLoadBaton(seriesLoadBaton *bat
freeSeriesGetContext(&baton->pmapi, 0);
dictRelease(baton->errors);
dictRelease(baton->wanted);
+ dictRelease(baton->exclude_pmids);
free(baton->metrics);
memset(baton, 0, sizeof(*baton));
diff -Naurp pcp-5.3.7.orig/src/libpcp_web/src/query.c pcp-5.3.7/src/libpcp_web/src/query.c
--- pcp-5.3.7.orig/src/libpcp_web/src/query.c 2023-07-05 13:42:25.523986237 +1000
+++ pcp-5.3.7/src/libpcp_web/src/query.c 2023-07-05 13:42:53.414025716 +1000
@@ -49,7 +49,7 @@ typedef struct seriesGetLookup {
} seriesGetLookup;
typedef struct seriesGetQuery {
- node_t root;
+ node_t *root;
timing_t timing;
} seriesGetQuery;
@@ -63,16 +63,14 @@ typedef struct seriesQueryBaton {
void *userdata;
redisSlots *slots;
int error;
- union {
- seriesGetLookup lookup;
- seriesGetQuery query;
- } u;
+ seriesGetLookup lookup;
+ seriesGetQuery query;
} seriesQueryBaton;
static void series_pattern_match(seriesQueryBaton *, node_t *);
static int series_union(series_set_t *, series_set_t *);
static int series_intersect(series_set_t *, series_set_t *);
-static int series_calculate(seriesQueryBaton *, node_t *, int);
+static int series_calculate(node_t *, int, void *);
static void series_redis_hash_expression(seriesQueryBaton *, char *, int);
static void series_node_get_metric_name(seriesQueryBaton *, seriesGetSID *, series_sample_set_t *);
static void series_node_get_desc(seriesQueryBaton *, sds, series_sample_set_t *);
@@ -88,8 +86,8 @@ static void
initSeriesGetQuery(seriesQueryBaton *baton, node_t *root, timing_t *timing)
{
seriesBatonCheckMagic(baton, MAGIC_QUERY, "initSeriesGetQuery");
- baton->u.query.root = *root;
- baton->u.query.timing = *timing;
+ baton->query.root = root;
+ baton->query.timing = *timing;
}
static int
@@ -106,18 +104,21 @@ skip_free_value_set(node_t *np)
}
static void
-freeSeriesQueryNode(node_t *np, int level)
+freeSeriesQueryNode(node_t *np)
{
int n_samples;
+
if (np == NULL)
return;
+
if (skip_free_value_set(np) != 0) {
int i, j, k;
+
for (i = 0; i < np->value_set.num_series; i++) {
n_samples = np->value_set.series_values[i].num_samples;
if (n_samples < 0) n_samples = -n_samples;
for (j = 0; j < n_samples; j++) {
- for (k=0; k < np->value_set.series_values[i].series_sample[j].num_instances; k++) {
+ for (k = 0; k < np->value_set.series_values[i].series_sample[j].num_instances; k++) {
sdsfree(np->value_set.series_values[i].series_sample[j].series_instance[k].timestamp);
sdsfree(np->value_set.series_values[i].series_sample[j].series_instance[k].series);
sdsfree(np->value_set.series_values[i].series_sample[j].series_instance[k].data);
@@ -127,13 +128,22 @@ freeSeriesQueryNode(node_t *np, int leve
sdsfree(np->value_set.series_values[i].sid->name);
free(np->value_set.series_values[i].sid);
free(np->value_set.series_values[i].series_sample);
+ sdsfree(np->value_set.series_values[i].series_desc.indom);
+ sdsfree(np->value_set.series_values[i].series_desc.pmid);
+ sdsfree(np->value_set.series_values[i].series_desc.semantics);
+ sdsfree(np->value_set.series_values[i].series_desc.source);
+ sdsfree(np->value_set.series_values[i].series_desc.type);
+ sdsfree(np->value_set.series_values[i].series_desc.units);
}
free(np->value_set.series_values);
}
- freeSeriesQueryNode(np->left, level+1);
- freeSeriesQueryNode(np->right, level+1);
- if (level != 0)
- free(np);
+ freeSeriesQueryNode(np->left);
+ freeSeriesQueryNode(np->right);
+ if (np->result.nseries)
+ free(np->result.series);
+ sdsfree(np->value);
+ sdsfree(np->key);
+ free(np);
}
static void
@@ -141,9 +151,7 @@ freeSeriesGetQuery(seriesQueryBaton *bat
{
seriesBatonCheckMagic(baton, MAGIC_QUERY, "freeSeriesGetQuery");
seriesBatonCheckCount(baton, "freeSeriesGetQuery");
- if (baton->error == 0) {
- freeSeriesQueryNode(&baton->u.query.root, 0);
- }
+ freeSeriesQueryNode(baton->query.root);
memset(baton, 0, sizeof(seriesQueryBaton));
free(baton);
}
@@ -226,15 +234,15 @@ initSeriesGetLookup(seriesQueryBaton *ba
/* pattern matching parameter, optional */
if (nseries == 0 && series != NULL)
- baton->u.lookup.pattern = *series;
+ baton->lookup.pattern = *series;
/* else lookup array of individual sids */
for (i = 0; i < nseries; i++) {
- sid = &baton->u.lookup.series[i];
+ sid = &baton->lookup.series[i];
initSeriesGetSID(sid, series[i], 0, baton);
}
- baton->u.lookup.nseries = nseries;
- baton->u.lookup.func = func;
- baton->u.lookup.map = map;
+ baton->lookup.nseries = nseries;
+ baton->lookup.func = func;
+ baton->lookup.map = map;
}
static void
@@ -247,9 +255,9 @@ freeSeriesGetLookup(seriesQueryBaton *ba
seriesBatonCheckMagic(baton, MAGIC_QUERY, "freeSeriesGetLookup");
seriesBatonCheckCount(baton, "freeSeriesGetLookup");
- nseries = baton->u.lookup.nseries;
+ nseries = baton->lookup.nseries;
for (i = 0; i < nseries; i++) {
- sid = &baton->u.lookup.series[i];
+ sid = &baton->lookup.series[i];
sdsfree(sid->name);
}
bytes = sizeof(seriesQueryBaton) + (nseries * sizeof(seriesGetSID));
@@ -400,7 +408,7 @@ extract_mapping(seriesQueryBaton *baton,
if (reply->type == REDIS_REPLY_STRING) {
key = sdsnewlen(reply->str, reply->len);
- entry = redisMapLookup(baton->u.lookup.map, key);
+ entry = redisMapLookup(baton->lookup.map, key);
sdsfree(key);
if (entry != NULL) {
key = redisMapValue(entry);
@@ -553,7 +561,14 @@ static int
use_next_sample(seriesSampling *sampling)
{
/* if the next timestamp is past our goal, use the current value */
- if (pmTimespec_cmp(&sampling->next_timespec, &sampling->goal) > 0) {
+ double goal_delta = pmTimespec_delta(&sampling->next_timespec, &sampling->goal);
+ if (goal_delta > 0) {
+ /* if the goal significantly lags behind, reset it */
+ /* this can happen when start < first sample or when there are gaps in the series */
+ if (goal_delta > 2 * sampling->delta.tv_sec) {
+ sampling->goal = sampling->next_timespec;
+ }
+
/* selected a value for this interval so move the goal posts */
pmTimespec_add(&sampling->goal, &sampling->delta);
return 0;
@@ -567,7 +582,7 @@ series_values_reply(seriesQueryBaton *ba
{
seriesSampling sampling = {0};
redisReply *reply, *sample, **elements;
- timing_t *tp = &baton->u.query.timing;
+ timing_t *tp = &baton->query.timing;
int n, sts, next, nelements;
sds msg, save_timestamp;
@@ -1360,14 +1375,14 @@ static void
on_series_solve_setup(void *arg)
{
if (pmDebugOptions.query)
- fprintf(stderr, "on_series_solve_setup\n");
+ fprintf(stderr, "%s\n", "on_series_solve_setup");
}
static void
on_series_solve_log(pmLogLevel level, sds message, void *arg)
{
if (pmDebugOptions.query)
- fprintf(stderr, "on_series_solve_log: %s\n", message);
+ fprintf(stderr, "%s: %s\n", "on_series_solve_log", message);
}
static void
@@ -1377,7 +1392,7 @@ on_series_solve_done(int status, void *a
seriesBatonCheckMagic(baton, MAGIC_QUERY, "on_series_solve_done");
if (pmDebugOptions.query && pmDebugOptions.desperate)
- fprintf(stderr, "on_series_solve_done: arg=%p status=%d\n", arg, status);
+ fprintf(stderr, "%s: arg=%p status=%d\n", "on_series_solve_done", arg, status);
baton->callbacks->on_done(status, baton->userdata);
}
@@ -1388,7 +1403,7 @@ on_series_solve_value(pmSID sid, pmSerie
seriesBatonCheckMagic(baton, MAGIC_QUERY, "on_series_solve_value");
if (pmDebugOptions.query && pmDebugOptions.desperate)
- fprintf(stderr, "on_series_solve_value: arg=%p %s %s %s\n",
+ fprintf(stderr, "%s: arg=%p %s %s %s\n", "on_series_solve_value",
arg, value->timestamp, value->data, value->series);
return baton->callbacks->on_value(sid, value, baton->userdata);
}
@@ -1399,9 +1414,9 @@ on_series_solve_inst_done(int status, vo
{
seriesQueryBaton *baton = arg;
- seriesBatonCheckMagic(baton, MAGIC_QUERY, "on_series_solve_done");
+ seriesBatonCheckMagic(baton, MAGIC_QUERY, "on_series_solve_inst_done");
if (pmDebugOptions.query && pmDebugOptions.desperate)
- fprintf(stderr, "on_series_solve_done: arg=%p status=%d\n", arg, status);
+ fprintf(stderr, "%s: arg=%p status=%d\n", "on_series_solve_done", arg, status);
/* on_done is called by series_query_finished */
seriesBatonDereference(baton, "on_series_solve_inst_done");
}
@@ -1419,7 +1434,7 @@ on_series_solve_inst_value(pmSID sid, pm
seriesBatonCheckMagic(baton, MAGIC_QUERY, "on_series_solve_inst_value");
if (pmDebugOptions.query) {
- fprintf(stderr, "on_series_solve_inst_value: arg=%p %s %s %s\n",
+ fprintf(stderr, "%s: arg=%p %s %s %s\n", "on_series_solve_inst_value",
arg, value->timestamp, value->data, value->series);
}
@@ -1478,11 +1493,11 @@ series_solve_sid_expr(pmSeriesSettings *
seriesBatonCheckMagic(sid, MAGIC_SID, "series_query_expr_reply");
seriesBatonCheckMagic(baton, MAGIC_QUERY, "series_query_expr_reply");
- if (pmDebugOptions.query) {
- fprintf(stderr, "series_solve_sid_expr: SID %s, "
- "seriesQueryBaton=%p, pmSeriesBaton=userdata=%p expr=\"%s\"\n",
- sid->name, baton, baton->userdata, expr->query);
- }
+ if (pmDebugOptions.query)
+ fprintf(stderr, "%s: SID %s, seriesQueryBaton=%p, "
+ "pmSeriesBaton=userdata=%p expr=\"%s\"\n",
+ "series_solve_sid_expr",
+ sid->name, baton, baton->userdata, expr->query);
/* ref baton until on_series_solve_done */
seriesBatonReference(baton, "series_solve_sid_expr");
@@ -1491,7 +1506,7 @@ series_solve_sid_expr(pmSeriesSettings *
pmSeriesSetSlots(&settings->module, baton->slots);
settings->module = *baton->module; /* struct cpy */
- sts = series_solve(settings, sp.expr, &baton->u.query.timing,
+ sts = series_solve(settings, sp.expr, &baton->query.timing,
PM_SERIES_FLAG_NONE, baton);
}
@@ -1587,7 +1602,7 @@ series_value_count_only(timing_t *tp)
static void
series_prepare_time(seriesQueryBaton *baton, series_set_t *result)
{
- timing_t *tp = &baton->u.query.timing;
+ timing_t *tp = &baton->query.timing;
unsigned char *series = result->series;
seriesGetSID *sid;
char buffer[64], revbuf[64];
@@ -1730,11 +1745,16 @@ series_query_report_matches(void *arg)
seriesBatonReference(baton, "series_query_report_matches");
- has_function = series_calculate(baton, &baton->u.query.root, 0);
-
+ has_function = series_calculate(baton->query.root, 0, arg);
+
+ /*
+ * Store the canonical query to Redis if this query statement has
+ * function operation.
+ */
if (has_function != 0)
series_redis_hash_expression(baton, hashbuf, sizeof(hashbuf));
- series_report_set(baton, &baton->u.query.root);
+
+ series_report_set(baton, baton->query.root);
series_query_end_phase(baton);
}
@@ -1747,7 +1767,7 @@ series_query_maps(void *arg)
seriesBatonCheckCount(baton, "series_query_maps");
seriesBatonReference(baton, "series_query_maps");
- series_prepare_maps(baton, &baton->u.query.root, 0);
+ series_prepare_maps(baton, baton->query.root, 0);
series_query_end_phase(baton);
}
@@ -1760,7 +1780,7 @@ series_query_eval(void *arg)
seriesBatonCheckCount(baton, "series_query_eval");
seriesBatonReference(baton, "series_query_eval");
- series_prepare_eval(baton, &baton->u.query.root, 0);
+ series_prepare_eval(baton, baton->query.root, 0);
series_query_end_phase(baton);
}
@@ -1773,7 +1793,7 @@ series_query_expr(void *arg)
seriesBatonCheckCount(baton, "series_query_expr");
seriesBatonReference(baton, "series_query_expr");
- series_prepare_expr(baton, &baton->u.query.root, 0);
+ series_prepare_expr(baton, baton->query.root, 0);
series_query_end_phase(baton);
}
@@ -1827,7 +1847,7 @@ series_values_store_to_node(seriesQueryB
{
seriesSampling sampling = {0};
redisReply *reply, *sample, **elements;
- timing_t *tp = &baton->u.query.timing;
+ timing_t *tp = &baton->query.timing;
int i, sts, next, nelements;
int idx_series = np->value_set.num_series;
int idx_sample = 0;
@@ -2540,9 +2560,9 @@ series_rate_check(pmSeriesDesc desc)
* The number of samples in result is one less than the original samples.
*/
static void
-series_calculate_rate(node_t *np)
+series_calculate_rate(node_t *np, void *arg)
{
- seriesQueryBaton *baton = (seriesQueryBaton *)np->baton;
+ seriesQueryBaton *baton = (seriesQueryBaton *)arg;
pmSeriesValue s_pmval, t_pmval;
unsigned int n_instances, n_samples, i, j, k;
double s_data, t_data, mult;
@@ -2627,9 +2647,9 @@ series_calculate_rate(node_t *np)
* Compare and pick the maximal instance value(s) among samples for each metric.
*/
static void
-series_calculate_max(node_t *np)
+series_calculate_max(node_t *np, void *arg)
{
- seriesQueryBaton *baton = (seriesQueryBaton *)np->baton;
+ seriesQueryBaton *baton = (seriesQueryBaton *)arg;
unsigned int n_series, n_samples, n_instances, i, j, k;
double max_data, data;
int max_pointer;
@@ -2686,9 +2706,9 @@ series_calculate_max(node_t *np)
* Compare and pick the minimal instance value(s) among samples for each metric.
*/
static void
-series_calculate_min(node_t *np)
+series_calculate_min(node_t *np, void *arg)
{
- seriesQueryBaton *baton = (seriesQueryBaton *)np->baton;
+ seriesQueryBaton *baton = (seriesQueryBaton *)arg;
unsigned int n_series, n_samples, n_instances, i, j, k;
double min_data, data;
int min_pointer;
@@ -2838,9 +2858,9 @@ series_pmAtomValue_conv_str(int type, ch
* metrics to be modified.
*/
static void
-series_calculate_rescale(node_t *np)
+series_calculate_rescale(node_t *np, void *arg)
{
- seriesQueryBaton *baton = (seriesQueryBaton *)np->baton;
+ seriesQueryBaton *baton = (seriesQueryBaton *)arg;
double mult;
pmUnits iunit;
char *errmsg, str_val[256];
@@ -2933,9 +2953,9 @@ series_abs_pmAtomValue(int type, pmAtomV
}
static void
-series_calculate_abs(node_t *np)
+series_calculate_abs(node_t *np, void *arg)
{
- seriesQueryBaton *baton = (seriesQueryBaton *)np->baton;
+ seriesQueryBaton *baton = (seriesQueryBaton *)arg;
pmAtomValue val;
int type, sts, str_len, i, j, k;
char str_val[256];
@@ -2976,9 +2996,10 @@ series_calculate_abs(node_t *np)
* calculate sum or avg series per-instance over time samples
*/
static void
-series_calculate_statistical(node_t *np, nodetype_t func)
+series_calculate_statistical(node_t *np, void *arg)
{
- seriesQueryBaton *baton = (seriesQueryBaton *)np->baton;
+ seriesQueryBaton *baton = (seriesQueryBaton *)arg;
+ nodetype_t func = np->type;
unsigned int n_series, n_samples, n_instances, i, j, k;
double sum_data, data;
char sum_data_str[64];
@@ -3072,9 +3093,9 @@ series_floor_pmAtomValue(int type, pmAto
}
static void
-series_calculate_floor(node_t *np)
+series_calculate_floor(node_t *np, void *arg)
{
- seriesQueryBaton *baton = (seriesQueryBaton *)np->baton;
+ seriesQueryBaton *baton = (seriesQueryBaton *)arg;
pmAtomValue val;
int type, sts, str_len, i, j, k;
char str_val[256];
@@ -3157,9 +3178,9 @@ series_log_pmAtomValue(int itype, int *o
* Return the logarithm of x to base b (log_b^x).
*/
static void
-series_calculate_log(node_t *np)
+series_calculate_log(node_t *np, void *arg)
{
- seriesQueryBaton *baton = (seriesQueryBaton *)np->baton;
+ seriesQueryBaton *baton = (seriesQueryBaton *)arg;
double base;
pmAtomValue val;
int i, j, k, itype, otype=PM_TYPE_UNKNOWN;
@@ -3251,9 +3272,9 @@ series_sqrt_pmAtomValue(int itype, int *
}
static void
-series_calculate_sqrt(node_t *np)
+series_calculate_sqrt(node_t *np, void *arg)
{
- seriesQueryBaton *baton = (seriesQueryBaton *)np->baton;
+ seriesQueryBaton *baton = (seriesQueryBaton *)arg;
pmAtomValue val;
int i, j, k, itype, otype=PM_TYPE_UNKNOWN;
int sts, str_len;
@@ -3320,9 +3341,9 @@ series_round_pmAtomValue(int type, pmAto
}
static void
-series_calculate_round(node_t *np)
+series_calculate_round(node_t *np, void *arg)
{
- seriesQueryBaton *baton = (seriesQueryBaton *)np->baton;
+ seriesQueryBaton *baton = (seriesQueryBaton *)arg;
pmAtomValue val;
int i, j, k, type, sts, str_len;
char str_val[256];
@@ -3393,6 +3414,7 @@ series_calculate_binary_check(int ope_ty
baton->error = -EPROTO;
return -1;
}
+
/*
* For addition and subtraction all dimensions for each of
* the operands and result are identical.
@@ -3711,9 +3733,9 @@ series_binary_meta_update(node_t *left,
}
static void
-series_calculate_plus(node_t *np)
+series_calculate_plus(node_t *np, void *arg)
{
- seriesQueryBaton *baton = (seriesQueryBaton *)np->baton;
+ seriesQueryBaton *baton = (seriesQueryBaton *)arg;
node_t *left = np->left, *right = np->right;
int l_type, r_type, otype=PM_TYPE_UNKNOWN;
int l_sem, r_sem, j, k;
@@ -3763,9 +3785,9 @@ series_calculate_plus(node_t *np)
}
static void
-series_calculate_minus(node_t *np)
+series_calculate_minus(node_t *np, void *arg)
{
- seriesQueryBaton *baton = (seriesQueryBaton *)np->baton;
+ seriesQueryBaton *baton = (seriesQueryBaton *)arg;
node_t *left = np->left, *right = np->right;
unsigned int num_samples, num_instances, j, k;
pmAtomValue l_val, r_val;
@@ -3815,9 +3837,9 @@ series_calculate_minus(node_t *np)
}
static void
-series_calculate_star(node_t *np)
+series_calculate_star(node_t *np, void *arg)
{
- seriesQueryBaton *baton = (seriesQueryBaton *)np->baton;
+ seriesQueryBaton *baton = (seriesQueryBaton *)arg;
node_t *left = np->left, *right = np->right;
unsigned int num_samples, num_instances, j, k;
pmAtomValue l_val, r_val;
@@ -3867,9 +3889,9 @@ series_calculate_star(node_t *np)
}
static void
-series_calculate_slash(node_t *np)
+series_calculate_slash(node_t *np, void *arg)
{
- seriesQueryBaton *baton = (seriesQueryBaton *)np->baton;
+ seriesQueryBaton *baton = (seriesQueryBaton *)arg;
node_t *left = np->left, *right = np->right;
unsigned int num_samples, num_instances, j, k;
pmAtomValue l_val, r_val;
@@ -3878,12 +3900,16 @@ series_calculate_slash(node_t *np)
int l_sem, r_sem;
sds msg;
- if (left->value_set.num_series == 0 || right->value_set.num_series == 0) return;
- if (series_calculate_binary_check(N_SLASH, baton, left, right, &l_type, &r_type, &l_sem, &r_sem,
- &l_units, &r_units, &large_units, left->value_set.series_values[0].series_desc.indom,
- right->value_set.series_values[0].series_desc.indom) != 0) {
+ if (left->value_set.num_series == 0 || right->value_set.num_series == 0)
return;
- }
+
+ if (series_calculate_binary_check(N_SLASH, baton, left, right,
+ &l_type, &r_type, &l_sem, &r_sem,
+ &l_units, &r_units, &large_units,
+ left->value_set.series_values[0].series_desc.indom,
+ right->value_set.series_values[0].series_desc.indom) != 0)
+ return;
+
num_samples = left->value_set.series_values[0].num_samples;
for (j = 0; j < num_samples; j++) {
@@ -3912,7 +3938,6 @@ series_calculate_slash(node_t *np)
series_binary_meta_update(left, &large_units, &l_sem, &r_sem, &otype);
np->value_set = left->value_set;
-
}
/*
@@ -3924,81 +3949,65 @@ series_calculate_slash(node_t *np)
* store them into this node.
*/
static int
-series_calculate(seriesQueryBaton *baton, node_t *np, int level)
+series_calculate(node_t *np, int level, void *arg)
{
int sts;
if (np == NULL)
return 0;
- if ((sts = series_calculate(baton, np->left, level+1)) < 0)
+ if ((sts = series_calculate(np->left, level+1, arg)) < 0)
return sts;
- if ((sts = series_calculate(baton, np->right, level+1)) < 0)
+ if ((sts = series_calculate(np->right, level+1, arg)) < 0)
return sts;
- np->baton = baton;
- switch (np->type) {
- case N_RATE:
- series_calculate_rate(np);
- sts = N_RATE;
- break;
- case N_MAX:
- series_calculate_max(np);
- sts = N_MAX;
- break;
- case N_MIN:
- series_calculate_min(np);
- sts = N_MIN;
- break;
- case N_RESCALE:
- series_calculate_rescale(np);
- sts = N_RESCALE;
- break;
- case N_ABS:
- series_calculate_abs(np);
- sts = N_ABS;
- break;
- case N_FLOOR:
- series_calculate_floor(np);
- sts = N_FLOOR;
- break;
- case N_LOG:
- series_calculate_log(np);
- sts = N_LOG;
- break;
- case N_SQRT:
- series_calculate_sqrt(np);
- sts = N_SQRT;
- break;
- case N_ROUND:
- series_calculate_round(np);
- sts = N_ROUND;
- break;
- case N_PLUS:
- series_calculate_plus(np);
- sts = N_PLUS;
- break;
- case N_MINUS:
- series_calculate_minus(np);
- sts = N_MINUS;
- break;
- case N_STAR:
- series_calculate_star(np);
- sts = N_STAR;
- break;
- case N_SLASH:
- series_calculate_slash(np);
- sts = N_SLASH;
- break;
- case N_AVG:
- series_calculate_statistical(np, N_AVG);
- sts = N_AVG;
- break;
- case N_SUM:
- series_calculate_statistical(np, N_SUM);
- sts = N_SUM;
- break;
- default:
- break;
+ switch ((sts = np->type)) {
+ case N_RATE:
+ series_calculate_rate(np, arg);
+ break;
+ case N_MAX:
+ series_calculate_max(np, arg);
+ break;
+ case N_MIN:
+ series_calculate_min(np, arg);
+ break;
+ case N_RESCALE:
+ series_calculate_rescale(np, arg);
+ sts = N_RESCALE;
+ break;
+ case N_ABS:
+ series_calculate_abs(np, arg);
+ break;
+ case N_FLOOR:
+ series_calculate_floor(np, arg);
+ break;
+ case N_LOG:
+ series_calculate_log(np, arg);
+ break;
+ case N_SQRT:
+ series_calculate_sqrt(np, arg);
+ break;
+ case N_ROUND:
+ series_calculate_round(np, arg);
+ break;
+ case N_PLUS:
+ series_calculate_plus(np, arg);
+ break;
+ case N_MINUS:
+ series_calculate_minus(np, arg);
+ break;
+ case N_STAR:
+ series_calculate_star(np, arg);
+ break;
+ case N_SLASH:
+ series_calculate_slash(np, arg);
+ break;
+ case N_AVG:
+ case N_SUM:
+ series_calculate_statistical(np, arg);
+ break;
+ default:
+ sts = 0; /* no function */
+ break;
}
return sts;
}
@@ -4006,9 +4015,9 @@ series_calculate(seriesQueryBaton *baton
static int
check_compatibility(pmUnits *units_a, pmUnits *units_b)
{
- if (compare_pmUnits_dim(units_a, units_b) == 0) {
+ if (compare_pmUnits_dim(units_a, units_b) == 0)
return 0;
- } else return -1;
+ return -1;
}
static void
@@ -4072,7 +4081,7 @@ series_redis_hash_expression(seriesQuery
unsigned char hash[20];
sds key, msg;
char *errmsg;
- node_t *np = &baton->u.query.root;
+ node_t *np = baton->query.root;
int i, j, num_series = np->value_set.num_series;
pmUnits units0, units1, large_units;
double mult;
@@ -4088,17 +4097,18 @@ series_redis_hash_expression(seriesQuery
for (i = 0; i < num_series; i++) {
if (!np->value_set.series_values[i].compatibility) {
- infofmt(msg, "Descriptors for metric '%s' do not satisfy compatibility between different hosts/sources.\n",
+ infofmt(msg, "Descriptors for metric '%s' between different sources are incompatible.\n",
np->value_set.series_values[i].metric_name);
batoninfo(baton, PMLOG_ERROR, msg);
baton->error = -EPROTO;
- continue;
+ break;
}
if (strncmp(np->value_set.series_values[i].series_desc.units, "none", 4) == 0)
memset(&units0, 0, sizeof(units0));
- else if (pmParseUnitsStr(np->value_set.series_values[i].series_desc.units,
- &units0, &mult, &errmsg) != 0) {
+ else if (pmParseUnitsStr(
+ np->value_set.series_values[i].series_desc.units,
+ &units0, &mult, &errmsg) != 0) {
np->value_set.series_values[i].compatibility = 0;
infofmt(msg, "Invalid units string: %s\n",
np->value_set.series_values[i].series_desc.units);
@@ -4127,7 +4137,12 @@ series_redis_hash_expression(seriesQuery
if (check_compatibility(&units0, &units1) != 0) {
np->value_set.series_values[j].compatibility = 0;
- infofmt(msg, "Incompatible units between operand metrics or expressions\n");
+ infofmt(msg, "Incompatible units between operand metrics or expressions\n"
+ "'%s' (%s)\nvs\'%s' (%s)\n",
+ np->value_set.series_values[i].metric_name,
+ np->value_set.series_values[i].series_desc.units,
+ np->value_set.series_values[j].metric_name,
+ np->value_set.series_values[j].series_desc.units);
batoninfo(baton, PMLOG_ERROR, msg);
baton->error = -EPROTO;
break;
@@ -4175,7 +4190,7 @@ series_query_report_values(void *arg)
seriesBatonCheckCount(baton, "series_query_report_values");
seriesBatonReference(baton, "series_query_report_values");
- series_prepare_time(baton, &baton->u.query.root.result);
+ series_prepare_time(baton, &baton->query.root->result);
series_query_end_phase(baton);
}
@@ -4192,7 +4207,7 @@ series_query_funcs_report_values(void *a
seriesBatonReference(baton, "series_query_funcs_report_values");
/* For function-type nodes, calculate actual values */
- has_function = series_calculate(baton, &baton->u.query.root, 0);
+ has_function = series_calculate(baton->query.root, 0, baton->userdata);
/*
* Store the canonical query to Redis if this query statement has
@@ -4202,7 +4217,7 @@ series_query_funcs_report_values(void *a
series_redis_hash_expression(baton, hashbuf, sizeof(hashbuf));
/* time series values saved in root node so report them directly. */
- series_node_values_report(baton, &baton->u.query.root);
+ series_node_values_report(baton, baton->query.root);
series_query_end_phase(baton);
}
@@ -4217,7 +4232,7 @@ series_query_funcs(void *arg)
seriesBatonReference(baton, "series_query_funcs");
/* Process function-type node */
- series_process_func(baton, &baton->u.query.root, 0);
+ series_process_func(baton, baton->query.root, 0);
series_query_end_phase(baton);
}
@@ -4230,7 +4245,7 @@ series_query_desc(void *arg)
seriesBatonCheckCount(baton, "series_query_desc");
seriesBatonReference(baton, "series_query_desc");
- series_expr_node_desc(baton, &baton->u.query.root);
+ series_expr_node_desc(baton, baton->query.root);
series_query_end_phase(baton);
}
@@ -4380,7 +4395,7 @@ series_map_lookup_expr_reply(redisCluste
baton->error = sts;
} else {
/* call the on_metric callback, whatever it's set to by the caller */
- baton->u.lookup.func(sid->name, query, baton->userdata);
+ baton->lookup.func(sid->name, query, baton->userdata);
}
}
series_query_end_phase(baton);
@@ -4434,8 +4449,8 @@ series_map_reply(seriesQueryBaton *baton
if (reply->type == REDIS_REPLY_STRING) {
sdsclear(key);
key = sdscatlen(key, reply->str, reply->len);
- if ((entry = redisMapLookup(baton->u.lookup.map, key)) != NULL)
- baton->u.lookup.func(series, redisMapValue(entry), baton->userdata);
+ if ((entry = redisMapLookup(baton->lookup.map, key)) != NULL)
+ baton->lookup.func(series, redisMapValue(entry), baton->userdata);
else {
infofmt(msg, "%s - timeseries string map", series);
batoninfo(baton, PMLOG_CORRUPT, msg);
@@ -4470,11 +4485,11 @@ series_map_keys_callback(
for (i = 0; i < reply->elements; i++) {
child = reply->element[i];
if (child->type == REDIS_REPLY_STRING) {
- if (baton->u.lookup.pattern != NULL &&
- fnmatch(baton->u.lookup.pattern, child->str, 0) != 0)
+ if (baton->lookup.pattern != NULL &&
+ fnmatch(baton->lookup.pattern, child->str, 0) != 0)
continue;
val = sdscpylen(val, child->str, child->len);
- baton->u.lookup.func(NULL, val, baton->userdata);
+ baton->lookup.func(NULL, val, baton->userdata);
} else {
infofmt(msg, "bad response for string map %s (%s)",
HVALS, redis_reply_type(child));
@@ -4599,7 +4614,7 @@ series_label_reply(seriesQueryBaton *bat
sdsclear(vmapID);
vmapID = sdscatlen(vmapID, elements[index+1]->str, elements[index+1]->len);
- if ((entry = redisMapLookup(baton->u.lookup.map, nmapID)) != NULL) {
+ if ((entry = redisMapLookup(baton->lookup.map, nmapID)) != NULL) {
pmwebapi_hash_str((unsigned char *)nmapID, hashbuf, sizeof(hashbuf));
vkey = sdscatfmt(sdsempty(), "label.%s.value", hashbuf);
vmap = redisMapCreate(vkey);
@@ -4680,9 +4695,9 @@ series_lookup_labels(void *arg)
seriesBatonCheckCount(baton, "series_lookup_labels");
/* unpack - iterate over series and extract labels names and values */
- for (i = 0; i < baton->u.lookup.nseries; i++) {
+ for (i = 0; i < baton->lookup.nseries; i++) {
seriesBatonReference(baton, "series_lookup_labels");
- sid = &baton->u.lookup.series[i];
+ sid = &baton->lookup.series[i];
key = sdscatfmt(sdsempty(), "pcp:labelvalue:series:%S", sid->name);
cmd = redis_command(2);
cmd = redis_param_str(cmd, HGETALL, HGETALL_LEN);
@@ -4712,7 +4727,7 @@ pmSeriesLabels(pmSeriesSettings *setting
initSeriesGetLookup(baton, nseries, series, settings->callbacks.on_label, labelsmap);
if (nseries == 0)
- return series_map_keys(baton, redisMapName(baton->u.lookup.map));
+ return series_map_keys(baton, redisMapName(baton->lookup.map));
baton->current = &baton->phases[0];
baton->phases[i++].func = series_lookup_services;
@@ -4780,8 +4795,8 @@ series_lookup_labelvalues(void *arg)
seriesBatonCheckMagic(baton, MAGIC_QUERY, "series_lookup_labelvalues");
seriesBatonCheckCount(baton, "series_lookup_labelvalues");
- for (i = 0; i < baton->u.lookup.nseries; i++) {
- sid = &baton->u.lookup.series[i];
+ for (i = 0; i < baton->lookup.nseries; i++) {
+ sid = &baton->lookup.series[i];
seriesBatonReference(baton, "series_lookup_labelvalues");
pmwebapi_string_hash(hash, sid->name, sdslen(sid->name));
@@ -4874,8 +4889,8 @@ series_lookup_desc(void *arg)
seriesBatonCheckMagic(baton, MAGIC_QUERY, "series_lookup_desc");
seriesBatonCheckCount(baton, "series_lookup_desc");
- for (i = 0; i < baton->u.lookup.nseries; i++) {
- sid = &baton->u.lookup.series[i];
+ for (i = 0; i < baton->lookup.nseries; i++) {
+ sid = &baton->lookup.series[i];
seriesBatonReference(baton, "series_lookup_desc");
key = sdscatfmt(sdsempty(), "pcp:desc:series:%S", sid->name);
@@ -5054,7 +5069,7 @@ series_inst_expr_reply(redisClusterAsync
baton->error = sts;
} else {
/* Parse the expr (with timing) and series solve the resulting expr tree */
- baton->u.query.timing.count = 1;
+ baton->query.timing.count = 1;
baton->error = series_solve_sid_expr(&series_solve_inst_settings, &expr, arg);
}
}
@@ -5121,9 +5136,9 @@ series_lookup_instances(void *arg)
seriesBatonCheckMagic(baton, MAGIC_QUERY, "series_lookup_instances_callback");
seriesBatonCheckCount(baton, "series_lookup_instances_callback");
- for (i = 0; i < baton->u.lookup.nseries; i++) {
+ for (i = 0; i < baton->lookup.nseries; i++) {
seriesBatonReference(baton, "series_lookup_instances_callback");
- sid = &baton->u.lookup.series[i];
+ sid = &baton->lookup.series[i];
key = sdscatfmt(sdsempty(), "pcp:instances:series:%S", sid->name);
cmd = redis_command(2);
cmd = redis_param_str(cmd, SMEMBERS, SMEMBERS_LEN);
@@ -5153,7 +5168,7 @@ pmSeriesInstances(pmSeriesSettings *sett
initSeriesGetLookup(baton, nseries, series, settings->callbacks.on_instance, instmap);
if (nseries == 0)
- return series_map_keys(baton, redisMapName(baton->u.lookup.map));
+ return series_map_keys(baton, redisMapName(baton->lookup.map));
baton->current = &baton->phases[0];
baton->phases[i++].func = series_lookup_services;
@@ -5177,7 +5192,7 @@ redis_lookup_mapping_callback(
/* unpack - produce reverse map of ids-to-names for each context */
if (LIKELY(reply && reply->type == REDIS_REPLY_ARRAY)) {
- reverse_map(baton, baton->u.lookup.map, reply->elements, reply->element);
+ reverse_map(baton, baton->lookup.map, reply->elements, reply->element);
} else {
infofmt(msg, "expected array from %s %s (type=%s)",
HGETALL, "pcp:map:context.name", redis_reply_type(reply));
@@ -5198,7 +5213,7 @@ series_lookup_mapping(void *arg)
seriesBatonCheckCount(baton, "series_lookup_mapping");
seriesBatonReference(baton, "series_lookup_mapping");
- key = sdscatfmt(sdsempty(), "pcp:map:%s", redisMapName(baton->u.lookup.map));
+ key = sdscatfmt(sdsempty(), "pcp:map:%s", redisMapName(baton->lookup.map));
cmd = redis_command(2);
cmd = redis_param_str(cmd, HGETALL, HGETALL_LEN);
cmd = redis_param_sds(cmd, key);
@@ -5328,9 +5343,9 @@ series_lookup_sources(void *arg)
seriesBatonCheckCount(baton, "series_lookup_sources");
seriesBatonReference(baton, "series_lookup_sources");
- for (i = 0; i < baton->u.lookup.nseries; i++) {
+ for (i = 0; i < baton->lookup.nseries; i++) {
seriesBatonReference(baton, "series_lookup_sources");
- sid = &baton->u.lookup.series[i];
+ sid = &baton->lookup.series[i];
key = sdscatfmt(sdsempty(), "pcp:context.name:source:%S", sid->name);
cmd = redis_command(2);
cmd = redis_param_str(cmd, SMEMBERS, SMEMBERS_LEN);
@@ -5361,7 +5376,7 @@ pmSeriesSources(pmSeriesSettings *settin
initSeriesGetLookup(baton, nsources, sources, settings->callbacks.on_context, contextmap);
if (nsources == 0)
- return series_map_keys(baton, redisMapName(baton->u.lookup.map));
+ return series_map_keys(baton, redisMapName(baton->lookup.map));
baton->current = &baton->phases[0];
baton->phases[i++].func = series_lookup_services;
@@ -5384,9 +5399,9 @@ series_lookup_metrics(void *arg)
seriesBatonCheckMagic(baton, MAGIC_QUERY, "series_lookup_metrics");
seriesBatonCheckCount(baton, "series_lookup_metrics");
- for (i = 0; i < baton->u.lookup.nseries; i++) {
+ for (i = 0; i < baton->lookup.nseries; i++) {
seriesBatonReference(baton, "series_lookup_metrics");
- sid = &baton->u.lookup.series[i];
+ sid = &baton->lookup.series[i];
key = sdscatfmt(sdsempty(), "pcp:metric.name:series:%S", sid->name);
cmd = redis_command(2);
cmd = redis_param_str(cmd, SMEMBERS, SMEMBERS_LEN);
@@ -5416,7 +5431,7 @@ pmSeriesMetrics(pmSeriesSettings *settin
initSeriesGetLookup(baton, nseries, series, settings->callbacks.on_metric, namesmap);
if (nseries == 0)
- return series_map_keys(baton, redisMapName(baton->u.lookup.map));
+ return series_map_keys(baton, redisMapName(baton->lookup.map));
baton->current = &baton->phases[0];
baton->phases[i++].func = series_lookup_services;
@@ -5517,55 +5532,66 @@ parseseries(seriesQueryBaton *baton, sds
}
static void
-initSeriesGetValues(seriesQueryBaton *baton, int nseries, sds *series,
+initSeriesGetValues(seriesQueryBaton *baton, int nseries, sds *inseries,
pmSeriesTimeWindow *window)
{
- struct series_set *result = &baton->u.query.root.result;
- struct timing *timing = &baton->u.query.timing;
+ struct node *node = NULL;
+ struct timing timing = {0};
+ unsigned char *series = NULL;
+ struct series_set *result;
struct timeval offset;
int i;
-
- /* validate and convert 40-byte (ASCII) SIDs to internal 20-byte form */
- result->nseries = nseries;
- if ((result->series = calloc(nseries, 20)) == NULL) {
+
+ /* allocate a local parse node, timing and result SIDs */
+ if (!(node = (node_t *)calloc(1, sizeof(node_t))))
+ baton->error = -ENOMEM;
+ else if (!(series = calloc(nseries, 20))) /* 20 byte SIDs */
baton->error = -ENOMEM;
- } else {
- for (i = 0; i < nseries; i++)
- parseseries(baton, series[i], result->series + (i * 20));
- }
if (baton->error) {
- if (result->series)
- free(result->series);
+ if (series) free(series);
+ if (node) free(node);
return;
- }
+ }
+
+ /* track this memory in the baton for async free later */
+ result = &node->result;
+ result->series = series;
+ result->nseries = nseries;
+
+ /* validate and convert 40-byte (ASCII) SIDs to internal 20-byte form */
+ for (i = 0; i < nseries; i++)
+ parseseries(baton, inseries[i], result->series + (i * 20));
/* validate and convert time window specification to internal struct */
- timing->window = *window;
+ timing.window = *window;
if (window->delta)
- parsedelta(baton, window->delta, &timing->delta, "delta");
+ parsedelta(baton, window->delta, &timing.delta, "delta");
if (window->align)
- parsetime(baton, window->align, &timing->align, "align");
+ parsetime(baton, window->align, &timing.align, "align");
if (window->start)
- parsetime(baton, window->start, &timing->start, "start");
+ parsetime(baton, window->start, &timing.start, "start");
if (window->end)
- parsetime(baton, window->end, &timing->end, "end");
+ parsetime(baton, window->end, &timing.end, "end");
if (window->range) {
- parsedelta(baton, window->range, &timing->start, "range");
+ parsedelta(baton, window->range, &timing.start, "range");
gettimeofday(&offset, NULL);
- tsub(&offset, &timing->start);
- timing->start = offset;
- timing->end.tv_sec = INT_MAX;
+ tsub(&offset, &timing.start);
+ timing.start = offset;
+ timing.end.tv_sec = INT_MAX;
}
if (window->count)
- parseuint(baton, window->count, &timing->count, "count");
+ parseuint(baton, window->count, &timing.count, "count");
if (window->offset)
- parseuint(baton, window->offset, &timing->offset, "offset");
+ parseuint(baton, window->offset, &timing.offset, "offset");
if (window->zone)
- parsezone(baton, window->zone, &timing->zone, "timezone");
+ parsezone(baton, window->zone, &timing.zone, "timezone");
/* if no time window parameters passed, default to latest value */
- if (!series_time_window(timing))
- timing->count = 1;
+ if (!series_time_window(&timing))
+ timing.count = 1;
+
+ baton->query.timing = timing; /* struct copy */
+ baton->query.root = node;
}
int
diff -Naurp pcp-5.3.7.orig/src/libpcp_web/src/schema.h pcp-5.3.7/src/libpcp_web/src/schema.h
--- pcp-5.3.7.orig/src/libpcp_web/src/schema.h 2023-07-05 13:42:25.533986251 +1000
+++ pcp-5.3.7/src/libpcp_web/src/schema.h 2023-07-05 13:42:53.414025716 +1000
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 2017-2021 Red Hat.
+ * Copyright (c) 2017-2022 Red Hat.
*
* This library is free software; you can redistribute it and/or modify it
* under the terms of the GNU Lesser General Public License as published
@@ -126,6 +126,7 @@ typedef struct seriesGetContext {
context_t context;
unsigned long long count; /* number of samples processed */
pmResult *result; /* currently active sample data */
+ int loaded; /* end of archive data reached */
int error; /* PMAPI error code from fetch */
redisDoneCallBack done;
@@ -152,6 +153,9 @@ typedef struct seriesLoadBaton {
const char **metrics; /* metric specification strings */
dict *errors; /* PMIDs where errors observed */
dict *wanted; /* allowed metrics list PMIDs */
+ sds *exclude_patterns; /* list of exclude metric patterns (e.g. proc.*) */
+ unsigned int exclude_npatterns; /* number of exclude metric patterns */
+ dict *exclude_pmids; /* dict of excluded pmIDs (pmID: NULL) */
int error;
void *arg;
diff -Naurp pcp-5.3.7.orig/src/libpcp_web/src/slots.c pcp-5.3.7/src/libpcp_web/src/slots.c
--- pcp-5.3.7.orig/src/libpcp_web/src/slots.c 2023-07-05 13:42:25.533986251 +1000
+++ pcp-5.3.7/src/libpcp_web/src/slots.c 2023-07-05 13:42:53.414025716 +1000
@@ -633,8 +633,10 @@ redisSlotsProxyConnect(redisSlots *slots
redisReader *reader = *readerp;
redisReply *reply = NULL;
dictEntry *entry;
- long long position, offset, length;
- sds cmd, key, msg;
+ size_t replyStartPosition;
+ long long position;
+ sds cmd, msg;
+ int hasKey;
int sts;
if (!reader &&
@@ -644,33 +646,45 @@ redisSlotsProxyConnect(redisSlots *slots
return -ENOMEM;
}
- offset = reader->pos;
- length = sdslen(reader->buf);
- if (redisReaderFeed(reader, buffer, nread) != REDIS_OK ||
- redisReaderGetReply(reader, (void **)&reply) != REDIS_OK) {
+ if (redisReaderFeed(reader, buffer, nread) != REDIS_OK) {
infofmt(msg, "failed to parse Redis protocol request");
info(PMLOG_REQUEST, msg, arg), sdsfree(msg);
return -EPROTO;
}
- if (reply != NULL) { /* client request is complete */
- key = cmd = NULL;
+ /* parse all Redis requests contained in buffer (Redis pipelining) */
+ while (1) {
+ replyStartPosition = reader->pos;
+ sts = redisReaderGetReply(reader, (void **)&reply);
+ if (sts != REDIS_OK) {
+ infofmt(msg, "failed to parse Redis protocol request");
+ info(PMLOG_REQUEST, msg, arg), sdsfree(msg);
+ return -EPROTO;
+ }
+ if (reply == NULL) {
+ break;
+ }
+
+ cmd = NULL;
+ hasKey = 0;
if (reply->type == REDIS_REPLY_ARRAY ||
reply->type == REDIS_REPLY_MAP ||
reply->type == REDIS_REPLY_SET)
cmd = sdsnew(reply->element[0]->str);
if (cmd && (entry = dictFind(slots->keymap, cmd)) != NULL) {
position = dictGetSignedIntegerVal(entry);
- if (position < reply->elements)
- key = sdsnew(reply->element[position]->str);
+ if (position > 0 && position < reply->elements)
+ hasKey = 1;
}
sdsfree(cmd);
- cmd = sdsnewlen(reader->buf + offset, sdslen(reader->buf) - length);
- if (key != NULL && position > 0)
+
+ cmd = sdsnewlen(reader->buf + replyStartPosition, reader->pos - replyStartPosition);
+ if (hasKey)
sts = redisSlotsRequest(slots, cmd, callback, arg);
else
sts = redisSlotsRequestFirstNode(slots, cmd, callback, arg);
- sdsfree(key);
+ sdsfree(cmd);
+
if (sts != REDIS_OK) {
redisReply *errorReply = calloc(1, sizeof(redisReply));
errorReply->type = REDIS_REPLY_ERROR;
diff -Naurp pcp-5.3.7.orig/src/libpcp_web/src/timer.c pcp-5.3.7/src/libpcp_web/src/timer.c
--- pcp-5.3.7.orig/src/libpcp_web/src/timer.c 2023-07-05 13:42:25.533986251 +1000
+++ pcp-5.3.7/src/libpcp_web/src/timer.c 2023-07-05 13:42:53.414025716 +1000
@@ -13,6 +13,7 @@
*/
#include <sys/resource.h>
#include "load.h"
+#include "maps.h"
#include "libpcp.h"
#include "mmv_stats.h"
@@ -23,6 +24,10 @@ typedef enum server_metric {
SERVER_CPU_TOT,
SERVER_MEM_MAXRSS,
SERVER_MEM_DATASZ,
+ SERVER_MAP_CONTEXT_SIZE,
+ SERVER_MAP_METRIC_SIZE,
+ SERVER_MAP_LABEL_SIZE,
+ SERVER_MAP_INST_SIZE,
NUM_SERVER_METRIC
} server_metric_t;
@@ -156,6 +161,7 @@ server_metrics_refresh(void *map)
{
double usr, sys;
unsigned long long datasz = 0;
+ unsigned int value;
struct rusage usage = {0};
__pmProcessDataSize((unsigned long*) &datasz);
@@ -173,6 +179,16 @@ server_metrics_refresh(void *map)
/* exported as uint64 but manipulated as ulong/ulong long */
mmv_set(map, server.metrics[SERVER_MEM_DATASZ], &datasz);
+
+ /* update global maps size metrics */
+ value = contextmap? dictSize(contextmap) : 0;
+ mmv_set(map, server.metrics[SERVER_MAP_CONTEXT_SIZE], &value);
+ value = namesmap? dictSize(namesmap) : 0;
+ mmv_set(map, server.metrics[SERVER_MAP_METRIC_SIZE], &value);
+ value = labelsmap? dictSize(labelsmap) : 0;
+ mmv_set(map, server.metrics[SERVER_MAP_LABEL_SIZE], &value);
+ value = instmap? dictSize(instmap) : 0;
+ mmv_set(map, server.metrics[SERVER_MAP_INST_SIZE], &value);
}
/*
@@ -181,6 +197,7 @@ server_metrics_refresh(void *map)
int
pmWebTimerSetMetricRegistry(struct mmv_registry *registry)
{
+ pmAtomValue **ap;
pmUnits nounits = MMV_UNITS(0,0,0,0,0,0);
pmUnits units_kbytes = MMV_UNITS(1, 0, 0, PM_SPACE_KBYTE, 0, 0);
pmUnits units_msec = MMV_UNITS(0, 1, 0, 0, PM_TIME_MSEC, 0);
@@ -228,18 +245,46 @@ pmWebTimerSetMetricRegistry(struct mmv_r
"virtual data size",
"Process memory virtual data size from sbrk(2)");
+ /*
+ * Reverse mapping dict metrics
+ */
+ mmv_stats_add_metric(registry, "map.context.size", SERVER_MAP_CONTEXT_SIZE,
+ MMV_TYPE_U32, MMV_SEM_INSTANT, nounits, MMV_INDOM_NULL,
+ "context map dictionary size",
+ "Number of entries in the context map dictionary");
+
+ mmv_stats_add_metric(registry, "map.metric.size", SERVER_MAP_METRIC_SIZE,
+ MMV_TYPE_U32, MMV_SEM_INSTANT, nounits, MMV_INDOM_NULL,
+ "metric names map dictionary size",
+ "Number of entries in the metric names map dictionary");
+
+ mmv_stats_add_metric(registry, "map.label.size", SERVER_MAP_LABEL_SIZE,
+ MMV_TYPE_U32, MMV_SEM_INSTANT, nounits, MMV_INDOM_NULL,
+ "label names map dictionary size",
+ "Number of entries in the labels map dictionary");
+
+ mmv_stats_add_metric(registry, "map.instance.size", SERVER_MAP_INST_SIZE,
+ MMV_TYPE_U32, MMV_SEM_INSTANT, nounits, MMV_INDOM_NULL,
+ "instance names map dictionary size",
+ "Number of entries in the instance name map dictionary");
+
if ((map = mmv_stats_start(registry)) == NULL) {
pmNotifyErr(LOG_ERR, "%s: server instrumentation disabled",
"pmWebTimerSetMetricRegistry");
return -EINVAL;
}
- server.metrics[SERVER_PID] = mmv_lookup_value_desc(map, "pid", NULL);
- server.metrics[SERVER_CPU_USR] = mmv_lookup_value_desc(map, "cpu.user", NULL);
- server.metrics[SERVER_CPU_SYS] = mmv_lookup_value_desc(map, "cpu.sys", NULL);
- server.metrics[SERVER_CPU_TOT] = mmv_lookup_value_desc(map, "cpu.total", NULL);
- server.metrics[SERVER_MEM_MAXRSS] = mmv_lookup_value_desc(map, "mem.maxrss", NULL);
- server.metrics[SERVER_MEM_DATASZ] = mmv_lookup_value_desc(map, "mem.datasz", NULL);
+ ap = server.metrics;
+ ap[SERVER_PID] = mmv_lookup_value_desc(map, "pid", NULL);
+ ap[SERVER_CPU_USR] = mmv_lookup_value_desc(map, "cpu.user", NULL);
+ ap[SERVER_CPU_SYS] = mmv_lookup_value_desc(map, "cpu.sys", NULL);
+ ap[SERVER_CPU_TOT] = mmv_lookup_value_desc(map, "cpu.total", NULL);
+ ap[SERVER_MEM_MAXRSS] = mmv_lookup_value_desc(map, "mem.maxrss", NULL);
+ ap[SERVER_MEM_DATASZ] = mmv_lookup_value_desc(map, "mem.datasz", NULL);
+ ap[SERVER_MAP_CONTEXT_SIZE] = mmv_lookup_value_desc(map, "map.context.size", NULL);
+ ap[SERVER_MAP_METRIC_SIZE] = mmv_lookup_value_desc(map, "map.metric.size", NULL);
+ ap[SERVER_MAP_LABEL_SIZE] = mmv_lookup_value_desc(map, "map.label.size", NULL);
+ ap[SERVER_MAP_INST_SIZE] = mmv_lookup_value_desc(map, "map.instance.size", NULL);
/* PID doesn't change, set it once */
mmv_set(map, server.metrics[SERVER_PID], &pid);
diff -Naurp pcp-5.3.7.orig/src/libpcp_web/src/webgroup.c pcp-5.3.7/src/libpcp_web/src/webgroup.c
--- pcp-5.3.7.orig/src/libpcp_web/src/webgroup.c 2023-07-05 13:42:25.533986251 +1000
+++ pcp-5.3.7/src/libpcp_web/src/webgroup.c 2023-07-05 13:42:53.414025716 +1000
@@ -44,10 +44,8 @@ enum matches { MATCH_EXACT, MATCH_GLOB,
enum profile { PROFILE_ADD, PROFILE_DEL };
enum webgroup_metric {
- CONTEXT_MAP_SIZE,
- NAMES_MAP_SIZE,
- LABELS_MAP_SIZE,
- INST_MAP_SIZE,
+ WEBGROUP_GC_COUNT,
+ WEBGROUP_GC_DROPS,
NUM_WEBGROUP_METRIC
};
@@ -64,7 +62,6 @@ typedef struct webgroups {
uv_mutex_t mutex;
unsigned int active;
- int timerid;
} webgroups;
static struct webgroups *
@@ -76,7 +73,6 @@ webgroups_lookup(pmWebGroupModule *modul
module->privdata = calloc(1, sizeof(struct webgroups));
groups = (struct webgroups *)module->privdata;
uv_mutex_init(&groups->mutex);
- groups->timerid = -1;
}
return groups;
}
@@ -292,8 +288,6 @@ webgroup_timers_stop(struct webgroups *g
if (groups->active) {
uv_timer_stop(&groups->timer);
uv_close((uv_handle_t *)&groups->timer, NULL);
- pmWebTimerRelease(groups->timerid);
- groups->timerid = -1;
groups->active = 0;
}
}
@@ -336,28 +330,15 @@ webgroup_garbage_collect(struct webgroup
uv_mutex_unlock(&groups->mutex);
}
+ mmv_set(groups->map, groups->metrics[WEBGROUP_GC_DROPS], &drops);
+ mmv_set(groups->map, groups->metrics[WEBGROUP_GC_COUNT], &count);
+
if (pmDebugOptions.http || pmDebugOptions.libweb)
fprintf(stderr, "%s: finished [%u drops from %u entries]\n",
"webgroup_garbage_collect", drops, count);
}
static void
-refresh_maps_metrics(void *data)
-{
- struct webgroups *groups = (struct webgroups *)data;
- unsigned int value;
-
- value = contextmap? dictSize(contextmap) : 0;
- mmv_set(groups->map, groups->metrics[CONTEXT_MAP_SIZE], &value);
- value = namesmap? dictSize(namesmap) : 0;
- mmv_set(groups->map, groups->metrics[NAMES_MAP_SIZE], &value);
- value = labelsmap? dictSize(labelsmap) : 0;
- mmv_set(groups->map, groups->metrics[LABELS_MAP_SIZE], &value);
- value = instmap? dictSize(instmap) : 0;
- mmv_set(groups->map, groups->metrics[INST_MAP_SIZE], &value);
-}
-
-static void
webgroup_worker(uv_timer_t *arg)
{
uv_handle_t *handle = (uv_handle_t *)arg;
@@ -425,8 +406,6 @@ webgroup_lookup_context(pmWebGroupSettin
groups->timer.data = (void *)groups;
uv_timer_start(&groups->timer, webgroup_worker,
default_worker, default_worker);
- /* timer for map stats refresh */
- groups->timerid = pmWebTimerRegister(refresh_maps_metrics, groups);
}
if (*id == NULL) {
@@ -2360,36 +2339,21 @@ pmWebGroupSetupMetrics(pmWebGroupModule
if (groups == NULL || groups->registry == NULL)
return; /* no metric registry has been set up */
- /*
- * Reverse mapping dict metrics
- */
- mmv_stats_add_metric(groups->registry, "contextmap.size", 1,
- MMV_TYPE_U32, MMV_SEM_INSTANT, nounits, MMV_INDOM_NULL,
- "context map dictionary size",
- "Number of entries in the context map dictionary");
-
- mmv_stats_add_metric(groups->registry, "namesmap.size", 2,
- MMV_TYPE_U32, MMV_SEM_INSTANT, nounits, MMV_INDOM_NULL,
- "metric names map dictionary size",
- "Number of entries in the metric names map dictionary");
-
- mmv_stats_add_metric(groups->registry, "labelsmap.size", 3,
+ mmv_stats_add_metric(groups->registry, "gc.context.scans", 1,
MMV_TYPE_U32, MMV_SEM_INSTANT, nounits, MMV_INDOM_NULL,
- "labels map dictionary size",
- "Number of entries in the labels map dictionary");
+ "contexts scanned in last garbage collection",
+ "Contexts scanned during most recent webgroup garbage collection");
- mmv_stats_add_metric(groups->registry, "instmap.size", 4,
+ mmv_stats_add_metric(groups->registry, "gc.context.drops", 2,
MMV_TYPE_U32, MMV_SEM_INSTANT, nounits, MMV_INDOM_NULL,
- "instance name map dictionary size",
- "Number of entries in the instance name map dictionary");
+ "contexts dropped in last garbage collection",
+ "Contexts dropped during most recent webgroup garbage collection");
groups->map = map = mmv_stats_start(groups->registry);
ap = groups->metrics;
- ap[CONTEXT_MAP_SIZE] = mmv_lookup_value_desc(map, "contextmap.size", NULL);
- ap[NAMES_MAP_SIZE] = mmv_lookup_value_desc(map, "namesmap.size", NULL);
- ap[LABELS_MAP_SIZE] = mmv_lookup_value_desc(map, "labelsmap.size", NULL);
- ap[INST_MAP_SIZE] = mmv_lookup_value_desc(map, "instmap.size", NULL);
+ ap[WEBGROUP_GC_DROPS] = mmv_lookup_value_desc(map, "gc.context.scans", NULL);
+ ap[WEBGROUP_GC_COUNT] = mmv_lookup_value_desc(map, "gc.context.drops", NULL);
}
diff -Naurp pcp-5.3.7.orig/src/pmproxy/src/http.c pcp-5.3.7/src/pmproxy/src/http.c
--- pcp-5.3.7.orig/src/pmproxy/src/http.c 2022-04-05 09:05:43.000000000 +1000
+++ pcp-5.3.7/src/pmproxy/src/http.c 2023-07-05 13:43:00.414035625 +1000
@@ -581,7 +581,7 @@ http_add_parameter(dict *parameters,
return 0;
}
-static int
+int
http_parameters(const char *url, size_t length, dict **parameters)
{
const char *end = url + length;
diff -Naurp pcp-5.3.7.orig/src/pmproxy/src/http.h pcp-5.3.7/src/pmproxy/src/http.h
--- pcp-5.3.7.orig/src/pmproxy/src/http.h 2022-04-05 09:05:43.000000000 +1000
+++ pcp-5.3.7/src/pmproxy/src/http.h 2023-07-05 13:43:00.414035625 +1000
@@ -64,6 +64,7 @@ extern void http_transfer(struct client
extern void http_reply(struct client *, sds, http_code_t, http_flags_t, http_options_t);
extern void http_error(struct client *, http_code_t, const char *);
+extern int http_parameters(const char *, size_t, dict **);
extern int http_decode(const char *, size_t, sds);
extern const char *http_status_mapping(http_code_t);
extern const char *http_content_type(http_flags_t);
diff -Naurp pcp-5.3.7.orig/src/pmproxy/src/search.c pcp-5.3.7/src/pmproxy/src/search.c
--- pcp-5.3.7.orig/src/pmproxy/src/search.c 2023-07-05 13:42:25.533986251 +1000
+++ pcp-5.3.7/src/pmproxy/src/search.c 2023-07-05 13:42:53.414025716 +1000
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 2020 Red Hat.
+ * Copyright (c) 2020,2022 Red Hat.
*
* This program is free software; you can redistribute it and/or modify it
* under the terms of the GNU Lesser General Public License as published
@@ -253,6 +253,9 @@ on_pmsearch_done(int status, void *arg)
flags |= HTTP_FLAG_JSON;
}
http_reply(client, msg, code, flags, options);
+
+ /* release lock of pmsearch_request_done */
+ client_put(client);
}
static void
@@ -481,6 +484,9 @@ pmsearch_request_done(struct client *cli
pmSearchBaton *baton = (pmSearchBaton *)client->u.http.data;
int sts;
+ /* reference to prevent freeing while waiting for a Redis reply callback */
+ client_get(client);
+
if (client->u.http.parser.status_code) {
on_pmsearch_done(-EINVAL, baton);
return 1;
diff -Naurp pcp-5.3.7.orig/src/pmproxy/src/series.c pcp-5.3.7/src/pmproxy/src/series.c
--- pcp-5.3.7.orig/src/pmproxy/src/series.c 2023-07-05 13:42:25.533986251 +1000
+++ pcp-5.3.7/src/pmproxy/src/series.c 2023-07-05 13:43:00.414035625 +1000
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 2019-2020 Red Hat.
+ * Copyright (c) 2019-2020,2022 Red Hat.
*
* This program is free software; you can redistribute it and/or modify it
* under the terms of the GNU Lesser General Public License as published
@@ -12,6 +12,7 @@
* License for more details.
*/
#include "server.h"
+#include "util.h"
#include <assert.h>
typedef enum pmSeriesRestKey {
@@ -592,6 +593,9 @@ on_pmseries_done(int status, void *arg)
flags |= HTTP_FLAG_JSON;
}
http_reply(client, msg, code, flags, options);
+
+ /* release lock of pmseries_request_done */
+ client_put(client);
}
static void
@@ -824,30 +828,26 @@ static int
pmseries_request_body(struct client *client, const char *content, size_t length)
{
pmSeriesBaton *baton = (pmSeriesBaton *)client->u.http.data;
- sds series;
if (pmDebugOptions.http)
fprintf(stderr, "series servlet body (client=%p)\n", client);
- if (client->u.http.parser.method != HTTP_POST)
+ if (client->u.http.parser.method != HTTP_POST || client->u.http.parameters != NULL)
return 0;
switch (baton->restkey) {
case RESTKEY_LOAD:
case RESTKEY_QUERY:
- sdsfree(baton->query);
- baton->query = sdsnewlen(content, length);
- break;
-
case RESTKEY_DESC:
case RESTKEY_INSTS:
case RESTKEY_LABELS:
case RESTKEY_METRIC:
case RESTKEY_SOURCE:
case RESTKEY_VALUES:
- series = sdsnewlen(content, length);
- baton->sids = sdssplitlen(series, length, "\n", 1, &baton->nsids);
- sdsfree(series);
+ /* parse URL encoded parameters in the request body */
+ /* in the same way as the URL query string */
+ http_parameters(content, length, &client->u.http.parameters);
+ pmseries_setup_request_parameters(client, baton, client->u.http.parameters);
break;
default:
@@ -890,10 +890,16 @@ pmseries_request_load(struct client *cli
message = sdsnewlen(failed, sizeof(failed) - 1);
http_reply(client, message, HTTP_STATUS_BAD_REQUEST,
HTTP_FLAG_JSON, baton->options);
+
+ /* release lock of pmseries_request_done */
+ client_put(client);
} else if (baton->working) {
message = sdsnewlen(loading, sizeof(loading) - 1);
http_reply(client, message, HTTP_STATUS_CONFLICT,
HTTP_FLAG_JSON, baton->options);
+
+ /* release lock of pmseries_request_done */
+ client_put(client);
} else {
baton->loading.data = baton;
uv_queue_work(client->proxy->events, &baton->loading,
@@ -907,6 +913,9 @@ pmseries_request_done(struct client *cli
pmSeriesBaton *baton = (pmSeriesBaton *)client->u.http.data;
int sts;
+ /* reference to prevent freeing while waiting for a Redis reply callback */
+ client_get(client);
+
if (client->u.http.parser.status_code) {
on_pmseries_done(-EINVAL, baton);
return 1;
diff -Naurp pcp-5.3.7.orig/src/pmproxy/src/server.c pcp-5.3.7/src/pmproxy/src/server.c
--- pcp-5.3.7.orig/src/pmproxy/src/server.c 2023-07-05 13:42:25.533986251 +1000
+++ pcp-5.3.7/src/pmproxy/src/server.c 2023-07-05 13:42:53.424025730 +1000
@@ -180,7 +180,9 @@ signal_init(struct proxy *proxy)
{
uv_loop_t *loop = proxy->events;
+#if defined(HAVE_SIGPIPE)
signal(SIGPIPE, SIG_IGN);
+#endif
uv_signal_init(loop, &sighup);
uv_signal_init(loop, &sigint);
@@ -310,6 +312,20 @@ on_write_callback(uv_callback_t *handle,
struct client *client = (struct client *)request->writer.data;
int sts;
+ /*
+ * client_write() checks if the client is opened, and calls
+ * uv_callback_fire(&proxy->write_callbacks, ...).
+ * In a later loop iteration, on_write_callback() is called and tries
+ * to write to the client. However, the client can be closed between
+ * the call to uv_callback_fire() and the actual on_write_callback()
+ * callback. Therefore we need to check this condition again.
+ */
+ if (client_is_closed(client)) {
+ /* release lock of client_write */
+ client_put(client);
+ return 0;
+ }
+
(void)handle;
if (pmDebugOptions.af)
fprintf(stderr, "%s: client=%p\n", "on_write_callback", client);
@@ -325,6 +341,9 @@ on_write_callback(uv_callback_t *handle,
}
} else
secure_client_write(client, request);
+
+ /* release lock of client_write */
+ client_put(client);
return 0;
}
@@ -353,6 +372,8 @@ client_write(struct client *client, sds
request->writer.data = client;
request->callback = on_client_write;
+ /* client must not get freed while waiting for the write callback to fire */
+ client_get(client);
uv_callback_fire(&proxy->write_callbacks, request, NULL);
} else {
client_close(client);
Reference in New Issue Copy Permalink