rpms
/
pcp
7
0
Fork 0
Code Issues Pull Requests Releases Activity
pcp/SOURCES/archive-discovery.patch

3176 lines
108 KiB
Diff
Raw Blame History

diff -Naurp pcp-5.0.2.orig/qa/1211.out pcp-5.0.2/qa/1211.out
--- pcp-5.0.2.orig/qa/1211.out 2019-12-06 15:18:26.000000000 +1100
+++ pcp-5.0.2/qa/1211.out 2020-02-03 13:23:15.258762963 +1100
@@ -144,6 +144,9 @@ kernel.uname.nodename
kernel.uname.release
kernel.uname.sysname
kernel.uname.version
+pmcd.pmlogger.archive
+pmcd.pmlogger.host
+pmcd.pmlogger.port
proc.fd.count
proc.id.egid
proc.id.egid_nm
@@ -267,6 +270,7 @@ List all instance names ...
030016 pmlogger -P -c config.default 20110930.17.20
1 minute
15 minute
+2950
5 minute
cpu0
cpu1
@@ -398,10 +402,10 @@ fecd5a4b4c6e1273eaa001287a6dd57b7bbd19f7
Values fetch for a single-valued query ...
d51624d12da45900bfee2fd73f1e23f3ccabb784
- [Mon Oct 3 09:10:22.959242000 2011] 172598244
- [Mon Oct 3 09:10:23.300460000 2011] 172598364
- [Mon Oct 3 09:10:23.802930000 2011] 172598481
[Mon Oct 3 09:10:24.305845000 2011] 172598559
+ [Mon Oct 3 09:10:23.802930000 2011] 172598481
+ [Mon Oct 3 09:10:23.300460000 2011] 172598364
+ [Mon Oct 3 09:10:22.959242000 2011] 172598244
Values fetch with a one-second interval ...
@@ -420,15 +424,18 @@ d51624d12da45900bfee2fd73f1e23f3ccabb784
Values fetch for a multi-valued query ...
fecd5a4b4c6e1273eaa001287a6dd57b7bbd19f7
- [Mon Oct 3 09:10:23.300460000 2011] 0.000000e+00 59181b1de54ff2b383cfd1cdd8636f86c880b69b
- [Mon Oct 3 09:10:23.300460000 2011] 2.000000e-02 ab010c7d45145aa33c8f8fa681a68c9d4102ae19
- [Mon Oct 3 09:10:23.300460000 2011] 5.000000e-02 9d418095c9f971ff4fd44d6828ead27f9d021dc3
- [Mon Oct 3 09:10:23.802930000 2011] 0.000000e+00 59181b1de54ff2b383cfd1cdd8636f86c880b69b
- [Mon Oct 3 09:10:23.802930000 2011] 2.000000e-02 ab010c7d45145aa33c8f8fa681a68c9d4102ae19
- [Mon Oct 3 09:10:23.802930000 2011] 5.000000e-02 9d418095c9f971ff4fd44d6828ead27f9d021dc3
[Mon Oct 3 09:10:24.305845000 2011] 0.000000e+00 59181b1de54ff2b383cfd1cdd8636f86c880b69b
[Mon Oct 3 09:10:24.305845000 2011] 2.000000e-02 ab010c7d45145aa33c8f8fa681a68c9d4102ae19
[Mon Oct 3 09:10:24.305845000 2011] 5.000000e-02 9d418095c9f971ff4fd44d6828ead27f9d021dc3
+ [Mon Oct 3 09:10:23.802930000 2011] 0.000000e+00 59181b1de54ff2b383cfd1cdd8636f86c880b69b
+ [Mon Oct 3 09:10:23.802930000 2011] 2.000000e-02 ab010c7d45145aa33c8f8fa681a68c9d4102ae19
+ [Mon Oct 3 09:10:23.802930000 2011] 5.000000e-02 9d418095c9f971ff4fd44d6828ead27f9d021dc3
+ [Mon Oct 3 09:10:23.300460000 2011] 0.000000e+00 59181b1de54ff2b383cfd1cdd8636f86c880b69b
+ [Mon Oct 3 09:10:23.300460000 2011] 2.000000e-02 ab010c7d45145aa33c8f8fa681a68c9d4102ae19
+ [Mon Oct 3 09:10:23.300460000 2011] 5.000000e-02 9d418095c9f971ff4fd44d6828ead27f9d021dc3
+ [Mon Oct 3 09:10:22.959242000 2011] 0.000000e+00 59181b1de54ff2b383cfd1cdd8636f86c880b69b
+ [Mon Oct 3 09:10:22.959242000 2011] 2.000000e-02 ab010c7d45145aa33c8f8fa681a68c9d4102ae19
+ [Mon Oct 3 09:10:22.959242000 2011] 5.000000e-02 9d418095c9f971ff4fd44d6828ead27f9d021dc3
Multi-series lookups from a multi-series query ...
2db1da4d276d81c42c578c2829e99188ae7cc898
diff -Naurp pcp-5.0.2.orig/qa/1573 pcp-5.0.2/qa/1573
--- pcp-5.0.2.orig/qa/1573 1970-01-01 10:00:00.000000000 +1000
+++ pcp-5.0.2/qa/1573 2020-02-03 13:36:17.288581801 +1100
@@ -0,0 +1,103 @@
+#!/bin/sh
+# PCP QA Test No. 1573
+# Exercise libpcp_web memory leak without a redis-server.
+#
+# Copyright (c) 2020 Red Hat.
+#
+
+seq=`basename $0`
+echo "QA output created by $seq"
+
+# get standard environment, filters and checks
+. ./common.product
+. ./common.filter
+. ./common.check
+
+_check_series
+
+_cleanup()
+{
+ cd $here
+ if $need_restore
+ then
+ need_restore=false
+ _service pmlogger stop >/dev/null
+ $sudo rm -rf $PCP_LOG_DIR/pmlogger
+ $sudo mv $PCP_LOG_DIR/pmlogger.$seq $PCP_LOG_DIR/pmlogger
+ _restore_config $PCP_ETC_DIR/pcp/pmlogger
+ _service pcp restart 2>&1 | _filter_pcp_stop | _filter_pcp_start
+ _wait_for_pmcd
+ _wait_for_pmlogger
+ echo === restarting pmproxy
+ _restore_config $PCP_SYSCONF_DIR/pmproxy
+ _service pmproxy restart 2>&1 | _filter_pcp_start
+ _wait_for_pmproxy
+ fi
+ $sudo rm -rf $tmp $tmp.*
+}
+
+status=1 # failure is the default!
+need_restore=false
+$sudo rm -rf $tmp $tmp.* $seq.full
+trap "_cleanup; exit \$status" 0 1 2 3 15
+
+# real QA test starts here
+_save_config $PCP_SYSCONF_DIR/pmproxy
+need_restore=true
+
+# only want the primary logger running
+_save_config $PCP_ETC_DIR/pcp/pmlogger
+_restore_pmlogger_control
+
+#$sudo rm -f $PCP_SYSCONF_DIR/pmproxy/*
+echo "[pmproxy]" > $tmp.conf
+echo "pcp.enabled = true" >> $tmp.conf
+echo "http.enabled = true" >> $tmp.conf
+echo "redis.enabled = true" >> $tmp.conf
+echo "[discover]" >> $tmp.conf
+echo "enabled = true" >> $tmp.conf
+echo "[pmseries]" >> $tmp.conf
+echo "enabled = false" >> $tmp.conf
+$sudo cp $tmp.conf $PCP_SYSCONF_DIR/pmproxy/pmproxy.conf
+
+_service pmlogger stop >/dev/null
+
+# move aside existing logs so we can measure base memory footprint
+[ -d $PCP_LOG_DIR/pmlogger.$seq ] && $sudo mv $PCP_LOG_DIR/pmlogger.$seq $PCP_LOG_DIR/pmlogger.$seq.saved
+$sudo mv $PCP_LOG_DIR/pmlogger $PCP_LOG_DIR/pmlogger.$seq
+$sudo mkdir -p $PCP_LOG_DIR/pmlogger
+$sudo chmod 775 $PCP_LOG_DIR/pmlogger
+$sudo chown $PCP_USER:$PCP_USER $PCP_LOG_DIR/pmlogger
+
+_service pmproxy restart 2>&1 | _filter_pcp_stop | _filter_pcp_start
+_wait_for_pmproxy
+
+pmproxy_pid=`_get_pids_by_name -a pmproxy`
+[ -z "$pmproxy_pid" ] && echo === pmproxy not running && status=1 && exit 1
+echo === extract initial rss
+pmproxy_rss1=`pminfo -f proc.memory.rss |
+ $PCP_AWK_PROG '{ if ($2 == "['$pmproxy_pid'") { print $NF} }'`
+
+echo === restarting pmlogger # primary only
+_service pmlogger restart 2>&1 | _filter_pcp_start
+_wait_for_pmlogger
+
+echo === wait for pmproxy to process filesystem events
+pmsleep 4.2
+
+echo === extract updated rss
+pmproxy_rss2=`pminfo -f proc.memory.rss |
+ $PCP_AWK_PROG '{ if ($2 == "['$pmproxy_pid'") { print $NF} }'`
+
+echo === checking rss within tolerance
+_within_tolerance "rss" $pmproxy_rss1 $pmproxy_rss2 10%
+[ $pmproxy_rss2 -gt 10000 ] && echo "Unexpected pmproxy RSS: $pmproxy_rss2, was initially $pmproxy_rss1"
+
+echo "RSS1 for PID $pmproxy_pid is $pmproxy_rss1" >> $here/$seq.full
+echo "RSS2 for PID $pmproxy_pid is $pmproxy_rss2" >> $here/$seq.full
+cat $PCP_LOG_DIR/pmproxy/pmproxy.log >>$seq.full
+echo === see $seq.full for pmproxy rss and logs
+
+# success, all done
+status=0
+exit
diff -Naurp pcp-5.0.2.orig/qa/1573.out pcp-5.0.2/qa/1573.out
--- pcp-5.0.2.orig/qa/1573.out 1970-01-01 10:00:00.000000000 +1000
+++ pcp-5.0.2/qa/1573.out 2020-02-03 13:23:15.259762953 +1100
@@ -0,0 +1,8 @@
+QA output created by 1573
+=== extract initial rss
+=== restarting pmlogger
+=== wait for pmproxy to process filesystem events
+=== extract updated rss
+=== checking rss within tolerance
+=== see 1573.full for pmproxy rss and logs
+=== restarting pmproxy
diff -Naurp pcp-5.0.2.orig/qa/1600 pcp-5.0.2/qa/1600
--- pcp-5.0.2.orig/qa/1600 2019-12-10 17:49:05.000000000 +1100
+++ pcp-5.0.2/qa/1600 2020-02-03 13:23:15.260762942 +1100
@@ -82,7 +82,11 @@ _filter_values()
_filter_label_values()
{
sed \
+ -e "s/^domainname: \"${domainname}\"/domainname: \"DOMAIN\"/g" \
+ -e "s/^machineid: \"${machineid}\"/machineid: \"MACHINE\"/g" \
-e "s/^hostname: \"${hostname}\"/hostname: \"HOSTNAME\"/g" \
+ -e "s/^groupid: $groupid/groupid: GID/g" \
+ -e "s/^userid: $userid/userid: UID/g" \
-e "s/changed: false, true/changed: false/g" \
-e "/metric_label: null/d" \
#end
diff -Naurp pcp-5.0.2.orig/qa/1600.out pcp-5.0.2/qa/1600.out
--- pcp-5.0.2.orig/qa/1600.out 2019-12-10 10:46:20.000000000 +1100
+++ pcp-5.0.2/qa/1600.out 2020-02-03 13:23:15.260762942 +1100
@@ -27,15 +27,15 @@ TIMESERIES
== verify metric labels
TIMESERIES
- inst [100 or "bin-100"] labels {"agent":"sample","hostname":"HOST","role":"testing"}
- inst [200 or "bin-200"] labels {"agent":"sample","hostname":"HOST","role":"testing"}
- inst [300 or "bin-300"] labels {"agent":"sample","hostname":"HOST","role":"testing"}
- inst [400 or "bin-400"] labels {"agent":"sample","hostname":"HOST","role":"testing"}
- inst [500 or "bin-500"] labels {"agent":"sample","hostname":"HOST","role":"testing"}
- inst [600 or "bin-600"] labels {"agent":"sample","hostname":"HOST","role":"testing"}
- inst [700 or "bin-700"] labels {"agent":"sample","hostname":"HOST","role":"testing"}
- inst [800 or "bin-800"] labels {"agent":"sample","hostname":"HOST","role":"testing"}
- inst [900 or "bin-900"] labels {"agent":"sample","hostname":"HOST","role":"testing"}
+ inst [100 or "bin-100"] labels {"agent":"sample","bin":100,"domainname":"DOMAIN","groupid":GID,"hostname":"HOST","latitude":-25.28496,"longitude":152.87886,"machineid":"MACHINE","role":"testing","userid":UID}
+ inst [200 or "bin-200"] labels {"agent":"sample","bin":200,"domainname":"DOMAIN","groupid":GID,"hostname":"HOST","latitude":-25.28496,"longitude":152.87886,"machineid":"MACHINE","role":"testing","userid":UID}
+ inst [300 or "bin-300"] labels {"agent":"sample","bin":300,"domainname":"DOMAIN","groupid":GID,"hostname":"HOST","latitude":-25.28496,"longitude":152.87886,"machineid":"MACHINE","role":"testing","userid":UID}
+ inst [400 or "bin-400"] labels {"agent":"sample","bin":400,"domainname":"DOMAIN","groupid":GID,"hostname":"HOST","latitude":-25.28496,"longitude":152.87886,"machineid":"MACHINE","role":"testing","userid":UID}
+ inst [500 or "bin-500"] labels {"agent":"sample","bin":500,"domainname":"DOMAIN","groupid":GID,"hostname":"HOST","latitude":-25.28496,"longitude":152.87886,"machineid":"MACHINE","role":"testing","userid":UID}
+ inst [600 or "bin-600"] labels {"agent":"sample","bin":600,"domainname":"DOMAIN","groupid":GID,"hostname":"HOST","latitude":-25.28496,"longitude":152.87886,"machineid":"MACHINE","role":"testing","userid":UID}
+ inst [700 or "bin-700"] labels {"agent":"sample","bin":700,"domainname":"DOMAIN","groupid":GID,"hostname":"HOST","latitude":-25.28496,"longitude":152.87886,"machineid":"MACHINE","role":"testing","userid":UID}
+ inst [800 or "bin-800"] labels {"agent":"sample","bin":800,"domainname":"DOMAIN","groupid":GID,"hostname":"HOST","latitude":-25.28496,"longitude":152.87886,"machineid":"MACHINE","role":"testing","userid":UID}
+ inst [900 or "bin-900"] labels {"agent":"sample","bin":900,"domainname":"DOMAIN","groupid":GID,"hostname":"HOST","latitude":-25.28496,"longitude":152.87886,"machineid":"MACHINE","role":"testing","userid":UID}
== verify metric values
TIMESERIES
@@ -43,15 +43,24 @@ TIMESERIES
[TIMESTAMP] VALUE
== verify label names and values
agent: "mmv", "sample", "pmcd"
+bin: 100, 200, 300, 400, 500, 600, 700, 800, 900
changed: false
clan: "mcdonell"
cluster: "zero"
+domainname: "DOMAIN"
+groupid: GID
hostname: "HOSTNAME"
indom_label: 42.001
+latitude: -25.28496
+longitude: 152.87886
+machineid: "MACHINE"
measure: "speed"
model: "RGB"
+registry_label: "string"
role: "testing"
+transient: false, true
units: "metres per second"
unitsystem: "SI"
+userid: UID
== verify archive removal
== all done
diff -Naurp pcp-5.0.2.orig/qa/1601.out pcp-5.0.2/qa/1601.out
--- pcp-5.0.2.orig/qa/1601.out 2019-11-27 16:01:34.000000000 +1100
+++ pcp-5.0.2/qa/1601.out 2020-02-03 13:23:15.261762932 +1100
@@ -131,7 +131,7 @@ Using series 01d8bc7fa75aaff98a08aa0b1c0
{
"series": "605fc77742cd0317597291329561ac4e50c0dd12",
"instance": "c3795d8b757506a2901c6b08b489ba56cae7f0d4",
- "timestamp": 1317633023300.460,
+ "timestamp": 1317633024305.845,
"value": "71661"
},
{
@@ -147,7 +147,7 @@ Using series 01d8bc7fa75aaff98a08aa0b1c0
{
"series": "605fc77742cd0317597291329561ac4e50c0dd12",
"instance": "c3795d8b757506a2901c6b08b489ba56cae7f0d4",
- "timestamp": 1317633023300.460,
+ "timestamp": 1317633024305.845,
"value": "71661"
},
{
@@ -163,7 +163,7 @@ Using series 01d8bc7fa75aaff98a08aa0b1c0
{
"series": "605fc77742cd0317597291329561ac4e50c0dd12",
"instance": "c3795d8b757506a2901c6b08b489ba56cae7f0d4",
- "timestamp": 1317633023300.460,
+ "timestamp": 1317633024305.845,
"value": "71661"
},
{
@@ -179,7 +179,7 @@ Using series 01d8bc7fa75aaff98a08aa0b1c0
{
"series": "605fc77742cd0317597291329561ac4e50c0dd12",
"instance": "c3795d8b757506a2901c6b08b489ba56cae7f0d4",
- "timestamp": 1317633023300.460,
+ "timestamp": 1317633024305.845,
"value": "71661"
},
{
diff -Naurp pcp-5.0.2.orig/qa/1661 pcp-5.0.2/qa/1661
--- pcp-5.0.2.orig/qa/1661 2019-12-10 17:04:20.000000000 +1100
+++ pcp-5.0.2/qa/1661 2020-02-03 13:23:15.261762932 +1100
@@ -41,8 +41,7 @@ _restore_pmlogger_control
echo;echo === restarting pmproxy service to ensure sane starting condition
_service pmlogger stop 2>&1 | _filter_pcp_stop
_service pmproxy restart 2>&1 | _filter_pcp_stop | _filter_pcp_start
-# give pmproxy a chance to startup
-pmsleep 2; _wait_for_pmproxy
+_wait_for_pmproxy
pmproxy_pid=`_get_pids_by_name -a pmproxy`
[ -z "$pmproxy_pid" ] && echo === pmproxy not running && status=1 && exit 1
diff -Naurp pcp-5.0.2.orig/qa/group pcp-5.0.2/qa/group
--- pcp-5.0.2.orig/qa/group 2019-12-11 14:06:06.000000000 +1100
+++ pcp-5.0.2/qa/group 2020-02-03 13:23:15.261762932 +1100
@@ -1688,6 +1688,7 @@ BAD
1545 pcp2xml python pcp2xxx local
1546 pmrep python local
1547 pmrep python local
+1573 pmproxy libpcp_web pmlogger local
1588 python pmiostat local
1598 pmda.statsd local
1599 pmda.statsd local
diff -Naurp pcp-5.0.2.orig/src/include/pcp/libpcp.h pcp-5.0.2/src/include/pcp/libpcp.h
--- pcp-5.0.2.orig/src/include/pcp/libpcp.h 2019-09-24 17:23:36.000000000 +1000
+++ pcp-5.0.2/src/include/pcp/libpcp.h 2020-02-03 13:23:15.261762932 +1100
@@ -7,7 +7,7 @@
* remain fixed across releases, and they may not work, or may
* provide different semantics at some point in the future.
*
- * Copyright (c) 2012-2019 Red Hat.
+ * Copyright (c) 2012-2020 Red Hat.
* Copyright (c) 2008-2009 Aconex. All Rights Reserved.
* Copyright (c) 1995-2002 Silicon Graphics, Inc. All Rights Reserved.
*
@@ -846,6 +846,13 @@ PCP_CALL extern int __pmLogPutText(__pmA
PCP_CALL extern int __pmLogWriteLabel(__pmFILE *, const __pmLogLabel *);
PCP_CALL extern int __pmLogLoadLabel(__pmArchCtl *, const char *);
PCP_CALL extern int __pmLogLoadMeta(__pmArchCtl *);
+PCP_CALL extern int __pmLogAddDesc(__pmArchCtl *, const pmDesc *);
+PCP_CALL extern int __pmLogAddInDom(__pmArchCtl *, const pmTimespec *, const pmInResult *, int *, int);
+PCP_CALL extern int __pmLogAddPMNSNode(__pmArchCtl *, pmID, const char *);
+PCP_CALL extern int __pmLogAddLabelSets(__pmArchCtl *, const pmTimespec *, unsigned int, unsigned int, int, pmLabelSet *);
+PCP_CALL extern int __pmLogAddText(__pmArchCtl *, unsigned int, unsigned int, const char *);
+PCP_CALL extern int __pmLogAddVolume(__pmArchCtl *, unsigned int);
+
#define PMLOGREAD_NEXT 0
#define PMLOGREAD_TO_EOF 1
PCP_CALL extern int __pmLogRead(__pmArchCtl *, int, __pmFILE *, pmResult **, int);
@@ -862,7 +869,9 @@ PCP_CALL extern int __pmLogLookupText(__
PCP_CALL extern int __pmLogNameInDom(__pmArchCtl *, pmInDom, pmTimeval *, int, char **);
PCP_CALL extern const char *__pmLogLocalSocketDefault(int, char *buf, size_t bufSize);
PCP_CALL extern const char *__pmLogLocalSocketUser(int, char *buf, size_t bufSize);
+PCP_CALL extern int __pmLogCompressedSuffix(const char *);
PCP_CALL extern char *__pmLogBaseName(char *);
+PCP_CALL extern char *__pmLogBaseNameVol(char *, int *);
PCP_DATA extern int __pmLogReads;
/* Convert opaque context handle to __pmContext pointer */
diff -Naurp pcp-5.0.2.orig/src/libpcp/src/exports.master pcp-5.0.2/src/libpcp/src/exports.master
--- pcp-5.0.2.orig/src/libpcp/src/exports.master 2019-10-02 14:40:30.000000000 +1000
+++ pcp-5.0.2/src/libpcp/src/exports.master 2020-02-03 13:23:15.262762921 +1100
@@ -683,3 +683,15 @@ PCP_3.26 {
global:
__pmDupLabelSets;
} PCP_3.25;
+
+PCP_3.26_1 {
+ global:
+ __pmLogAddDesc;
+ __pmLogAddInDom;
+ __pmLogAddPMNSNode;
+ __pmLogAddLabelSets;
+ __pmLogAddText;
+ __pmLogAddVolume;
+ __pmLogCompressedSuffix;
+ __pmLogBaseNameVol;
+} PCP_3.26;
diff -Naurp pcp-5.0.2.orig/src/libpcp/src/io.c pcp-5.0.2/src/libpcp/src/io.c
--- pcp-5.0.2.orig/src/libpcp/src/io.c 2018-06-09 11:43:34.000000000 +1000
+++ pcp-5.0.2/src/libpcp/src/io.c 2020-02-03 13:23:15.262762921 +1100
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 2017-2018 Red Hat.
+ * Copyright (c) 2017-2018,2020 Red Hat.
*
* This library is free software; you can redistribute it and/or modify it
* under the terms of the GNU Lesser General Public License as published
@@ -47,7 +47,7 @@ extern __pm_fops __pm_xz;
#endif
static const struct {
- const char *suff;
+ const char *suffix;
const int appl;
__pm_fops *handler;
} compress_ctl[] = {
@@ -61,40 +61,43 @@ static const struct {
};
static const int ncompress = sizeof(compress_ctl) / sizeof(compress_ctl[0]);
+int
+__pmLogCompressedSuffix(const char *suffix)
+{
+ int i;
+
+ for (i = 0; i < ncompress; i++)
+ if (strcmp(suffix, compress_ctl[i].suffix) == 0)
+ return 1;
+ return 0;
+}
+
/*
- * If name contains '.' and the suffix is "index", "meta" or a string of
- * digits, all optionally followed by one of the compression suffixes,
- * strip the suffix.
- *
- * Modifications are performed on the argument string in-place. If modifications
- * are made, a pointer to the start of the modified string is returned.
- * Otherwise, NULL is returned.
+ * Variant of __pmLogBaseName() - see below that also returns log
+ * the volume number if the file name is an archive log volume.
+ * If the vol argument is NULL it will be ignored.
*/
char *
-__pmLogBaseName(char *name)
+__pmLogBaseNameVol(char *name, int *vol)
{
- char *q;
- int strip;
- int i;
+ char *q, *q2;
+ int strip = 0;
- strip = 0;
+ if (vol)
+ *vol = -1;
if ((q = strrchr(name, '.')) != NULL) {
- for (i = 0; i < ncompress; i++) {
- if (strcmp(q, compress_ctl[i].suff) == 0) {
- char *q2;
- /*
- * The name ends with one of the supported compressed file
- * suffixes. Strip it before checking for other known suffixes.
- */
- *q = '\0';
- if ((q2 = strrchr(name, '.')) == NULL) {
- /* no . to the left of the suffix */
- *q = '.';
- goto done;
- }
- q = q2;
- break;
+ if (__pmLogCompressedSuffix(q)) {
+ /*
+ * The name ends with one of the supported compressed file
+ * suffixes. Strip it before checking for other known suffixes.
+ */
+ *q = '\0';
+ if ((q2 = strrchr(name, '.')) == NULL) {
+ /* no . to the left of the suffix */
+ *q = '.';
+ goto done;
}
+ q = q2;
}
if (strcmp(q, ".index") == 0) {
strip = 1;
@@ -109,16 +112,10 @@ __pmLogBaseName(char *name)
*/
if (q[1] != '\0') {
char *end;
- /*
- * Below we don't care about the value from strtol(),
- * we're interested in updating the pointer "end".
- * The messiness is thanks to gcc and glibc ... strtol()
- * is marked __attribute__((warn_unused_result)) ...
- * to avoid warnings on all platforms, assign to a
- * dummy variable that is explicitly marked unused.
- */
- long tmpl __attribute__((unused));
+ long tmpl;
tmpl = strtol(q+1, &end, 10);
+ if (vol)
+ *vol = tmpl;
if (*end == '\0') strip = 1;
}
}
@@ -131,6 +128,21 @@ done:
return NULL; /* not the name of an archive file. */
}
+/*
+ * If name contains '.' and the suffix is "index", "meta" or a string of
+ * digits, all optionally followed by one of the compression suffixes,
+ * strip the suffix.
+ *
+ * Modifications are performed on the argument string in-place. If modifications
+ * are made, a pointer to the start of the modified string is returned.
+ * Otherwise, NULL is returned.
+ */
+char *
+__pmLogBaseName(char *name)
+{
+ return __pmLogBaseNameVol(name, NULL);
+}
+
static int
popen_uncompress(const char *cmd, const char *arg, const char *fname, int fd)
{
@@ -319,7 +331,7 @@ __pmCompressedFileIndex(char *fname, siz
char tmpname[MAXPATHLEN];
for (i = 0; i < ncompress; i++) {
- suffix = compress_ctl[i].suff;
+ suffix = compress_ctl[i].suffix;
pmsprintf(tmpname, sizeof(tmpname), "%s%s", fname, suffix);
sts = access(tmpname, R_OK);
if (sts == 0 || (errno != ENOENT && errno != ENOTDIR)) {
@@ -358,7 +370,7 @@ index_compress(char *fname, size_t flen)
suffix = strrchr(fname, '.');
if (suffix != NULL) {
for (i = 0; i < ncompress; i++) {
- if (strcmp(suffix, compress_ctl[i].suff) == 0)
+ if (strcmp(suffix, compress_ctl[i].suffix) == 0)
return i;
}
}
@@ -731,7 +743,7 @@ compress_suffix_list(void)
const char *q;
for (i = 0; i < ncompress; i++) {
- q = compress_ctl[i].suff;
+ q = compress_ctl[i].suffix;
if (i > 0)
*p++ = ' ';
while (*q) {
diff -Naurp pcp-5.0.2.orig/src/libpcp/src/logmeta.c pcp-5.0.2/src/libpcp/src/logmeta.c
--- pcp-5.0.2.orig/src/libpcp/src/logmeta.c 2018-09-14 10:22:56.000000000 +1000
+++ pcp-5.0.2/src/libpcp/src/logmeta.c 2020-02-03 13:23:15.262762921 +1100
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 2013-2018 Red Hat.
+ * Copyright (c) 2013-2018, 2020 Red Hat.
* Copyright (c) 1995-2002 Silicon Graphics, Inc. All Rights Reserved.
*
* This library is free software; you can redistribute it and/or modify it
@@ -490,7 +490,7 @@ check_dup_labels(const __pmArchCtl *acp)
}
static int
-addtext(__pmArchCtl *acp, unsigned int ident, unsigned int type, char *buffer)
+addtext(__pmArchCtl *acp, unsigned int ident, unsigned int type, const char *buffer)
{
__pmLogCtl *lcp = acp->ac_log;
__pmHashNode *hp;
@@ -553,6 +553,92 @@ PM_FAULT_POINT("libpcp/" __FILE__ ":15",
return sts;
}
+int
+__pmLogAddDesc(__pmArchCtl *acp, const pmDesc *newdp)
+{
+ __pmHashNode *hp;
+ __pmLogCtl *lcp = acp->ac_log;
+ pmDesc *dp, *olddp;
+
+ if ((hp = __pmHashSearch((int)newdp->pmid, &lcp->l_hashpmid)) != NULL) {
+ /* PMID is already in the hash table - check for conflicts. */
+ olddp = (pmDesc *)hp->data;
+ if (newdp->type != olddp->type)
+ return PM_ERR_LOGCHANGETYPE;
+ if (newdp->sem != olddp->sem)
+ return PM_ERR_LOGCHANGESEM;
+ if (newdp->indom != olddp->indom)
+ return PM_ERR_LOGCHANGEINDOM;
+ if (newdp->units.dimSpace != olddp->units.dimSpace ||
+ newdp->units.dimTime != olddp->units.dimTime ||
+ newdp->units.dimCount != olddp->units.dimCount ||
+ newdp->units.scaleSpace != olddp->units.scaleSpace ||
+ newdp->units.scaleTime != olddp->units.scaleTime ||
+ newdp->units.scaleCount != olddp->units.scaleCount)
+ return PM_ERR_LOGCHANGEUNITS;
+
+ /* PMID is already known and checks out - we're done here. */
+ return 0;
+ }
+
+ /* Add a copy of the descriptor into the PMID:desc hash table. */
+PM_FAULT_POINT("libpcp/" __FILE__ ":2", PM_FAULT_ALLOC);
+ if ((dp = (pmDesc *)malloc(sizeof(pmDesc))) == NULL)
+ return -oserror();
+ *dp = *newdp;
+
+ return __pmHashAdd((int)dp->pmid, (void *)dp, &lcp->l_hashpmid);
+}
+
+int
+__pmLogAddPMNSNode(__pmArchCtl *acp, pmID pmid, const char *name)
+{
+ __pmLogCtl *lcp = acp->ac_log;
+ int sts;
+
+ /*
+ * If we see a duplicate name with a different PMID, its a
+ * recoverable error.
+ * We wont be able to see all of the data in the log, but
+ * its better to provide access to some rather than none,
+ * esp. when only one or two metric IDs may be corrupted
+ * in this way (which we may not be interested in anyway).
+ */
+ sts = __pmAddPMNSNode(lcp->l_pmns, pmid, name);
+ if (sts == PM_ERR_PMID)
+ sts = 0;
+ return sts;
+}
+
+int
+__pmLogAddInDom(__pmArchCtl *acp, const pmTimespec *when, const pmInResult *in,
+ int *tbuf, int allinbuf)
+{
+ pmTimeval tv;
+
+ tv.tv_sec = when->tv_sec;
+ tv.tv_usec = when->tv_nsec / 1000;
+ return addindom(acp->ac_log, in->indom, &tv,
+ in->numinst, in->instlist, in->namelist, tbuf, allinbuf);
+}
+
+int
+__pmLogAddLabelSets(__pmArchCtl *acp, const pmTimespec *when, unsigned int type,
+ unsigned int ident, int nsets, pmLabelSet *labelsets)
+{
+ pmTimeval tv;
+
+ tv.tv_sec = when->tv_sec;
+ tv.tv_usec = when->tv_nsec / 1000;
+ return addlabel(acp, type, ident, nsets, labelsets, &tv);
+}
+
+int
+__pmLogAddText(__pmArchCtl *acp, unsigned int ident, unsigned int type, const char *buffer)
+{
+ return addtext(acp, ident, type, buffer);
+}
+
/*
* Load _all_ of the hashed pmDesc and __pmLogInDom structures from the metadata
* log file -- used at the initialization (NewContext) of an archive.
@@ -563,11 +649,8 @@ int
__pmLogLoadMeta(__pmArchCtl *acp)
{
__pmLogCtl *lcp = acp->ac_log;
- __pmHashNode *hp;
int rlen;
int check;
- pmDesc *dp;
- pmDesc *olddp;
int sts = 0;
__pmLogHdr h;
__pmFILE *f = lcp->l_mdfp;
@@ -615,13 +698,10 @@ __pmLogLoadMeta(__pmArchCtl *acp)
}
rlen = h.len - (int)sizeof(__pmLogHdr) - (int)sizeof(int);
if (h.type == TYPE_DESC) {
+ pmDesc desc;
+
numpmid++;
-PM_FAULT_POINT("libpcp/" __FILE__ ":2", PM_FAULT_ALLOC);
- if ((dp = (pmDesc *)malloc(sizeof(pmDesc))) == NULL) {
- sts = -oserror();
- goto end;
- }
- if ((n = (int)__pmFread(dp, 1, sizeof(pmDesc), f)) != sizeof(pmDesc)) {
+ if ((n = (int)__pmFread(&desc, 1, sizeof(pmDesc), f)) != sizeof(pmDesc)) {
if (pmDebugOptions.logmeta) {
fprintf(stderr, "__pmLogLoadMeta: pmDesc read -> %d: expected: %d\n",
n, (int)sizeof(pmDesc));
@@ -632,67 +712,25 @@ PM_FAULT_POINT("libpcp/" __FILE__ ":2",
}
else
sts = PM_ERR_LOGREC;
- free(dp);
goto end;
}
- else {
- /* swab desc */
- dp->type = ntohl(dp->type);
- dp->sem = ntohl(dp->sem);
- dp->indom = __ntohpmInDom(dp->indom);
- dp->units = __ntohpmUnits(dp->units);
- dp->pmid = __ntohpmID(dp->pmid);
- }
- /* Add it to the hash pmid hash table. */
- if ((hp = __pmHashSearch((int)dp->pmid, &lcp->l_hashpmid)) != NULL) {
- /*
- * This pmid is already in the hash table. Check for conflicts.
- */
- olddp = (pmDesc *)hp->data;
- if (dp->type != olddp->type) {
- sts = PM_ERR_LOGCHANGETYPE;
- free(dp);
- goto end;
- }
- if (dp->sem != olddp->sem) {
- sts = PM_ERR_LOGCHANGESEM;
- free(dp);
- goto end;
- }
- if (dp->indom != olddp->indom) {
- sts = PM_ERR_LOGCHANGEINDOM;
- free(dp);
- goto end;
- }
- if (dp->units.dimSpace != olddp->units.dimSpace ||
- dp->units.dimTime != olddp->units.dimTime ||
- dp->units.dimCount != olddp->units.dimCount ||
- dp->units.scaleSpace != olddp->units.scaleSpace ||
- dp->units.scaleTime != olddp->units.scaleTime ||
- dp->units.scaleCount != olddp->units.scaleCount) {
- sts = PM_ERR_LOGCHANGEUNITS;
- free(dp);
- goto end;
- }
- /*
- * This pmid is already known, and matches. We can free the newly
- * read copy and use the one in the hash table.
- */
- free(dp);
- dp = olddp;
- }
- else if ((sts = __pmHashAdd((int)dp->pmid, (void *)dp, &lcp->l_hashpmid)) < 0) {
- free(dp);
+ /* swab desc */
+ desc.type = ntohl(desc.type);
+ desc.sem = ntohl(desc.sem);
+ desc.indom = __ntohpmInDom(desc.indom);
+ desc.units = __ntohpmUnits(desc.units);
+ desc.pmid = __ntohpmID(desc.pmid);
+
+ if ((sts = __pmLogAddDesc(acp, &desc)) < 0)
goto end;
- }
/* read in the names & store in PMNS tree ... */
if ((n = (int)__pmFread(&numnames, 1, sizeof(numnames), f)) !=
sizeof(numnames)) {
if (pmDebugOptions.logmeta) {
- fprintf(stderr, "__pmLogLoadMeta: numnames read -> %d: expected: %d\n",
- n, (int)sizeof(numnames));
+ fprintf(stderr, "%s: numnames read -> %d: expected: %d\n",
+ "__pmLogLoadMeta", n, (int)sizeof(numnames));
}
if (__pmFerror(f)) {
__pmClearerr(f);
@@ -711,8 +749,8 @@ PM_FAULT_POINT("libpcp/" __FILE__ ":2",
if ((n = (int)__pmFread(&len, 1, sizeof(len), f)) !=
sizeof(len)) {
if (pmDebugOptions.logmeta) {
- fprintf(stderr, "__pmLogLoadMeta: len name[%d] read -> %d: expected: %d\n",
- i, n, (int)sizeof(len));
+ fprintf(stderr, "%s: len name[%d] read -> %d: expected: %d\n",
+ "__pmLogLoadMeta", i, n, (int)sizeof(len));
}
if (__pmFerror(f)) {
__pmClearerr(f);
@@ -729,8 +767,8 @@ PM_FAULT_POINT("libpcp/" __FILE__ ":2",
if ((n = (int)__pmFread(name, 1, len, f)) != len) {
if (pmDebugOptions.logmeta) {
- fprintf(stderr, "__pmLogLoadMeta: name[%d] read -> %d: expected: %d\n",
- i, n, len);
+ fprintf(stderr, "%s: name[%d] read -> %d: expected: %d\n",
+ "__pmLogLoadMeta", i, n, len);
}
if (__pmFerror(f)) {
__pmClearerr(f);
@@ -743,36 +781,23 @@ PM_FAULT_POINT("libpcp/" __FILE__ ":2",
name[len] = '\0';
if (pmDebugOptions.logmeta) {
char strbuf[20];
- fprintf(stderr, "__pmLogLoadMeta: PMID: %s name: %s\n",
- pmIDStr_r(dp->pmid, strbuf, sizeof(strbuf)), name);
+ fprintf(stderr, "%s: PMID: %s name: %s\n",
+ "__pmLogLoadMeta",
+ pmIDStr_r(desc.pmid, strbuf, sizeof(strbuf)), name);
}
- /* Add the new PMNS node */
- if ((sts = __pmAddPMNSNode(lcp->l_pmns, dp->pmid, name)) < 0) {
- /*
- * If we see a duplicate name with a different PMID, its a
- * recoverable error.
- * We wont be able to see all of the data in the log, but
- * its better to provide access to some rather than none,
- * esp. when only one or two metric IDs may be corrupted
- * in this way (which we may not be interested in anyway).
- */
- if (sts != PM_ERR_PMID)
- goto end;
- }
+
+ /* Add the new PMNS node into this context */
+ if ((sts = __pmLogAddPMNSNode(acp, desc.pmid, name)) < 0)
+ goto end;
}/*for*/
}
else if (h.type == TYPE_INDOM) {
- int *tbuf;
- pmInDom indom;
- pmTimeval *when;
- int numinst;
- int *instlist;
- char **namelist;
+ pmTimeval *tv;
+ pmTimespec when;
+ pmInResult in;
char *namebase;
- int *stridx;
- int i;
- int k;
- int allinbuf = 0;
+ int *tbuf, *stridx;
+ int i, k, allinbuf = 0;
PM_FAULT_POINT("libpcp/" __FILE__ ":3", PM_FAULT_ALLOC);
if ((tbuf = (int *)malloc(rlen)) == NULL) {
@@ -781,8 +806,8 @@ PM_FAULT_POINT("libpcp/" __FILE__ ":3",
}
if ((n = (int)__pmFread(tbuf, 1, rlen, f)) != rlen) {
if (pmDebugOptions.logmeta) {
- fprintf(stderr, "__pmLogLoadMeta: indom read -> %d: expected: %d\n",
- n, rlen);
+ fprintf(stderr, "%s: indom read -> %d: expected: %d\n",
+ "__pmLogLoadMeta", n, rlen);
}
if (__pmFerror(f)) {
__pmClearerr(f);
@@ -795,44 +820,44 @@ PM_FAULT_POINT("libpcp/" __FILE__ ":3",
}
k = 0;
- when = (pmTimeval *)&tbuf[k];
- when->tv_sec = ntohl(when->tv_sec);
- when->tv_usec = ntohl(when->tv_usec);
- k += sizeof(*when)/sizeof(int);
- indom = __ntohpmInDom((unsigned int)tbuf[k++]);
- numinst = ntohl(tbuf[k++]);
- if (numinst > 0) {
- instlist = &tbuf[k];
- k += numinst;
+ tv = (pmTimeval *)&tbuf[k];
+ when.tv_sec = ntohl(tv->tv_sec);
+ when.tv_nsec = ntohl(tv->tv_usec) * 1000;
+ k += sizeof(*tv)/sizeof(int);
+ in.indom = __ntohpmInDom((unsigned int)tbuf[k++]);
+ in.numinst = ntohl(tbuf[k++]);
+ if (in.numinst > 0) {
+ in.instlist = &tbuf[k];
+ k += in.numinst;
stridx = &tbuf[k];
#if defined(HAVE_32BIT_PTR)
- namelist = (char **)stridx;
+ in.namelist = (char **)stridx;
allinbuf = 1; /* allocation is all in tbuf */
#else
allinbuf = 0; /* allocation for namelist + tbuf */
/* need to allocate to hold the pointers */
PM_FAULT_POINT("libpcp/" __FILE__ ":4", PM_FAULT_ALLOC);
- namelist = (char **)malloc(numinst*sizeof(char*));
- if (namelist == NULL) {
+ in.namelist = (char **)malloc(in.numinst * sizeof(char*));
+ if (in.namelist == NULL) {
sts = -oserror();
free(tbuf);
goto end;
}
#endif
- k += numinst;
+ k += in.numinst;
namebase = (char *)&tbuf[k];
- for (i = 0; i < numinst; i++) {
- instlist[i] = ntohl(instlist[i]);
- namelist[i] = &namebase[ntohl(stridx[i])];
+ for (i = 0; i < in.numinst; i++) {
+ in.instlist[i] = ntohl(in.instlist[i]);
+ in.namelist[i] = &namebase[ntohl(stridx[i])];
}
- if ((sts = addindom(lcp, indom, when, numinst, instlist, namelist, tbuf, allinbuf)) < 0)
+ if ((sts = __pmLogAddInDom(acp, &when, &in, tbuf, allinbuf)) < 0)
goto end;
/* If this indom was a duplicate, then we need to free tbuf and
namelist, as appropriate. */
if (sts == PMLOGPUTINDOM_DUP) {
free(tbuf);
- if (namelist != NULL && !allinbuf)
- free(namelist);
+ if (in.namelist != NULL && !allinbuf)
+ free(in.namelist);
}
}
else {
@@ -860,8 +885,8 @@ PM_FAULT_POINT("libpcp/" __FILE__ ":11",
}
if ((n = (int)__pmFread(tbuf, 1, rlen, f)) != rlen) {
if (pmDebugOptions.logmeta) {
- fprintf(stderr, "__pmLogLoadMeta: label read -> %d: expected: %d\n",
- n, rlen);
+ fprintf(stderr, "%s: label read -> %d: expected: %d\n",
+ "__pmLogLoadMeta", n, rlen);
}
if (__pmFerror(f)) {
__pmClearerr(f);
@@ -908,7 +933,8 @@ PM_FAULT_POINT("libpcp/" __FILE__ ":11",
if (jsonlen < 0 || jsonlen > PM_MAXLABELJSONLEN) {
if (pmDebugOptions.logmeta)
- fprintf(stderr, "__pmLogLoadMeta: corrupted json in labelset. jsonlen=%d\n", jsonlen);
+ fprintf(stderr, "%s: corrupted json in labelset. jsonlen=%d\n",
+ "__pmLogLoadMeta", jsonlen);
sts = PM_ERR_LOGREC;
free(labelsets);
free(tbuf);
@@ -935,7 +961,8 @@ PM_FAULT_POINT("libpcp/" __FILE__ ":11",
if (nlabels > PM_MAXLABELS || k + nlabels * sizeof(pmLabel) > rlen) {
/* corrupt archive metadata detected. GH #475 */
if (pmDebugOptions.logmeta)
- fprintf(stderr, "__pmLogLoadMeta: corrupted labelset. nlabels=%d\n", nlabels);
+ fprintf(stderr, "%s: corrupted labelset. nlabels=%d\n",
+ "__pmLogLoadMeta", nlabels);
sts = PM_ERR_LOGREC;
free(labelsets);
free(tbuf);
@@ -975,8 +1002,8 @@ PM_FAULT_POINT("libpcp/" __FILE__ ":16",
}
if ((n = (int)__pmFread(tbuf, 1, rlen, f)) != rlen) {
if (pmDebugOptions.logmeta) {
- fprintf(stderr, "__pmLogLoadMeta: text read -> %d: expected: %d\n",
- n, rlen);
+ fprintf(stderr, "%s: text read -> %d: expected: %d\n",
+ "__pmLogLoadMeta", n, rlen);
}
if (__pmFerror(f)) {
__pmClearerr(f);
@@ -1005,8 +1032,8 @@ PM_FAULT_POINT("libpcp/" __FILE__ ":16",
ident = __ntohpmID(*((unsigned int *)&tbuf[k]));
else {
if (pmDebugOptions.logmeta) {
- fprintf(stderr, "__pmLogLoadMeta: bad text ident -> %x\n",
- type);
+ fprintf(stderr, "%s: bad text ident -> %x\n",
+ "__pmLogLoadMeta", type);
}
free(tbuf);
continue;
@@ -1024,8 +1051,9 @@ PM_FAULT_POINT("libpcp/" __FILE__ ":16",
check = ntohl(check);
if (n != sizeof(check) || h.len != check) {
if (pmDebugOptions.logmeta) {
- fprintf(stderr, "__pmLogLoadMeta: trailer read -> %d or len=%d: expected %d @ offset=%d\n",
- n, check, h.len, (int)(__pmFtell(f) - sizeof(check)));
+ fprintf(stderr, "%s: trailer read -> %d or len=%d: "
+ "expected %d @ offset=%d\n", "__pmLogLoadMeta",
+ n, check, h.len, (int)(__pmFtell(f) - sizeof(check)));
}
if (__pmFerror(f)) {
__pmClearerr(f);
@@ -1046,7 +1074,7 @@ end:
if (sts == 0) {
if (numpmid == 0) {
if (pmDebugOptions.logmeta) {
- fprintf(stderr, "__pmLogLoadMeta: no metrics found?\n");
+ fprintf(stderr, "%s: no metrics found?\n", "__pmLogLoadMeta");
}
sts = PM_ERR_LOGREC;
}
diff -Naurp pcp-5.0.2.orig/src/libpcp/src/logutil.c pcp-5.0.2/src/libpcp/src/logutil.c
--- pcp-5.0.2.orig/src/libpcp/src/logutil.c 2018-07-08 10:58:08.000000000 +1000
+++ pcp-5.0.2/src/libpcp/src/logutil.c 2020-02-03 13:23:15.263762911 +1100
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 2012-2017 Red Hat.
+ * Copyright (c) 2012-2017,2020 Red Hat.
* Copyright (c) 1995-2002,2004 Silicon Graphics, Inc. All Rights Reserved.
*
* This library is free software; you can redistribute it and/or modify it
@@ -764,6 +764,22 @@ __pmLogClose(__pmArchCtl *acp)
}
int
+__pmLogAddVolume(__pmArchCtl *acp, unsigned int vol)
+{
+ __pmLogCtl *lcp = acp->ac_log;
+
+ if (lcp->l_minvol == -1) {
+ lcp->l_minvol = vol;
+ lcp->l_maxvol = vol;
+ } else if (vol < lcp->l_minvol) {
+ lcp->l_minvol = vol;
+ } else if (vol > lcp->l_maxvol) {
+ lcp->l_maxvol = vol;
+ }
+ return 0;
+}
+
+int
__pmLogLoadLabel(__pmArchCtl *acp, const char *name)
{
__pmLogCtl *lcp = acp->ac_log;
@@ -876,21 +892,14 @@ __pmLogLoadLabel(__pmArchCtl *acp, const
}
}
else {
- char *q;
- int vol;
- vol = (int)strtol(tp, &q, 10);
+ char *q;
+ unsigned int vol;
+
+ vol = (unsigned int)strtoul(tp, &q, 10);
if (*q == '\0') {
exists = 1;
- if (lcp->l_minvol == -1) {
- lcp->l_minvol = vol;
- lcp->l_maxvol = vol;
- }
- else {
- if (vol < lcp->l_minvol)
- lcp->l_minvol = vol;
- if (vol > lcp->l_maxvol)
- lcp->l_maxvol = vol;
- }
+ if ((sts = __pmLogAddVolume(acp, vol)) < 0)
+ goto cleanup;
}
}
}
@@ -2282,7 +2291,7 @@ __pmLogSetTime(__pmContext *ctxp)
int match = 0;
int vol;
int numti = lcp->l_numti;
- __pmFILE *f;
+ __pmFILE *f;
__pmLogTI *tip = lcp->l_ti;
double t_lo;
struct stat sbuf;
diff -Naurp pcp-5.0.2.orig/src/libpcp_web/src/discover.c pcp-5.0.2/src/libpcp_web/src/discover.c
--- pcp-5.0.2.orig/src/libpcp_web/src/discover.c 2019-12-10 17:04:20.000000000 +1100
+++ pcp-5.0.2/src/libpcp_web/src/discover.c 2020-02-03 13:36:11.958637560 +1100
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 2018-2019 Red Hat.
+ * Copyright (c) 2018-2020 Red Hat.
*
* This library is free software; you can redistribute it and/or modify it
* under the terms of the GNU Lesser General Public License as published
@@ -14,6 +14,8 @@
#include "discover.h"
#include "slots.h"
#include "util.h"
+#include <dirent.h>
+#include <sys/stat.h>
/* Decode various archive metafile records (desc, indom, labels, helptext) */
static int pmDiscoverDecodeMetaDesc(uint32_t *, int, pmDesc *, int *, char ***);
@@ -24,11 +26,15 @@ static int pmDiscoverDecodeMetaLabelSet(
/* array of registered callbacks, see pmDiscoverSetup() */
static int discoverCallBackTableSize;
static pmDiscoverCallBacks **discoverCallBackTable;
+static char *pmDiscoverFlagsStr(pmDiscover *);
/* internal hash table of discovered paths */
-#define PM_DISCOVER_HASHTAB_SIZE 64
+#define PM_DISCOVER_HASHTAB_SIZE 16
static pmDiscover *discover_hashtable[PM_DISCOVER_HASHTAB_SIZE];
+/* pmlogger_daily log-roll lock count */
+static int lockcnt = 0;
+
/* FNV string hash algorithm. Return unsigned in range 0 .. limit-1 */
static unsigned int
strhash(const char *s, unsigned int limit)
@@ -43,18 +49,38 @@ strhash(const char *s, unsigned int limi
return h % limit;
}
+/* ctime string - note static buf is returned */
+static char *
+stamp(void)
+{
+ time_t now = time(NULL);
+ char *p, *c = ctime(&now);
+
+ if ((p = strrchr(c, '\n')) != NULL)
+ *p = '\0';
+ return c;
+}
+
/*
- * Lookup or Add a discovered file path (directory or PCP archive file)
+ * Lookup or Add a discovered file path (directory or PCP archive file).
+ * Note: the fullpath suffix (.meta, .[0-9]+) should already be stripped.
* Return path table entry (new or existing).
*/
static pmDiscover *
-pmDiscoverLookupAdd(const char *path, pmDiscoverModule *module, void *arg)
+pmDiscoverLookupAdd(const char *fullpath, pmDiscoverModule *module, void *arg)
{
pmDiscover *p, *h;
- unsigned int k = strhash(path, PM_DISCOVER_HASHTAB_SIZE);
+ unsigned int k;
+ sds name;
+
+ name = sdsnew(fullpath);
+ k = strhash(name, PM_DISCOVER_HASHTAB_SIZE);
+
+ if (pmDebugOptions.discovery)
+ fprintf(stderr, "pmDiscoverLookupAdd: name=%s\n", name);
for (p = NULL, h = discover_hashtable[k]; h != NULL; p = h, h = h->next) {
- if (strcmp(h->context.name, path) == 0)
+ if (sdscmp(h->context.name, name) == 0)
break;
}
@@ -65,14 +91,24 @@ pmDiscoverLookupAdd(const char *path, pm
h->ctx = -1; /* no PMAPI context initially */
h->flags = PM_DISCOVER_FLAGS_NEW;
h->context.type = PM_CONTEXT_ARCHIVE;
- h->context.name = sdsnew(path);
+ h->context.name = name;
h->module = module;
h->data = arg;
if (p == NULL)
discover_hashtable[k] = h;
else
p->next = h;
+ if (pmDebugOptions.discovery)
+ fprintf(stderr, "pmDiscoverLookupAdd: --> new entry %s\n", name);
+
+ }
+ else {
+ /* already in hash table, so free the buffer */
+ if (pmDebugOptions.discovery)
+ fprintf(stderr, "pmDiscoverLookupAdd: --> existing entry %s\n", name);
+ sdsfree(name);
}
+
return h;
}
@@ -82,12 +118,6 @@ pmDiscoverLookup(const char *path)
return pmDiscoverLookupAdd(path, NULL, NULL);
}
-static pmDiscover *
-pmDiscoverAdd(const char *path, pmDiscoverModule *module, void *arg)
-{
- return pmDiscoverLookupAdd(path, module, arg);
-}
-
static void
pmDiscoverFree(pmDiscover *p)
{
@@ -101,39 +131,42 @@ pmDiscoverFree(pmDiscover *p)
sdsfree(p->context.source);
if (p->context.labelset)
pmFreeLabelSets(p->context.labelset, 1);
+ if (p->event_handle) {
+ uv_fs_event_stop(p->event_handle);
+ free(p->event_handle);
+ p->event_handle = NULL;
+ }
+
memset(p, 0, sizeof(*p));
free(p);
}
/*
- * Delete tracking of a previously discovered path. Frees resources and
- * destroy PCP context (if any).
+ * Traverse and invoke callback for all paths matching any bit
+ * in the flags bitmap. Callback can be NULL to just get a count.
+ * Return count of matching paths, may be 0.
*/
-static void
-pmDiscoverDelete(sds path)
+static int
+pmDiscoverTraverse(unsigned int flags, void (*callback)(pmDiscover *))
{
- pmDiscover *p, *h;
- unsigned int k = strhash(path, PM_DISCOVER_HASHTAB_SIZE);
+ int count = 0, i;
+ pmDiscover *p;
- for (p = NULL, h = discover_hashtable[k]; h != NULL; p = h, h = h->next) {
- if (sdscmp(h->context.name, path) == 0) {
- if (p == NULL)
- discover_hashtable[k] = NULL;
- else
- p->next = h->next;
- pmDiscoverFree(h);
- break;
+ for (i = 0; i < PM_DISCOVER_HASHTAB_SIZE; i++) {
+ for (p = discover_hashtable[i]; p; p = p->next) {
+ if (p->flags & flags) {
+ if (callback)
+ callback(p);
+ count++;
+ }
}
}
+ return count;
}
-/*
- * Traverse and invoke callback for all paths matching any bit
- * in the flags bitmap. Callback can be NULL to just get a count.
- * Return count of matching paths, may be 0.
- */
+/* as above, but with an extra (void *)arg passed to the cb */
static int
-pmDiscoverTraverse(unsigned int flags, void (*callback)(pmDiscover *))
+pmDiscoverTraverseArg(unsigned int flags, void (*callback)(pmDiscover *, void *), void *arg)
{
int count = 0, i;
pmDiscover *p;
@@ -142,7 +175,7 @@ pmDiscoverTraverse(unsigned int flags, v
for (p = discover_hashtable[i]; p; p = p->next) {
if (p->flags & flags) {
if (callback)
- callback(p);
+ callback(p, arg);
count++;
}
}
@@ -150,6 +183,7 @@ pmDiscoverTraverse(unsigned int flags, v
return count;
}
+
/*
* Traverse and purge deleted entries
* Return count of purged entries.
@@ -173,6 +207,9 @@ pmDiscoverPurgeDeleted(void)
prev->next = next;
else
discover_hashtable[i] = next;
+ if (pmDebugOptions.discovery)
+ fprintf(stderr, "pmDiscoverPurgeDeleted: deleted %s %s\n",
+ p->context.name, pmDiscoverFlagsStr(p));
pmDiscoverFree(p);
count++;
}
@@ -180,14 +217,32 @@ pmDiscoverPurgeDeleted(void)
}
}
- if (pmDebugOptions.discovery)
- fprintf(stderr, "%s: purged %d entries\n",
- "pmDiscoverPurgeDeleted", count);
-
return count;
}
/*
+ * if string ends with given suffix then return pointer
+ * to start of suffix in string, else NULL
+ */
+static char *
+strsuffix(char *s, const char *suffix)
+{
+ int slen, suflen;
+ char *ret = NULL;
+
+ if (s && suffix) {
+ slen = strlen(s);
+ suflen = strlen(suffix);
+ if (slen >= suflen) {
+ ret = s + (slen - suflen);
+ if (strncmp(ret, suffix, suflen) != 0)
+ ret = NULL;
+ }
+ }
+ return ret;
+}
+
+/*
* Discover dirs and archives - add new entries or refresh existing.
* Call this for each top-level directory. Discovered paths are not
* automatically monitored. After discovery, need to traverse and
@@ -196,44 +251,88 @@ pmDiscoverPurgeDeleted(void)
static int
pmDiscoverArchives(const char *dir, pmDiscoverModule *module, void *arg)
{
- uv_fs_t sreq, req;
- uv_dirent_t dent;
- uv_stat_t *s;
+ DIR *dirp;
+ struct dirent *dent;
+ struct stat *s;
+ struct stat statbuf;
pmDiscover *a;
+ char *suffix;
char path[MAXNAMELEN];
- char basepath[MAXNAMELEN];
int sep = pmPathSeparator();
+ int vol;
+
+ /*
+ * note: pmDiscoverLookupAdd sets PM_DISCOVER_FLAGS_NEW
+ * if this is a newly discovered archive or directory
+ */
+ a = pmDiscoverLookupAdd(dir, module, arg);
+ a->flags |= PM_DISCOVER_FLAGS_DIRECTORY;
- if (uv_fs_scandir(NULL, &req, dir, 0, NULL) < 0)
+ if ((dirp = opendir(dir)) == NULL) {
+ if (pmDebugOptions.discovery)
+ fprintf(stderr, "pmDiscoverArchives: opendir %s failed %s: err %d\n", dir, path, errno);
return -ESRCH;
+ }
- a = pmDiscoverAdd(dir, module, arg);
- a->flags |= PM_DISCOVER_FLAGS_DIRECTORY;
+ while ((dent = readdir(dirp)) != NULL) {
+ if (dent->d_name[0] == '.')
+ continue;
+ pmsprintf(path, sizeof(path), "%s%c%s", dir, sep, dent->d_name);
+
+ if (pmDebugOptions.discovery)
+ fprintf(stderr, "pmDiscoverArchives: readdir found %s\n", path);
- while (uv_fs_scandir_next(&req, &dent) != UV_EOF) {
- pmsprintf(path, sizeof(path), "%s%c%s", dir, sep, dent.name);
- if (uv_fs_stat(NULL, &sreq, path, NULL) < 0)
+ if (stat(path, &statbuf) < 0) {
+ if (pmDebugOptions.discovery)
+ fprintf(stderr, "pmDiscoverArchives: stat failed %s, err %d\n", path, errno);
continue;
- s = &sreq.statbuf;
- strncpy(basepath, path, sizeof(basepath)); /* __pmLogBaseName modifies it's argument */
- if (S_ISREG(s->st_mode) && __pmLogBaseName(basepath) != NULL) {
- /*
- * An archive file (index, meta or data vol). If compressed, then
- * it is read-only and we don't have to monitor it for growth.
- */
- a = pmDiscoverAdd(path, module, arg);
- a->flags &= ~PM_DISCOVER_FLAGS_DELETED;
+ }
- if (strstr(path, ".meta"))
- a->flags |= PM_DISCOVER_FLAGS_META;
- else if (strstr(path, ".index"))
- a->flags |= PM_DISCOVER_FLAGS_INDEX;
- else
- a->flags |= PM_DISCOVER_FLAGS_DATAVOL;
-
- /* compare to libpcp io.c for suffix list */
- if (strstr(path, ".xz") || strstr(path, ".gz"))
- a->flags |= PM_DISCOVER_FLAGS_COMPRESSED;
+ s = &statbuf;
+ if (S_ISREG(s->st_mode)) {
+ if ((suffix = strsuffix(path, ".meta")) != NULL) {
+ /*
+ * An uncompressed PCP archive meta file. Track the meta
+ * file - the matching logvol filename varies because logvols
+ * are periodically rolled by pmlogger. Importantly, process all
+ * available metadata to EOF before processing any logvol data.
+ */
+ *suffix = '\0'; /* strip suffix from path giving archive name */
+ a = pmDiscoverLookupAdd(path, module, arg);
+
+ /*
+ * note: pmDiscoverLookupAdd sets PM_DISCOVER_FLAGS_NEW
+ * if this is a newly discovered archive, otherwise we're
+ * already tracking this archive.
+ */
+ a->flags |= PM_DISCOVER_FLAGS_META;
+ }
+ else if ((suffix = __pmLogBaseNameVol(path, &vol)) != NULL && vol >= 0) {
+ /*
+ * An archive logvol. This logvol may have been created since
+ * the context was first opened. Update the context maxvol
+ * to be sure pmFetchArchive can switch to it in due course.
+ */
+ if ((a = pmDiscoverLookup(path)) != NULL) {
+ a->flags |= PM_DISCOVER_FLAGS_DATAVOL;
+ /* ensure archive context knows about this volume */
+ if (pmDebugOptions.discovery)
+ fprintf(stderr, "pmDiscoverArchives: found logvol %s %s vol=%d\n",
+ a->context.name, pmDiscoverFlagsStr(a), vol);
+ if (a->ctx >= 0 && vol >= 0) {
+ __pmContext *ctxp = __pmHandleToPtr(a->ctx);
+ __pmArchCtl *acp = ctxp->c_archctl;
+
+ __pmLogAddVolume(acp, vol);
+ PM_UNLOCK(ctxp->c_lock);
+ }
+ if (pmDebugOptions.discovery)
+ fprintf(stderr, "pmDiscoverArchives: added logvol %s %s vol=%d\n",
+ a->context.name, pmDiscoverFlagsStr(a), vol);
+ }
+ } else if (pmDebugOptions.discovery) {
+ fprintf(stderr, "pmDiscoverArchives: ignored regular file %s\n", path);
+ }
}
else if (S_ISDIR(s->st_mode)) {
/*
@@ -241,29 +340,117 @@ pmDiscoverArchives(const char *dir, pmDi
*/
pmDiscoverArchives(path, module, arg);
}
- uv_fs_req_cleanup(&sreq);
}
- uv_fs_req_cleanup(&req);
+ if (dirp)
+ closedir(dirp);
/* success */
return 0;
}
+/*
+ * Return 1 if monitored path has been deleted.
+ * For archives, we only check the meta file because
+ * a logvol can be deleted (e.g. via compression when
+ * the logvol is rolled to a new volume) without
+ * actually deleting the archive.
+ */
+static int
+is_deleted(pmDiscover *p, struct stat *sbuf)
+{
+ int ret = 0;
+
+ if (p->flags & PM_DISCOVER_FLAGS_DIRECTORY) {
+ if (stat(p->context.name, sbuf) < 0)
+ ret = 1; /* directory has been deleted */
+ }
+
+ if (p->flags & (PM_DISCOVER_FLAGS_META|PM_DISCOVER_FLAGS_DATAVOL)) {
+ sds meta = sdsnew(p->context.name);
+ meta = sdscat(meta, ".meta");
+ if (stat(meta, sbuf) < 0) {
+ /*
+ * Archive metadata file has been deleted (or compressed)
+ * hence consider the archive to be deleted because there
+ * is no more data to logtail.
+ */
+ ret = 1;
+ }
+ sdsfree(meta);
+ }
+
+ if (pmDebugOptions.discovery) {
+ fprintf(stderr, "is_deleted: checking %s (%s) -> %s\n",
+ p->context.name, pmDiscoverFlagsStr(p), ret ? "DELETED" : "no");
+ }
+
+ return ret;
+}
+
+static void
+logdir_is_locked_callBack(pmDiscover *p, void *arg)
+{
+ int *cntp = (int *)arg;
+ char sep = pmPathSeparator();
+ char path[MAXNAMELEN];
+
+ pmsprintf(path, sizeof(path), "%s%c%s", p->context.name, sep, "lock");
+ if (access(path, F_OK) == 0)
+ (*cntp)++;
+}
+
+static void
+check_deleted(pmDiscover *p)
+{
+ struct stat sbuf;
+ if (!(p->flags & PM_DISCOVER_FLAGS_DELETED) && is_deleted(p, &sbuf))
+ p->flags |= PM_DISCOVER_FLAGS_DELETED;
+}
+
static void
fs_change_callBack(uv_fs_event_t *handle, const char *filename, int events, int status)
{
char buffer[MAXNAMELEN];
size_t bytes = sizeof(buffer) - 1;
pmDiscover *p;
- uv_fs_t sreq;
+ char *s;
sds path;
- int path_changed = 0;
+ int count = 0;
+ struct stat statbuf;
+
+ /*
+ * check if logs are currently being rolled by pmlogger_daily et al
+ * in any of the directories we are tracking. For mutex, the log control
+ * scripts use a 'lock' file in each directory as it is processed.
+ */
+ pmDiscoverTraverseArg(PM_DISCOVER_FLAGS_DIRECTORY,
+ logdir_is_locked_callBack, (void *)&count);
+
+ if (lockcnt == 0 && count > 0) {
+ /* log-rolling has started */
+ fprintf(stderr, "%s discovery callback ignored: log-rolling is now in progress\n", stamp());
+ lockcnt = count;
+ return;
+ }
+
+ if (lockcnt > 0 && count > 0) {
+ /* log-rolling is still in progress */
+ lockcnt = count;
+ return;
+ }
+
+ if (lockcnt > 0 && count == 0) {
+ /* log-rolling is finished: check what got deleted, and then purge */
+ fprintf(stderr, "%s discovery callback: finished log-rolling\n", stamp());
+ pmDiscoverTraverse(PM_DISCOVER_FLAGS_META|PM_DISCOVER_FLAGS_DATAVOL, check_deleted);
+ }
+ lockcnt = count;
uv_fs_event_getpath(handle, buffer, &bytes);
path = sdsnewlen(buffer, bytes);
if (pmDebugOptions.discovery) {
- fprintf(stderr, "%s: event on %s -", "fs_change_callBack", path);
+ fprintf(stderr, "fs_change_callBack: event on %s -", path);
if (events & UV_RENAME)
fprintf(stderr, " renamed");
if (events & UV_CHANGE)
@@ -271,38 +458,40 @@ fs_change_callBack(uv_fs_event_t *handle
fputc('\n', stderr);
}
+
/*
- * Lookup the path, stat and update it's flags accordingly. If the
- * path has been deleted, stop it's event monitor and free the req buffer.
- * Then call the pmDiscovery callback.
+ * Strip ".meta" suffix (if any) and lookup the path. stat and update it's
+ * flags accordingly. If the path has been deleted, stop it's event monitor
+ * and free the req buffer, else call the pmDiscovery callback.
*/
- if ((p = pmDiscoverLookup(path)) == NULL) {
+ if ((s = strsuffix(path, ".meta")) != NULL)
+ *s = '\0';
+
+ p = pmDiscoverLookup(path);
+ if (p && pmDebugOptions.discovery) {
+ fprintf(stderr, "fs_change_callBack: ---> found entry %s (%s)\n",
+ p->context.name, pmDiscoverFlagsStr(p));
+ }
+
+ if (p == NULL) {
if (pmDebugOptions.discovery)
- fprintf(stderr, "%s: filename %s lookup failed\n",
- "fs_change_callBack", filename);
+ fprintf(stderr, "fs_change_callBack: %s lookup failed\n", filename);
}
- else if (uv_fs_stat(NULL, &sreq, p->context.name, NULL) < 0) {
- p->flags |= PM_DISCOVER_FLAGS_DELETED;
- if (p->event_handle) {
- uv_fs_event_stop(p->event_handle);
- free(p->event_handle);
- p->event_handle = NULL;
- }
+ else if (is_deleted(p, &statbuf)) {
/* path has been deleted. statbuf is invalid */
+ p->flags |= PM_DISCOVER_FLAGS_DELETED;
memset(&p->statbuf, 0, sizeof(p->statbuf));
- path_changed = 1;
- }
- else {
- /* avoid spurious events. only call the callBack if it really changed */
- if (p->statbuf.st_mtim.tv_sec != sreq.statbuf.st_mtim.tv_sec ||
- p->statbuf.st_mtim.tv_nsec != sreq.statbuf.st_mtim.tv_nsec)
- path_changed = 1;
- p->statbuf = sreq.statbuf; /* struct copy */
- uv_fs_req_cleanup(&sreq);
+ if (pmDebugOptions.discovery)
+ fprintf(stderr, "fs_change_callBack: %s (%s) has been deleted",
+ p->context.name, pmDiscoverFlagsStr(p));
}
- if (p && p->changed && path_changed && !(p->flags & PM_DISCOVER_FLAGS_DELETED))
- p->changed(p);
+ /*
+ * Something in the directory changed - new or deleted archive, or
+ * a tracked archive meta data file or logvolume grew
+ */
+ if (p)
+ p->changed(p); /* returns immediately if PM_DISCOVER_FLAGS_DELETED */
sdsfree(path);
}
@@ -316,9 +505,14 @@ pmDiscoverMonitor(sds path, void (*callb
{
discoverModuleData *data;
pmDiscover *p;
+ sds eventfilename;
- if ((p = pmDiscoverLookup(path)) == NULL)
+ if ((p = pmDiscoverLookup(path)) == NULL) {
+ if (pmDebugOptions.discovery) {
+ fprintf(stderr, "pmDiscoverMonitor: lookup failed for %s\n", path);
+ }
return -ESRCH;
+ }
data = getDiscoverModuleData(p->module);
/* save the discovery callback to be invoked */
@@ -330,9 +524,29 @@ pmDiscoverMonitor(sds path, void (*callb
* Start monitoring, using given uv loop. Up to the caller to create
* a PCP PMAPI context and to fetch/logtail in the changed callback.
*/
+ eventfilename = sdsnew(p->context.name);
uv_fs_event_init(data->events, p->event_handle);
- uv_fs_event_start(p->event_handle, fs_change_callBack, p->context.name,
+
+ if (p->flags & PM_DISCOVER_FLAGS_DIRECTORY) {
+ uv_fs_event_start(p->event_handle, fs_change_callBack, eventfilename,
+ UV_FS_EVENT_WATCH_ENTRY);
+ }
+ else {
+ /*
+ * Monitor an archive file. This tracks the archive meta file
+ * but the change callback processes both meta and logvol on
+ * every callback (meta before logvol).
+ */
+ eventfilename = sdscat(eventfilename, ".meta");
+ uv_fs_event_start(p->event_handle, fs_change_callBack, eventfilename,
UV_FS_EVENT_WATCH_ENTRY);
+ }
+
+ if (pmDebugOptions.discovery) {
+ fprintf(stderr, "pmDiscoverMonitor: added event for %s (%s)\n",
+ eventfilename, pmDiscoverFlagsStr(p));
+ }
+ sdsfree(eventfilename);
}
return 0;
@@ -411,41 +625,23 @@ static void changed_callback(pmDiscover
static void
created_callback(pmDiscover *p)
{
+ if (p->flags & (PM_DISCOVER_FLAGS_COMPRESSED|PM_DISCOVER_FLAGS_INDEX))
+ return; /* compressed archives don't grow and we ignore archive index files */
+
if (pmDebugOptions.discovery)
fprintf(stderr, "CREATED %s, %s\n", p->context.name, pmDiscoverFlagsStr(p));
- p->flags &= ~PM_DISCOVER_FLAGS_NEW;
-
- if (p->flags & PM_DISCOVER_FLAGS_COMPRESSED)
- return; /* compressed archives don't grow */
-
if (p->flags & PM_DISCOVER_FLAGS_DIRECTORY) {
if (pmDebugOptions.discovery)
fprintf(stderr, "MONITOR directory %s\n", p->context.name);
pmDiscoverMonitor(p->context.name, changed_callback);
}
-
- if (p->flags & PM_DISCOVER_FLAGS_DATAVOL) {
+ else if (p->flags & (PM_DISCOVER_FLAGS_META|PM_DISCOVER_FLAGS_DATAVOL)) {
if (pmDebugOptions.discovery)
- fprintf(stderr, "MONITOR logvol %s\n", p->context.name);
+ fprintf(stderr, "MONITOR archive %s\n", p->context.name);
pmDiscoverMonitor(p->context.name, changed_callback);
}
-
- if (p->flags & PM_DISCOVER_FLAGS_META) {
- if (pmDebugOptions.discovery)
- fprintf(stderr, "MONITOR metadata %s\n", p->context.name);
- pmDiscoverMonitor(p->context.name, changed_callback);
- }
-}
-
-static void
-deleted_callback(pmDiscover *p)
-{
- if (pmDebugOptions.discovery)
- fprintf(stderr, "DELETED %s (%s)\n", p->context.name,
- pmDiscoverFlagsStr(p));
- pmDiscoverDelete(p->context.name);
- /* p is now no longer valid */
+ p->flags &= ~PM_DISCOVER_FLAGS_NEW;
}
static void
@@ -509,37 +705,84 @@ static void
pmDiscoverInvokeMetricCallBacks(pmDiscover *p, pmTimespec *ts, pmDesc *desc,
int numnames, char **names)
{
+ discoverModuleData *data = getDiscoverModuleData(p->module);
pmDiscoverCallBacks *callbacks;
pmDiscoverEvent event;
char buf[32];
- int i;
+ int i, sts;
if (pmDebugOptions.discovery) {
fprintf(stderr, "%s[%s]: %s name%s", "pmDiscoverInvokeMetricCallBacks",
timespec_str(ts, buf, sizeof(buf)),
p->context.source, numnames > 0 ? " " : "(none)\n");
for (i = 0; i < numnames; i++)
- printf("\"%s\"%s", names[i], i < numnames - 1 ? ", " : "\n");
+ fprintf(stderr, "[%u/%u] \"%s\"%s", i+1, numnames, names[i],
+ i < numnames - 1 ? ", " : "\n");
pmPrintDesc(stderr, desc);
if (pmDebugOptions.labels)
fprintf(stderr, "context labels %s\n", p->context.labelset->json);
}
+ if (data->pmids) {
+ if (dictFind(data->pmids, &desc->pmid) != NULL)
+ goto out; /* metric contains an already excluded PMID */
+ for (i = 0; i < numnames; i++) {
+ if (regexec(&data->exclude_names, names[i], 0, NULL, 0) == 0)
+ break;
+ }
+ if (i != numnames) {
+ if (pmDebugOptions.discovery)
+ fprintf(stderr, "%s: excluding metric %s\n",
+ "pmDiscoverInvokeMetricCallBacks", names[i]);
+ /* add this pmid to the exclusion list and return early */
+ dictAdd(data->pmids, &desc->pmid, NULL);
+ goto out;
+ }
+ }
+ if (data->indoms) {
+ if (dictFind(data->indoms, &desc->indom) != NULL)
+ goto out; /* metric contains an already excluded InDom */
+ }
+
+ if (p->ctx >= 0 && p->context.type == PM_CONTEXT_ARCHIVE) {
+ __pmContext *ctxp = __pmHandleToPtr(p->ctx);
+ __pmArchCtl *acp = ctxp->c_archctl;
+ char idstr[32];
+
+ if ((sts = __pmLogAddDesc(acp, desc)) < 0)
+ fprintf(stderr, "%s: failed to add metric descriptor for %s\n",
+ "pmDiscoverInvokeMetricCallBacks",
+ pmIDStr_r(desc->pmid, idstr, sizeof(idstr)));
+ for (i = 0; i < numnames; i++) {
+ if ((sts = __pmLogAddPMNSNode(acp, desc->pmid, names[i])) < 0)
+ fprintf(stderr, "%s: failed to add metric name %s for %s\n",
+ "pmDiscoverInvokeMetricCallBacks", names[i],
+ pmIDStr_r(desc->pmid, idstr, sizeof(idstr)));
+ }
+ PM_UNLOCK(ctxp->c_lock);
+ }
+
discover_event_init(p, ts, &event);
for (i = 0; i < discoverCallBackTableSize; i++) {
if ((callbacks = discoverCallBackTable[i]) &&
callbacks->on_metric != NULL)
callbacks->on_metric(&event, desc, numnames, names, p->data);
}
+
+out:
+ for (i = 0; i < numnames; i++)
+ free(names[i]);
+ free(names);
}
static void
pmDiscoverInvokeInDomCallBacks(pmDiscover *p, pmTimespec *ts, pmInResult *in)
{
+ discoverModuleData *data = getDiscoverModuleData(p->module);
pmDiscoverCallBacks *callbacks;
pmDiscoverEvent event;
char buf[32], inbuf[32];
- int i;
+ int i, sts = PMLOGPUTINDOM_DUP; /* free after callbacks */
if (pmDebugOptions.discovery) {
fprintf(stderr, "%s[%s]: %s numinst %d indom %s\n",
@@ -551,22 +794,48 @@ pmDiscoverInvokeInDomCallBacks(pmDiscove
fprintf(stderr, "context labels %s\n", p->context.labelset->json);
}
+ if (data->indoms) {
+ if (dictFind(data->indoms, &in->indom) != NULL)
+ goto out; /* excluded InDom */
+ }
+
+ if (p->ctx >= 0 && p->context.type == PM_CONTEXT_ARCHIVE) {
+ __pmContext *ctxp = __pmHandleToPtr(p->ctx);
+ __pmArchCtl *acp = ctxp->c_archctl;
+ char errmsg[PM_MAXERRMSGLEN];
+
+ if ((sts = __pmLogAddInDom(acp, ts, in, NULL, 0)) < 0)
+ fprintf(stderr, "%s: failed to add indom for %s: %s\n",
+ "pmDiscoverInvokeInDomCallBacks", pmIDStr(in->indom),
+ pmErrStr_r(sts, errmsg, sizeof(errmsg)));
+ PM_UNLOCK(ctxp->c_lock);
+ }
+
discover_event_init(p, ts, &event);
for (i = 0; i < discoverCallBackTableSize; i++) {
if ((callbacks = discoverCallBackTable[i]) &&
callbacks->on_indom != NULL)
callbacks->on_indom(&event, in, p->data);
}
+
+out:
+ if (sts == PMLOGPUTINDOM_DUP) {
+ for (i = 0; i < in->numinst; i++)
+ free(in->namelist[i]);
+ free(in->namelist);
+ free(in->instlist);
+ }
}
static void
pmDiscoverInvokeLabelsCallBacks(pmDiscover *p, pmTimespec *ts,
int ident, int type, pmLabelSet *sets, int nsets)
{
+ discoverModuleData *data = getDiscoverModuleData(p->module);
pmDiscoverCallBacks *callbacks;
pmDiscoverEvent event;
char buf[32], idbuf[64];
- int i;
+ int i, sts = -EAGAIN; /* free labelsets after callbacks */
if (pmDebugOptions.discovery) {
__pmLabelIdentString(ident, type, idbuf, sizeof(idbuf));
@@ -579,22 +848,48 @@ pmDiscoverInvokeLabelsCallBacks(pmDiscov
fprintf(stderr, "context labels %s\n", p->context.labelset->json);
}
+ if ((type & PM_LABEL_ITEM) && data->pmids) {
+ if (dictFind(data->pmids, &ident) != NULL)
+ goto out; /* text from an already excluded InDom */
+ }
+ if ((type & (PM_LABEL_INDOM|PM_LABEL_INSTANCES)) && data->indoms) {
+ if (dictFind(data->indoms, &ident) != NULL)
+ goto out; /* text from an already excluded InDom */
+ }
+
+ if (p->ctx >= 0 && p->context.type == PM_CONTEXT_ARCHIVE) {
+ __pmContext *ctxp = __pmHandleToPtr(p->ctx);
+ __pmArchCtl *acp = ctxp->c_archctl;
+ char errmsg[PM_MAXERRMSGLEN];
+
+ if ((sts = __pmLogAddLabelSets(acp, ts, type, ident, nsets, sets)) < 0)
+ fprintf(stderr, "%s: failed to add log labelset: %s\n",
+ "pmDiscoverInvokeLabelsCallBacks",
+ pmErrStr_r(sts, errmsg, sizeof(errmsg)));
+ PM_UNLOCK(ctxp->c_lock);
+ }
+
discover_event_init(p, ts, &event);
for (i = 0; i < discoverCallBackTableSize; i++) {
if ((callbacks = discoverCallBackTable[i]) &&
callbacks->on_labels != NULL)
callbacks->on_labels(&event, ident, type, sets, nsets, p->data);
}
+
+out:
+ if (sts < 0)
+ pmFreeLabelSets(sets, nsets);
}
static void
pmDiscoverInvokeTextCallBacks(pmDiscover *p, pmTimespec *ts,
int ident, int type, char *text)
{
+ discoverModuleData *data = getDiscoverModuleData(p->module);
pmDiscoverCallBacks *callbacks;
pmDiscoverEvent event;
char buf[32];
- int i;
+ int i, sts;
if (pmDebugOptions.discovery) {
fprintf(stderr, "%s[%s]: %s ", "pmDiscoverInvokeTextCallBacks",
@@ -612,12 +907,36 @@ pmDiscoverInvokeTextCallBacks(pmDiscover
fprintf(stderr, "context labels %s\n", p->context.labelset->json);
}
+ if ((type & PM_TEXT_PMID) && data->pmids) {
+ if (dictFind(data->pmids, &ident) != NULL)
+ goto out; /* text from an already excluded InDom */
+ }
+ if ((type & PM_TEXT_INDOM) && data->indoms) {
+ if (dictFind(data->indoms, &ident) != NULL)
+ goto out; /* text from an already excluded InDom */
+ }
+
+ if (p->ctx >= 0 && p->context.type == PM_CONTEXT_ARCHIVE) {
+ __pmContext *ctxp = __pmHandleToPtr(p->ctx);
+ __pmArchCtl *acp = ctxp->c_archctl;
+ char errmsg[PM_MAXERRMSGLEN];
+
+ if ((sts = __pmLogAddText(acp, ident, type, text)) < 0)
+ fprintf(stderr, "%s: failed to add %u text for %u: %s\n",
+ "pmDiscoverInvokeTextCallBacks", type, ident,
+ pmErrStr_r(sts, errmsg, sizeof(errmsg)));
+ PM_UNLOCK(ctxp->c_lock);
+ }
+
discover_event_init(p, ts, &event);
for (i = 0; i < discoverCallBackTableSize; i++) {
if ((callbacks = discoverCallBackTable[i]) &&
callbacks->on_text != NULL)
callbacks->on_text(&event, ident, type, text, p->data);
}
+
+out:
+ free(text);
}
static void
@@ -645,8 +964,8 @@ pmDiscoverNewSource(pmDiscover *p, int c
p->context.labelset = labelset;
/* use timestamp from file creation as starting time */
- timestamp.tv_sec = p->statbuf.st_birthtim.tv_sec;
- timestamp.tv_nsec = p->statbuf.st_birthtim.tv_nsec;
+ timestamp.tv_sec = p->statbuf.st_ctim.tv_sec;
+ timestamp.tv_nsec = p->statbuf.st_ctim.tv_nsec;
/* inform utilities that a source has been discovered */
pmDiscoverInvokeSourceCallBacks(p, &timestamp);
@@ -664,7 +983,7 @@ process_metadata(pmDiscover *p)
pmDesc desc;
off_t off;
char *buffer;
- int e, i, nb, len, nsets;
+ int e, nb, len, nsets;
int type, id; /* pmID or pmInDom */
int nnames;
char **names;
@@ -674,6 +993,8 @@ process_metadata(pmDiscover *p)
__pmLogHdr hdr;
sds msg, source;
static uint32_t *buf = NULL;
+ int deleted;
+ struct stat sbuf;
static int buflen = 0;
/*
@@ -683,14 +1004,17 @@ process_metadata(pmDiscover *p)
*/
p->flags |= PM_DISCOVER_FLAGS_META_IN_PROGRESS;
if (pmDebugOptions.discovery)
- fprintf(stderr, "%s: in progress, flags=%s\n",
- "process_metadata", pmDiscoverFlagsStr(p));
+ fprintf(stderr, "process_metadata: %s in progress %s\n",
+ p->context.name, pmDiscoverFlagsStr(p));
for (;;) {
off = lseek(p->fd, 0, SEEK_CUR);
nb = read(p->fd, &hdr, sizeof(__pmLogHdr));
- if (nb <= 0) {
- /* we're at EOF or an error. But may still be part way through a record */
+ deleted = is_deleted(p, &sbuf);
+ if (nb <= 0 || deleted) {
+ /* we're at EOF or an error, or deleted. But may still be part way through a record */
+ if (deleted)
+ p->flags |= PM_DISCOVER_FLAGS_DELETED;
break;
}
@@ -750,10 +1074,6 @@ process_metadata(pmDiscover *p)
ts.tv_sec = p->statbuf.st_mtim.tv_sec;
ts.tv_nsec = p->statbuf.st_mtim.tv_nsec;
pmDiscoverInvokeMetricCallBacks(p, &ts, &desc, nnames, names);
- for (i = 0; i < nnames; i++)
- free(names[i]);
- if (names)
- free(names);
break;
case TYPE_INDOM:
@@ -765,12 +1085,6 @@ process_metadata(pmDiscover *p)
break;
}
pmDiscoverInvokeInDomCallBacks(p, &ts, &inresult);
- if (inresult.numinst > 0) {
- for (i = 0; i < inresult.numinst; i++)
- free(inresult.namelist[i]);
- free(inresult.namelist);
- free(inresult.instlist);
- }
break;
case TYPE_LABEL:
@@ -795,13 +1109,13 @@ process_metadata(pmDiscover *p)
} else {
sdsfree(p->context.source);
p->context.source = source;
- p->context.labelset = labelset;
+ if (p->context.labelset)
+ pmFreeLabelSets(p->context.labelset, 1);
+ p->context.labelset = __pmDupLabelSets(labelset, 1);
pmDiscoverInvokeSourceCallBacks(p, &ts);
}
}
pmDiscoverInvokeLabelsCallBacks(p, &ts, id, type, labelset, nsets);
- if (labelset != p->context.labelset)
- pmFreeLabelSets(labelset, nsets);
break;
case TYPE_TEXT:
@@ -819,8 +1133,6 @@ process_metadata(pmDiscover *p)
ts.tv_sec = p->statbuf.st_mtim.tv_sec;
ts.tv_nsec = p->statbuf.st_mtim.tv_nsec;
pmDiscoverInvokeTextCallBacks(p, &ts, id, type, buffer);
- if (buffer)
- free(buffer);
break;
default:
@@ -833,38 +1145,89 @@ process_metadata(pmDiscover *p)
}
if (partial == 0)
- /* flag that all available metadata has been now been read */
+ /* flag that all available metadata has now been read */
p->flags &= ~PM_DISCOVER_FLAGS_META_IN_PROGRESS;
if (pmDebugOptions.discovery)
- fprintf(stderr, "%s : completed, partial=%d flags=%s\n",
- "process_metadata", partial, pmDiscoverFlagsStr(p));
+ fprintf(stderr, "%s: completed, partial=%d %s %s\n",
+ "process_metadata", partial, p->context.name, pmDiscoverFlagsStr(p));
}
/*
- * fetch metric values to EOF and call all registered callbacks
+ * Fetch metric values to EOF and call all registered callbacks.
+ * Always process metadata thru to EOF before any logvol data.
*/
static void
-process_logvol_callback(pmDiscover *p)
+process_logvol(pmDiscover *p)
{
+ int sts;
pmResult *r;
pmTimespec ts;
+ int oldcurvol;
+ __pmContext *ctxp;
+ __pmArchCtl *acp;
+
+ for (;;) {
+ pmUseContext(p->ctx);
+ ctxp = __pmHandleToPtr(p->ctx);
+ acp = ctxp->c_archctl;
+ oldcurvol = acp->ac_curvol;
+ PM_UNLOCK(ctxp->c_lock);
+
+ if ((sts = pmFetchArchive(&r)) < 0) {
+ /* err handling to skip to the next vol */
+ ctxp = __pmHandleToPtr(p->ctx);
+ acp = ctxp->c_archctl;
+ if (oldcurvol < acp->ac_curvol) {
+ __pmLogChangeVol(acp, acp->ac_curvol);
+ acp->ac_offset = 0; /* __pmLogFetch will fix it up */
+ }
+ PM_UNLOCK(ctxp->c_lock);
+
+ if (sts == PM_ERR_EOL) {
+ if (pmDebugOptions.discovery)
+ fprintf(stderr, "process_logvol: %s end of archive reached\n",
+ p->context.name);
+
+ /* succesfully processed to current end of log */
+ break;
+ } else {
+ /*
+ * This log vol was probably deleted (likely compressed)
+ * under our feet. Try and skip to the next volume.
+ * We hold the context lock during error recovery here.
+ */
+ if (pmDebugOptions.discovery)
+ fprintf(stderr, "process_logvol: %s fetch failed:%s\n",
+ p->context.name, pmErrStr(sts));
+ }
- pmUseContext(p->ctx);
- while (pmFetchArchive(&r) == 0) {
+ /* we are done - return and wait for another callback */
+ break;
+ }
+
+ /*
+ * Fetch succeeded - call the values callback and continue
+ */
if (pmDebugOptions.discovery) {
char tbuf[64], bufs[64];
- fprintf(stderr, "FETCHED @%s [%s] %d metrics\n",
- timeval_str(&r->timestamp, tbuf, sizeof(tbuf)),
+ fprintf(stderr, "process_logvol: %s FETCHED @%s [%s] %d metrics\n",
+ p->context.name, timeval_str(&r->timestamp, tbuf, sizeof(tbuf)),
timeval_stream_str(&r->timestamp, bufs, sizeof(bufs)),
r->numpmid);
}
+
+ /*
+ * TODO: persistently save current timestamp, so after being restarted,
+ * pmproxy can resume where it left off for each archive.
+ */
ts.tv_sec = r->timestamp.tv_sec;
ts.tv_nsec = r->timestamp.tv_usec * 1000;
pmDiscoverInvokeValuesCallBack(p, &ts, r);
pmFreeResult(r);
}
+
/* datavol is now up-to-date and at EOF */
p->flags &= ~PM_DISCOVER_FLAGS_DATAVOL_READY;
}
@@ -874,12 +1237,13 @@ pmDiscoverInvokeCallBacks(pmDiscover *p)
{
int sts;
sds msg;
+ sds metaname;
if (p->ctx < 0) {
/*
* once off initialization on the first event
*/
- if (p->flags & PM_DISCOVER_FLAGS_DATAVOL) {
+ if (p->flags & (PM_DISCOVER_FLAGS_DATAVOL | PM_DISCOVER_FLAGS_META)) {
struct timeval tvp;
/* create the PMAPI context (once off) */
@@ -898,28 +1262,25 @@ pmDiscoverInvokeCallBacks(pmDiscover *p)
p->ctx = -1;
return;
}
+ /* seek to end of archive for logvol data - see TODO in process_logvol() */
pmSetMode(PM_MODE_FORW, &tvp, 1);
- /* note: we do not scan pre-existing logvol data. */
- }
- else if (p->flags & PM_DISCOVER_FLAGS_META) {
- if ((sts = pmNewContext(p->context.type, p->context.name)) < 0) {
- infofmt(msg, "pmNewContext failed for %s: %s\n",
- p->context.name, pmErrStr(sts));
- moduleinfo(p->module, PMLOG_ERROR, msg, p->data);
- return;
- }
- pmDiscoverNewSource(p, sts);
- /* for archive meta files, p->fd is the direct file descriptor */
- if ((p->fd = open(p->context.name, O_RDONLY)) < 0) {
- infofmt(msg, "open failed for %s: %s\n", p->context.name,
- osstrerror());
+ /*
+ * For archive meta files, p->fd is the direct file descriptor
+ * and we pre-scan existing metadata. Note: we do NOT scan
+ * pre-existing logvol data (see pmSetMode above)
+ */
+ metaname = sdsnew(p->context.name);
+ metaname = sdscat(metaname, ".meta");
+ if ((p->fd = open(metaname, O_RDONLY)) < 0) {
+ infofmt(msg, "open failed for %s: %s\n", metaname, osstrerror());
moduleinfo(p->module, PMLOG_ERROR, msg, p->data);
+ sdsfree(metaname);
return;
}
-
- /* process all existing metadata */
+ /* pre-process all existing metadata */
process_metadata(p);
+ sdsfree(metaname);
}
}
@@ -943,15 +1304,61 @@ pmDiscoverInvokeCallBacks(pmDiscover *p)
}
if (p->flags & PM_DISCOVER_FLAGS_META) {
- /* process metadata */
+ /* process new metadata, if any */
process_metadata(p);
}
- /* process any unprocessed datavol callbacks */
- pmDiscoverTraverse(PM_DISCOVER_FLAGS_DATAVOL_READY, process_logvol_callback);
+ if ((p->flags & PM_DISCOVER_FLAGS_META_IN_PROGRESS) == 0) {
+ /* no metdata read in progress, so process new datavol data, if any */
+ process_logvol(p);
+ }
+}
+
+static void
+print_callback(pmDiscover *p)
+{
+ if (p->flags & PM_DISCOVER_FLAGS_DIRECTORY) {
+ fprintf(stderr, " DIRECTORY %s %s\n",
+ p->context.name, pmDiscoverFlagsStr(p));
+ }
+ else {
+ __pmContext *ctxp;
+ __pmArchCtl *acp;
- /* finally, purge deleted entries, if any */
- pmDiscoverPurgeDeleted();
+ if (p->ctx >= 0 && (ctxp = __pmHandleToPtr(p->ctx)) != NULL) {
+ acp = ctxp->c_archctl;
+ fprintf(stderr, " ARCHIVE %s fd=%d ctx=%d maxvol=%d ac_curvol=%d ac_offset=%ld %s\n",
+ p->context.name, p->fd, p->ctx, acp->ac_log->l_maxvol, acp->ac_curvol,
+ acp->ac_offset, pmDiscoverFlagsStr(p));
+ PM_UNLOCK(ctxp->c_lock);
+ } else {
+ /* no context yet - probably PM_DISCOVER_FLAGS_NEW */
+ fprintf(stderr, " ARCHIVE %s fd=%d ctx=%d %s\n",
+ p->context.name, p->fd, p->ctx, pmDiscoverFlagsStr(p));
+ }
+ }
+}
+
+/*
+ * p is a tracked archive and arg is a directory path.
+ * If p is in the directory, call it's callbacks to
+ * process metadata and logvol data. This allows better
+ * scalability because we only process archives in the
+ * directories that have changed.
+ */
+static void
+directory_changed_cb(pmDiscover *p, void *arg)
+{
+ char *dirpath = (char *)arg;
+ int dlen = strlen(dirpath);
+
+ if (strncmp(p->context.name, dirpath, dlen) == 0) {
+ /* this archive is in this directory - process it's metadata and logvols */
+ if (pmDebugOptions.discovery)
+ fprintf(stderr, "directory_changed_cb: archive %s is in dir %s\n",
+ p->context.name, dirpath);
+ pmDiscoverInvokeCallBacks(p);
+ }
}
static void
@@ -962,27 +1369,46 @@ changed_callback(pmDiscover *p)
pmDiscoverFlagsStr(p));
if (p->flags & PM_DISCOVER_FLAGS_DELETED) {
- /* path or directory has been deleted - remove from hash table */
- deleted_callback(p);
- }
- else if (p->flags & PM_DISCOVER_FLAGS_DIRECTORY) {
/*
- * A changed directory path means a new archive or subdirectory
- * has been created - traverse and update the hash table.
+ * Path has been deleted. Do nothing for now. Will be purged
+ * in due course by pmDiscoverPurgeDeleted.
*/
- pmDiscoverArchives(p->context.name, p->module, p->data);
- pmDiscoverTraverse(PM_DISCOVER_FLAGS_NEW, created_callback);
+ return;
+
}
- else if (p->flags & PM_DISCOVER_FLAGS_COMPRESSED) {
+
+ if (p->flags & PM_DISCOVER_FLAGS_COMPRESSED) {
/* we do not monitor compressed files - do nothing */
- ; /**/
+ return;
}
- else if (p->flags & (PM_DISCOVER_FLAGS_DATAVOL|PM_DISCOVER_FLAGS_META)) {
- /*
- * We only monitor uncompressed logvol and metadata paths. Fetch new data
- * (metadata or logvol) and call the registered callbacks.
+
+ if (p->flags & PM_DISCOVER_FLAGS_DIRECTORY) {
+ /*
+ * A changed directory path means a new archive or subdirectory may have
+ * been created or deleted - traverse and update the hash table.
*/
- pmDiscoverInvokeCallBacks(p);
+ if (pmDebugOptions.discovery) {
+ fprintf(stderr, "%s DIRECTORY CHANGED %s (%s)\n",
+ stamp(), p->context.name, pmDiscoverFlagsStr(p));
+ }
+ pmDiscoverArchives(p->context.name, p->module, p->data);
+ pmDiscoverTraverse(PM_DISCOVER_FLAGS_NEW, created_callback);
+
+ /*
+ * Walk directory and invoke callbacks for tracked archives in this
+ * directory that have changed
+ */
+ pmDiscoverTraverseArg(PM_DISCOVER_FLAGS_DATAVOL|PM_DISCOVER_FLAGS_META,
+ directory_changed_cb, (void *)p->context.name);
+
+ /* finally, purge deleted entries (globally), if any */
+ pmDiscoverPurgeDeleted();
+ }
+
+ if (pmDebugOptions.discovery) {
+ fprintf(stderr, "%s -- tracking status\n", stamp());
+ pmDiscoverTraverse(PM_DISCOVER_FLAGS_ALL, print_callback);
+ fprintf(stderr, "--\n");
}
}
@@ -995,18 +1421,9 @@ dir_callback(pmDiscover *p)
static void
archive_callback(pmDiscover *p)
{
- if (p->flags & PM_DISCOVER_FLAGS_COMPRESSED)
- return; /* compressed archives don't grow */
-
- if (p->flags & PM_DISCOVER_FLAGS_DATAVOL) {
- if (pmDebugOptions.discovery)
- fprintf(stderr, "DISCOVERED ARCHIVE LOGVOL %s\n", p->context.name);
- pmDiscoverMonitor(p->context.name, changed_callback);
- }
-
if (p->flags & PM_DISCOVER_FLAGS_META) {
if (pmDebugOptions.discovery)
- fprintf(stderr, "DISCOVERED ARCHIVE METADATA %s\n", p->context.name);
+ fprintf(stderr, "DISCOVERED ARCHIVE %s\n", p->context.name);
pmDiscoverMonitor(p->context.name, changed_callback);
}
}
@@ -1048,9 +1465,9 @@ pmDiscoverRegister(const char *dir, pmDi
}
if (pmDebugOptions.discovery) {
- fprintf(stderr, "Now managing %d directories and %d archive files\n",
+ fprintf(stderr, "Now tracking %d directories and %d archives\n",
pmDiscoverTraverse(PM_DISCOVER_FLAGS_DIRECTORY, NULL),
- pmDiscoverTraverse(PM_DISCOVER_FLAGS_DATAVOL, NULL));
+ pmDiscoverTraverse(PM_DISCOVER_FLAGS_DATAVOL|PM_DISCOVER_FLAGS_META, NULL));
}
/* monitor the directories */
diff -Naurp pcp-5.0.2.orig/src/libpcp_web/src/discover.h pcp-5.0.2/src/libpcp_web/src/discover.h
--- pcp-5.0.2.orig/src/libpcp_web/src/discover.h 2019-12-10 17:04:20.000000000 +1100
+++ pcp-5.0.2/src/libpcp_web/src/discover.h 2020-02-03 13:36:09.904659047 +1100
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 2018-2019 Red Hat.
+ * Copyright (c) 2018-2020 Red Hat.
*
* This library is free software; you can redistribute it and/or modify it
* under the terms of the GNU Lesser General Public License as published
@@ -18,7 +18,9 @@
#include "libpcp.h"
#include "mmv_stats.h"
#include "slots.h"
-
+#ifdef HAVE_REGEX_H
+#include <regex.h>
+#endif
#ifdef HAVE_LIBUV
#include <uv.h>
#else
@@ -84,8 +86,8 @@ typedef struct pmDiscover {
int fd; /* meta file descriptor */
#ifdef HAVE_LIBUV
uv_fs_event_t *event_handle; /* uv fs_notify event handle */
- uv_stat_t statbuf; /* stat buffer from event CB */
#endif
+ struct stat statbuf; /* stat buffer */
void *baton; /* private internal lib data */
void *data; /* opaque user data pointer */
} pmDiscover;
@@ -115,6 +117,10 @@ typedef struct discoverModuleData {
struct dict *config; /* configuration dict */
uv_loop_t *events; /* event library loop */
redisSlots *slots; /* server slots data */
+ regex_t exclude_names; /* metric names to exclude */
+ struct dict *pmids; /* dict of excluded PMIDs */
+ unsigned int exclude_indoms; /* exclude instance domains */
+ struct dict *indoms; /* dict of excluded InDoms */
void *data; /* user-supplied pointer */
} discoverModuleData;
diff -Naurp pcp-5.0.2.orig/src/libpcp_web/src/exports pcp-5.0.2/src/libpcp_web/src/exports
--- pcp-5.0.2.orig/src/libpcp_web/src/exports 2019-11-26 16:29:58.000000000 +1100
+++ pcp-5.0.2/src/libpcp_web/src/exports 2020-02-03 13:23:15.264762900 +1100
@@ -178,3 +178,8 @@ PCP_WEB_1.11 {
global:
pmSeriesLabelValues;
} PCP_WEB_1.10;
+
+PCP_WEB_1.12 {
+ global:
+ SDS_NOINIT;
+} PCP_WEB_1.11;
diff -Naurp pcp-5.0.2.orig/src/libpcp_web/src/load.c pcp-5.0.2/src/libpcp_web/src/load.c
--- pcp-5.0.2.orig/src/libpcp_web/src/load.c 2019-12-11 14:01:53.000000000 +1100
+++ pcp-5.0.2/src/libpcp_web/src/load.c 2020-02-03 13:36:03.947721365 +1100
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 2017-2019 Red Hat.
+ * Copyright (c) 2017-2020 Red Hat.
*
* This library is free software; you can redistribute it and/or modify it
* under the terms of the GNU Lesser General Public License as published
@@ -112,22 +112,41 @@ load_prepare_metric(const char *name, vo
* Iterate over an instance domain and extract names and labels
* for each instance.
*/
-static unsigned int
-get_instance_metadata(seriesLoadBaton *baton, pmInDom indom)
+static void
+get_instance_metadata(seriesLoadBaton *baton, pmInDom indom, int force_refresh)
{
context_t *cp = &baton->pmapi.context;
- unsigned int count = 0;
domain_t *dp;
indom_t *ip;
if (indom != PM_INDOM_NULL) {
if ((dp = pmwebapi_add_domain(cp, pmInDom_domain(indom))))
pmwebapi_add_domain_labels(cp, dp);
- if ((ip = pmwebapi_add_indom(cp, dp, indom)) &&
- (count = pmwebapi_add_indom_instances(cp, ip)) > 0)
- pmwebapi_add_instances_labels(cp, ip);
+ if ((ip = pmwebapi_add_indom(cp, dp, indom)) != NULL) {
+ if (force_refresh)
+ ip->updated = 1;
+ if (ip->updated) {
+ pmwebapi_add_indom_instances(cp, ip);
+ pmwebapi_add_instances_labels(cp, ip);
+ }
+ }
}
- return count;
+}
+
+static void
+get_metric_metadata(seriesLoadBaton *baton, metric_t *metric)
+{
+ context_t *context = &baton->pmapi.context;
+
+ if (metric->cluster) {
+ if (metric->cluster->domain)
+ pmwebapi_add_domain_labels(context, metric->cluster->domain);
+ pmwebapi_add_cluster_labels(context, metric->cluster);
+ }
+ if (metric->indom)
+ pmwebapi_add_instances_labels(context, metric->indom);
+ pmwebapi_add_item_labels(context, metric);
+ pmwebapi_metric_hash(metric);
}
static metric_t *
@@ -140,18 +159,25 @@ new_metric(seriesLoadBaton *baton, pmVal
char **nameall = NULL;
int count, sts, i;
- if ((sts = pmLookupDesc(vsp->pmid, &desc)) < 0) {
+ if ((sts = pmUseContext(context->context)) < 0) {
+ fprintf(stderr, "%s: failed to use context for PMID %s: %s\n",
+ "new_metric",
+ pmIDStr_r(vsp->pmid, idbuf, sizeof(idbuf)),
+ pmErrStr_r(sts, errmsg, sizeof(errmsg)));
+ } else if ((sts = pmLookupDesc(vsp->pmid, &desc)) < 0) {
if (sts == PM_ERR_IPC)
context->setup = 0;
if (pmDebugOptions.series)
- fprintf(stderr, "failed to lookup metric %s descriptor: %s",
+ fprintf(stderr, "%s: failed to lookup metric %s descriptor: %s\n",
+ "new_metric",
pmIDStr_r(vsp->pmid, idbuf, sizeof(idbuf)),
pmErrStr_r(sts, errmsg, sizeof(errmsg)));
} else if ((sts = count = pmNameAll(vsp->pmid, &nameall)) < 0) {
if (sts == PM_ERR_IPC)
context->setup = 0;
if (pmDebugOptions.series)
- fprintf(stderr, "failed to lookup metric %s names: %s",
+ fprintf(stderr, "%s: failed to lookup metric %s names: %s\n",
+ "new_metric",
pmIDStr_r(vsp->pmid, idbuf, sizeof(idbuf)),
pmErrStr_r(sts, errmsg, sizeof(errmsg)));
}
@@ -160,18 +186,10 @@ new_metric(seriesLoadBaton *baton, pmVal
if ((metric = pmwebapi_new_metric(context, NULL, &desc, count, nameall)) == NULL)
return NULL;
- if (metric->cluster) {
- if (metric->cluster->domain)
- pmwebapi_add_domain_labels(context, metric->cluster->domain);
- pmwebapi_add_cluster_labels(context, metric->cluster);
- }
- if (metric->indom)
- pmwebapi_add_instances_labels(context, metric->indom);
- pmwebapi_add_item_labels(context, metric);
- pmwebapi_metric_hash(metric);
+ get_metric_metadata(baton, metric);
if (pmDebugOptions.series) {
- fprintf(stderr, "new_metric [%s] names:",
+ fprintf(stderr, "%s [%s] names:\n", "new_metric",
pmIDStr_r(vsp->pmid, idbuf, sizeof(idbuf)));
for (i = 0; i < count; i++) {
pmwebapi_hash_str(metric->names[i].hash, idbuf, sizeof(idbuf));
@@ -409,7 +427,7 @@ pmwebapi_add_valueset(metric_t *metric,
}
static void
-series_cache_update(seriesLoadBaton *baton)
+series_cache_update(seriesLoadBaton *baton, struct dict *exclude)
{
seriesGetContext *context = &baton->pmapi;
context_t *cp = &context->context;
@@ -418,7 +436,7 @@ series_cache_update(seriesLoadBaton *bat
metric_t *metric = NULL;
char ts[64];
sds timestamp;
- int i, write_meta, write_data;
+ int i, write_meta, write_inst, write_data;
timestamp = sdsnew(timeval_stream_str(&result->timestamp, ts, sizeof(ts)));
write_data = (!(baton->flags & PM_SERIES_FLAG_METADATA));
@@ -441,6 +459,12 @@ series_cache_update(seriesLoadBaton *bat
dictFetchValue(baton->wanted, &vsp->pmid) == NULL)
continue;
+ /* check if metric to be skipped (optional metric exclusion) */
+ if (exclude && (dictFind(exclude, &vsp->pmid)) != NULL)
+ continue;
+
+ write_meta = write_inst = 0;
+
/* check if pmid already in hash list */
if ((metric = dictFetchValue(cp->pmids, &vsp->pmid)) == NULL) {
/* create a new metric, and add it to load context */
@@ -448,21 +472,22 @@ series_cache_update(seriesLoadBaton *bat
continue;
write_meta = 1;
} else { /* pmid already observed */
- write_meta = 0;
+ if ((write_meta = metric->cached) == 0)
+ get_metric_metadata(baton, metric);
}
/* iterate through result instances and ensure metric_t is complete */
if (metric->error == 0 && vsp->numval < 0)
write_meta = 1;
if (pmwebapi_add_valueset(metric, vsp) != 0)
- write_meta = 1;
+ write_meta = write_inst = 1;
/* record the error code in the cache */
metric->error = (vsp->numval < 0) ? vsp->numval : 0;
/* make PMAPI calls to cache metadata */
- if (write_meta && get_instance_metadata(baton, metric->desc.indom) != 0)
- continue;
+ if (write_meta)
+ get_instance_metadata(baton, metric->desc.indom, write_inst);
/* initiate writes to backend caching servers (Redis) */
server_cache_metric(baton, metric, timestamp, write_meta, write_data);
@@ -549,7 +574,7 @@ server_cache_window(void *arg)
(finish->tv_sec == result->timestamp.tv_sec &&
finish->tv_usec >= result->timestamp.tv_usec)) {
context->done = server_cache_update_done;
- series_cache_update(baton);
+ series_cache_update(baton, NULL);
}
else {
if (pmDebugOptions.series)
@@ -1023,7 +1048,7 @@ pmSeriesDiscoverSource(pmDiscoverEvent *
sds msg;
int i;
- if (data == NULL || data->slots == NULL)
+ if (data == NULL || data->slots == NULL || data->slots->setup == 0)
return;
baton = (seriesLoadBaton *)calloc(1, sizeof(seriesLoadBaton));
@@ -1032,22 +1057,31 @@ pmSeriesDiscoverSource(pmDiscoverEvent *
moduleinfo(module, PMLOG_ERROR, msg, arg);
return;
}
+ if ((set = pmwebapi_labelsetdup(p->context.labelset)) == NULL) {
+ infofmt(msg, "%s: out of memory for labels", "pmSeriesDiscoverSource");
+ moduleinfo(module, PMLOG_ERROR, msg, arg);
+ free(baton);
+ return;
+ }
+
initSeriesLoadBaton(baton, module, 0 /*flags*/,
module->on_info, series_discover_done,
data->slots, arg);
initSeriesGetContext(&baton->pmapi, baton);
p->baton = baton;
+ cp = &baton->pmapi.context;
+
if (pmDebugOptions.discovery)
- fprintf(stderr, "%s: new source %s context=%d\n",
- "pmSeriesDiscoverSource", p->context.name, p->ctx);
+ fprintf(stderr, "%s: new source %s context=%p ctxid=%d\n",
+ "pmSeriesDiscoverSource", p->context.name, cp, p->ctx);
- cp = &baton->pmapi.context;
cp->context = p->ctx;
cp->type = p->context.type;
cp->name.sds = sdsdup(p->context.name);
- cp->host = p->context.hostname;
- cp->labelset = set = p->context.labelset;
+ cp->host = sdsdup(p->context.hostname);
+ cp->labelset = set;
+
pmwebapi_source_hash(cp->name.hash, set->json, set->jsonlen);
pmwebapi_setup_context(cp);
set_source_origin(cp);
@@ -1095,21 +1129,22 @@ pmSeriesDiscoverLabels(pmDiscoverEvent *
sds msg;
int i, id;
+ if (baton == NULL || baton->slots == NULL || baton->slots->setup == 0)
+ return;
+
switch (type) {
case PM_LABEL_CONTEXT:
if (pmDebugOptions.discovery)
fprintf(stderr, "%s: context\n", "pmSeriesDiscoverLabels");
if ((labels = pmwebapi_labelsetdup(sets)) != NULL) {
-#if 0 /* PCP GH#800 do not free this labelset - it's owned by the discover code */
if (cp->labelset)
pmFreeLabelSets(cp->labelset, 1);
-#endif
cp->labelset = labels;
pmwebapi_locate_context(cp);
cp->updated = 1;
} else {
- infofmt(msg, "failed to duplicate label set");
+ infofmt(msg, "failed to duplicate %s label set", "context");
moduleinfo(event->module, PMLOG_ERROR, msg, arg);
}
break;
@@ -1125,8 +1160,8 @@ pmSeriesDiscoverLabels(pmDiscoverEvent *
pmFreeLabelSets(domain->labelset, 1);
domain->labelset = labels;
domain->updated = 1;
- } else {
- infofmt(msg, "failed to duplicate label set");
+ } else if (domain) {
+ infofmt(msg, "failed to duplicate %s label set", "domain");
moduleinfo(event->module, PMLOG_ERROR, msg, arg);
}
break;
@@ -1142,8 +1177,8 @@ pmSeriesDiscoverLabels(pmDiscoverEvent *
pmFreeLabelSets(cluster->labelset, 1);
cluster->labelset = labels;
cluster->updated = 1;
- } else {
- infofmt(msg, "failed to duplicate label set");
+ } else if (cluster) {
+ infofmt(msg, "failed to duplicate %s label set", "cluster");
moduleinfo(event->module, PMLOG_ERROR, msg, arg);
}
break;
@@ -1159,8 +1194,8 @@ pmSeriesDiscoverLabels(pmDiscoverEvent *
pmFreeLabelSets(metric->labelset, 1);
metric->labelset = labels;
metric->updated = 1;
- } else {
- infofmt(msg, "failed to duplicate label set");
+ } else if (metric) {
+ infofmt(msg, "failed to duplicate %s label set", "item");
moduleinfo(event->module, PMLOG_ERROR, msg, arg);
}
break;
@@ -1177,8 +1212,8 @@ pmSeriesDiscoverLabels(pmDiscoverEvent *
pmFreeLabelSets(indom->labelset, 1);
indom->labelset = labels;
indom->updated = 1;
- } else {
- infofmt(msg, "failed to duplicate label set");
+ } else if (indom) {
+ infofmt(msg, "failed to duplicate %s label set", "indom");
moduleinfo(event->module, PMLOG_ERROR, msg, arg);
}
break;
@@ -1196,7 +1231,7 @@ pmSeriesDiscoverLabels(pmDiscoverEvent *
if ((instance = dictFetchValue(indom->insts, &id)) == NULL)
continue;
if ((labels = pmwebapi_labelsetdup(&sets[i])) == NULL) {
- infofmt(msg, "failed to dup %s instance labels: %s",
+ infofmt(msg, "failed to dup indom %s instance label set: %s",
pmInDomStr_r(indom->indom, idbuf, sizeof(idbuf)),
pmErrStr_r(-ENOMEM, errmsg, sizeof(errmsg)));
moduleinfo(event->module, PMLOG_ERROR, msg, arg);
@@ -1229,10 +1264,13 @@ pmSeriesDiscoverMetric(pmDiscoverEvent *
if (pmDebugOptions.discovery) {
for (i = 0; i < numnames; i++)
- fprintf(stderr, "pmSeriesDiscoverMetric: [%d/%d] %s - %s\n",
+ fprintf(stderr, "%s: [%d/%d] %s - %s\n", "pmSeriesDiscoverMetric",
i + 1, numnames, pmIDStr(desc->pmid), names[i]);
}
+ if (baton == NULL || baton->slots == NULL || baton->slots->setup == 0)
+ return;
+
if ((metric = pmwebapi_add_metric(&baton->pmapi.context,
NULL, desc, numnames, names)) == NULL) {
infofmt(msg, "%s: failed metric discovery", "pmSeriesDiscoverMetric");
@@ -1244,18 +1282,23 @@ pmSeriesDiscoverMetric(pmDiscoverEvent *
void
pmSeriesDiscoverValues(pmDiscoverEvent *event, pmResult *result, void *arg)
{
+ pmDiscoverModule *module = event->module;
pmDiscover *p = (pmDiscover *)event->data;
seriesLoadBaton *baton = p->baton;
seriesGetContext *context = &baton->pmapi;
+ discoverModuleData *data = getDiscoverModuleData(module);
if (pmDebugOptions.discovery)
fprintf(stderr, "%s: result numpmids=%d\n", "pmSeriesDiscoverValues", result->numpmid);
+ if (baton == NULL || baton->slots == NULL || baton->slots->setup == 0)
+ return;
+
seriesBatonReference(context, "pmSeriesDiscoverValues");
baton->arg = arg;
context->result = result;
- series_cache_update(baton);
+ series_cache_update(baton, data->pmids);
}
void
@@ -1271,7 +1314,10 @@ pmSeriesDiscoverInDom(pmDiscoverEvent *e
int i;
if (pmDebugOptions.discovery)
- fprintf(stderr, "pmSeriesDiscoverInDom: %s\n", pmInDomStr(id));
+ fprintf(stderr, "%s: %s\n", "pmSeriesDiscoverInDom", pmInDomStr(id));
+
+ if (baton == NULL || baton->slots == NULL || baton->slots->setup == 0)
+ return;
if ((domain = pmwebapi_add_domain(context, pmInDom_domain(id))) == NULL) {
infofmt(msg, "%s: failed indom discovery (domain %u)",
@@ -1303,11 +1349,10 @@ pmSeriesDiscoverText(pmDiscoverEvent *ev
pmDiscover *p = (pmDiscover *)event->data;
seriesLoadBaton *baton = p->baton;
- (void)baton;
- (void)ident;
- (void)type;
- (void)text;
- (void)arg;
+ if (pmDebugOptions.discovery)
+ fprintf(stderr, "%s: ident=%u type=%u arg=%p\n",
+ "pmSeriesDiscoverText", ident, type, arg);
- /* for Redis, help text will need special handling (RediSearch) */
+ if (baton == NULL || baton->slots == NULL || baton->slots->setup == 0)
+ return;
}
diff -Naurp pcp-5.0.2.orig/src/libpcp_web/src/query.c pcp-5.0.2/src/libpcp_web/src/query.c
--- pcp-5.0.2.orig/src/libpcp_web/src/query.c 2019-12-05 17:29:43.000000000 +1100
+++ pcp-5.0.2/src/libpcp_web/src/query.c 2020-02-03 13:23:15.265762890 +1100
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 2017-2019 Red Hat.
+ * Copyright (c) 2017-2020 Red Hat.
*
* This library is free software; you can redistribute it and/or modify it
* under the terms of the GNU Lesser General Public License as published
@@ -1243,24 +1243,43 @@ series_prepare_time_reply(
series_query_end_phase(baton);
}
+unsigned int
+series_value_count_only(timing_t *tp)
+{
+ if (tp->window.range || tp->window.delta ||
+ tp->window.start || tp->window.end)
+ return 0;
+ return tp->count;
+}
+
static void
series_prepare_time(seriesQueryBaton *baton, series_set_t *result)
{
timing_t *tp = &baton->u.query.timing;
unsigned char *series = result->series;
seriesGetSID *sid;
- char buffer[64];
+ char buffer[64], revbuf[64];
sds start, end, key, cmd;
- unsigned int i;
+ unsigned int i, revlen = 0, reverse = 0;
+
+ /* if only 'count' is requested, work back from most recent value */
+ if ((reverse = series_value_count_only(tp)) != 0) {
+ revlen = pmsprintf(revbuf, sizeof(revbuf), "%u", reverse);
+ start = sdsnew("+");
+ } else {
+ start = sdsnew(timeval_stream_str(&tp->start, buffer, sizeof(buffer)));
+ }
- start = sdsnew(timeval_stream_str(&tp->start, buffer, sizeof(buffer)));
if (pmDebugOptions.series)
fprintf(stderr, "START: %s\n", start);
- if (tp->end.tv_sec)
+ if (reverse)
+ end = sdsnew("-");
+ else if (tp->end.tv_sec)
end = sdsnew(timeval_stream_str(&tp->end, buffer, sizeof(buffer)));
else
end = sdsnew("+"); /* "+" means "no end" - to the most recent */
+
if (pmDebugOptions.series)
fprintf(stderr, "END: %s\n", end);
@@ -1277,12 +1296,21 @@ series_prepare_time(seriesQueryBaton *ba
key = sdscatfmt(sdsempty(), "pcp:values:series:%S", sid->name);
- /* XRANGE key t1 t2 */
- cmd = redis_command(4);
- cmd = redis_param_str(cmd, XRANGE, XRANGE_LEN);
+ /* X[REV]RANGE key t1 t2 [count N] */
+ if (reverse) {
+ cmd = redis_command(6);
+ cmd = redis_param_str(cmd, XREVRANGE, XREVRANGE_LEN);
+ } else {
+ cmd = redis_command(4);
+ cmd = redis_param_str(cmd, XRANGE, XRANGE_LEN);
+ }
cmd = redis_param_sds(cmd, key);
cmd = redis_param_sds(cmd, start);
cmd = redis_param_sds(cmd, end);
+ if (reverse) {
+ cmd = redis_param_str(cmd, "COUNT", sizeof("COUNT")-1);
+ cmd = redis_param_str(cmd, revbuf, revlen);
+ }
redisSlotsRequest(baton->slots, XRANGE, key, cmd,
series_prepare_time_reply, sid);
}
diff -Naurp pcp-5.0.2.orig/src/libpcp_web/src/schema.c pcp-5.0.2/src/libpcp_web/src/schema.c
--- pcp-5.0.2.orig/src/libpcp_web/src/schema.c 2019-11-18 19:35:11.000000000 +1100
+++ pcp-5.0.2/src/libpcp_web/src/schema.c 2020-02-03 13:36:03.948721355 +1100
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 2017-2019 Red Hat.
+ * Copyright (c) 2017-2020 Red Hat.
*
* This library is free software; you can redistribute it and/or modify it
* under the terms of the GNU Lesser General Public License as published
@@ -819,7 +819,7 @@ redis_series_metric(redisSlots *slots, m
*/
/* ensure all metric name strings are mapped */
- for (i = 0; i < metric->numnames; i++) {
+ for (i = 0; metric->cached == 0 && i < metric->numnames; i++) {
assert(metric->names[i].sds != NULL);
seriesBatonReference(baton, "redis_series_metric");
redisGetMap(slots,
@@ -830,7 +830,8 @@ redis_series_metric(redisSlots *slots, m
/* ensure all metric or instance label strings are mapped */
if (metric->desc.indom == PM_INDOM_NULL || metric->u.vlist == NULL) {
- series_metric_label_mapping(metric, baton);
+ if (metric->cached == 0)
+ series_metric_label_mapping(metric, baton);
} else {
for (i = 0; i < metric->u.vlist->listcount; i++) {
value = &metric->u.vlist->value[i];
@@ -847,7 +848,8 @@ redis_series_metric(redisSlots *slots, m
series_name_mapping_callback,
baton->info, baton->userdata, baton);
- series_instance_label_mapping(metric, instance, baton);
+ if (instance->cached == 0)
+ series_instance_label_mapping(metric, instance, baton);
}
}
@@ -941,6 +943,9 @@ redis_series_metadata(context_t *context
sds cmd, key;
int i;
+ if (metric->cached)
+ goto check_instances;
+
indom = pmwebapi_indom_str(metric, ibuf, sizeof(ibuf));
pmid = pmwebapi_pmid_str(metric, pbuf, sizeof(pbuf));
sem = pmwebapi_semantics_str(metric, sbuf, sizeof(sbuf));
@@ -1000,16 +1005,24 @@ redis_series_metadata(context_t *context
cmd = redis_param_sha(cmd, metric->names[i].hash);
redisSlotsRequest(slots, SADD, key, cmd, redis_series_source_callback, arg);
+check_instances:
if (metric->desc.indom == PM_INDOM_NULL || metric->u.vlist == NULL) {
- redis_series_labelset(slots, metric, NULL, baton);
+ if (metric->cached == 0) {
+ redis_series_labelset(slots, metric, NULL, baton);
+ metric->cached = 1;
+ }
} else {
for (i = 0; i < metric->u.vlist->listcount; i++) {
value = &metric->u.vlist->value[i];
if ((instance = dictFetchValue(metric->indom->insts, &value->inst)) == NULL)
continue;
- redis_series_instance(slots, metric, instance, baton);
- redis_series_labelset(slots, metric, instance, baton);
+ if (instance->cached == 0 || metric->cached == 0) {
+ redis_series_instance(slots, metric, instance, baton);
+ redis_series_labelset(slots, metric, instance, baton);
+ }
+ instance->cached = 1;
}
+ metric->cached = 1;
}
}
@@ -1210,7 +1223,6 @@ redis_series_stream(redisSlots *slots, s
redisSlotsRequest(slots, XADD, key, cmd, redis_series_stream_callback, baton);
-
key = sdscatfmt(sdsempty(), "pcp:values:series:%s", hash);
cmd = redis_command(3); /* EXPIRE key timer */
cmd = redis_param_str(cmd, EXPIRE, EXPIRE_LEN);
@@ -1228,9 +1240,6 @@ redis_series_streamed(sds stamp, metric_
char hashbuf[42];
int i;
- if (metric->updated == 0)
- return;
-
for (i = 0; i < metric->numnames; i++) {
pmwebapi_hash_str(metric->names[i].hash, hashbuf, sizeof(hashbuf));
redis_series_stream(slots, stamp, metric, hashbuf, arg);
@@ -1545,7 +1554,10 @@ redis_load_slots_callback(
redisSlots *slots = baton->slots;
seriesBatonCheckMagic(baton, MAGIC_SLOTS, "redis_load_slots_callback");
+
+ slots->setup = 1; /* we've received initial response from Redis */
slots->refresh = 0; /* we're processing CLUSTER SLOTS command now */
+
/* no cluster redirection checking is needed for this callback */
sdsfree(cmd);
@@ -1832,12 +1844,47 @@ pmDiscoverSetup(pmDiscoverModule *module
const char fallback[] = "/var/log/pcp";
const char *paths[] = { "pmlogger", "pmmgr" };
const char *logdir = pmGetOptionalConfig("PCP_LOG_DIR");
+ struct dict *config;
+ unsigned int domain, serial;
+ pmInDom indom;
char path[MAXPATHLEN];
char sep = pmPathSeparator();
- int i, sts, count = 0;
+ sds option, *ids;
+ int i, sts, nids, count = 0;
if (data == NULL)
return -ENOMEM;
+ config = data->config;
+
+ /* double-check that we are supposed to be in here */
+ if ((option = pmIniFileLookup(config, "discover", "enabled"))) {
+ if (strcasecmp(option, "false") == 0)
+ return 0;
+ }
+
+ /* prepare for optional metric and indom exclusion */
+ if ((option = pmIniFileLookup(config, "discover", "exclude.metrics"))) {
+ if ((data->pmids = dictCreate(&intKeyDictCallBacks, NULL)) == NULL)
+ return -ENOMEM;
+ /* parse regular expression string for matching on metric names */
+ regcomp(&data->exclude_names, option, REG_EXTENDED|REG_NOSUB);
+ }
+ if ((option = pmIniFileLookup(config, "discover", "exclude.indoms"))) {
+ if ((data->indoms = dictCreate(&intKeyDictCallBacks, NULL)) == NULL)
+ return -ENOMEM;
+ /* parse comma-separated indoms in 'option', convert to pmInDom */
+ if ((ids = sdssplitlen(option, sdslen(option), ",", 1, &nids))) {
+ data->exclude_indoms = nids;
+ for (i = 0; i < nids; i++) {
+ if (sscanf(ids[i], "%u.%u", &domain, &serial) == 2) {
+ indom = pmInDom_build(domain, serial);
+ dictAdd(data->indoms, &indom, NULL);
+ }
+ sdsfree(ids[i]);
+ }
+ free(ids);
+ }
+ }
/* create global EVAL hashes and string map caches */
redisGlobalsInit(data->config);
diff -Naurp pcp-5.0.2.orig/src/libpcp_web/src/schema.h pcp-5.0.2/src/libpcp_web/src/schema.h
--- pcp-5.0.2.orig/src/libpcp_web/src/schema.h 2019-10-11 17:16:29.000000000 +1100
+++ pcp-5.0.2/src/libpcp_web/src/schema.h 2020-02-03 13:23:15.266762879 +1100
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 2017-2018 Red Hat.
+ * Copyright (c) 2017-2020 Red Hat.
*
* This library is free software; you can redistribute it and/or modify it
* under the terms of the GNU Lesser General Public License as published
@@ -51,6 +51,10 @@
#define HSET_LEN (sizeof(HSET)-1)
#define HVALS "HVALS"
#define HVALS_LEN (sizeof(HVALS)-1)
+#define INFO "INFO"
+#define INFO_LEN (sizeof(INFO)-1)
+#define PING "PING"
+#define PING_LEN (sizeof(PING)-1)
#define PUBLISH "PUBLISH"
#define PUBLISH_LEN (sizeof(PUBLISH)-1)
#define SADD "SADD"
@@ -63,6 +67,8 @@
#define XADD_LEN (sizeof(XADD)-1)
#define XRANGE "XRANGE"
#define XRANGE_LEN (sizeof(XRANGE)-1)
+#define XREVRANGE "XREVRANGE"
+#define XREVRANGE_LEN (sizeof(XREVRANGE)-1)
/* create a Redis protocol command (e.g. XADD, SMEMBER) */
static inline sds
diff -Naurp pcp-5.0.2.orig/src/libpcp_web/src/slots.c pcp-5.0.2/src/libpcp_web/src/slots.c
--- pcp-5.0.2.orig/src/libpcp_web/src/slots.c 2019-10-11 17:16:29.000000000 +1100
+++ pcp-5.0.2/src/libpcp_web/src/slots.c 2020-02-03 13:23:15.266762879 +1100
@@ -356,6 +356,21 @@ redisSlotsRequest(redisSlots *slots, con
if (UNLIKELY(pmDebugOptions.desperate))
fputs(cmd, stderr);
+ if (UNLIKELY(!key && !slots->setup)) {
+ /*
+ * First request must be CLUSTER, PING, or similar - must
+ * not allow regular requests until these have completed.
+ * This is because the low layers accumulate async requests
+ * until connection establishment, which might not happen.
+ * Over time this becomes a memory leak - if we do not ever
+ * establish an initial connection).
+ */
+ if (strcmp(topic, CLUSTER) != 0 &&
+ strcmp(topic, PING) != 0 && strcmp(topic, INFO) != 0) {
+ sdsfree(cmd);
+ return -ENOTCONN;
+ }
+ }
sts = redisAsyncFormattedCommand(context, callback, cmd, arg);
if (key)
diff -Naurp pcp-5.0.2.orig/src/libpcp_web/src/slots.h pcp-5.0.2/src/libpcp_web/src/slots.h
--- pcp-5.0.2.orig/src/libpcp_web/src/slots.h 2019-04-08 09:11:00.000000000 +1000
+++ pcp-5.0.2/src/libpcp_web/src/slots.h 2020-02-03 13:23:15.266762879 +1100
@@ -44,10 +44,11 @@ typedef struct redisSlotRange {
typedef struct redisSlots {
unsigned int counter;
unsigned int nslots;
+ unsigned int setup; /* slots info all successfully setup */
+ unsigned int refresh; /* do slot refresh whenever possible */
redisSlotRange *slots; /* all instances; e.g. CLUSTER SLOTS */
redisMap *keymap; /* map command names to key position */
dict *contexts; /* async contexts access by hostspec */
- unsigned int refresh; /* do slot refresh whenever possible */
void *events;
} redisSlots;
diff -Naurp pcp-5.0.2.orig/src/libpcp_web/src/util.c pcp-5.0.2/src/libpcp_web/src/util.c
--- pcp-5.0.2.orig/src/libpcp_web/src/util.c 2019-12-10 17:39:49.000000000 +1100
+++ pcp-5.0.2/src/libpcp_web/src/util.c 2020-02-03 13:23:15.266762879 +1100
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 2017-2019 Red Hat.
+ * Copyright (c) 2017-2020 Red Hat.
*
* This library is free software; you can redistribute it and/or modify it
* under the terms of the GNU Lesser General Public License as published
@@ -535,6 +535,8 @@ pmwebapi_metric_hash(metric_t *metric)
sdsclear(identifier);
}
sdsfree(identifier);
+
+ metric->cached = 0;
}
void
@@ -574,6 +576,8 @@ pmwebapi_instance_hash(indom_t *ip, inst
SHA1Update(&shactx, (unsigned char *)identifier, sdslen(identifier));
SHA1Final(instance->name.hash, &shactx);
sdsfree(identifier);
+
+ instance->cached = 0;
}
sds
@@ -1046,7 +1050,6 @@ pmwebapi_add_instance(struct indom *indo
instance->name.sds = sdscatlen(instance->name.sds, name, length);
pmwebapi_string_hash(instance->name.id, name, length);
pmwebapi_instance_hash(indom, instance);
- instance->cached = 0;
}
return instance;
}
@@ -1202,12 +1205,14 @@ struct metric *
pmwebapi_add_metric(context_t *cp, const sds base, pmDesc *desc, int numnames, char **names)
{
struct metric *metric;
- sds name = sdsempty();
+ sds name;
int i;
/* search for a match on any of the given names */
if (base && (metric = dictFetchValue(cp->metrics, base)) != NULL)
return metric;
+
+ name = sdsempty();
for (i = 0; i < numnames; i++) {
sdsclear(name);
name = sdscat(name, names[i]);
@@ -1217,6 +1222,7 @@ pmwebapi_add_metric(context_t *cp, const
}
}
sdsfree(name);
+
return pmwebapi_new_metric(cp, base, desc, numnames, names);
}
@@ -1230,21 +1236,24 @@ pmwebapi_new_pmid(context_t *cp, const s
int sts, numnames;
if ((sts = pmUseContext(cp->context)) < 0) {
- fprintf(stderr, "failed to use context for PMID %s: %s",
+ fprintf(stderr, "%s: failed to use context for PMID %s: %s\n",
+ "pmwebapi_new_pmid",
pmIDStr_r(pmid, buffer, sizeof(buffer)),
pmErrStr_r(sts, errmsg, sizeof(errmsg)));
} else if ((sts = pmLookupDesc(pmid, &desc)) < 0) {
if (sts == PM_ERR_IPC)
cp->setup = 0;
if (pmDebugOptions.series)
- fprintf(stderr, "failed to lookup metric %s descriptor: %s",
+ fprintf(stderr, "%s: failed to lookup metric %s descriptor: %s\n",
+ "pmwebapi_new_pmid",
pmIDStr_r(pmid, buffer, sizeof(buffer)),
pmErrStr_r(sts, errmsg, sizeof(errmsg)));
} else if ((numnames = sts = pmNameAll(pmid, &names)) < 0) {
if (sts == PM_ERR_IPC)
cp->setup = 0;
if (pmDebugOptions.series)
- fprintf(stderr, "failed to lookup metric %s names: %s",
+ fprintf(stderr, "%s: failed to lookup metric %s names: %s\n",
+ "pmwebapi_new_pmid",
pmIDStr_r(pmid, buffer, sizeof(buffer)),
pmErrStr_r(sts, errmsg, sizeof(errmsg)));
} else {
diff -Naurp pcp-5.0.2.orig/src/pmproxy/pmproxy.conf pcp-5.0.2/src/pmproxy/pmproxy.conf
--- pcp-5.0.2.orig/src/pmproxy/pmproxy.conf 2019-08-09 15:50:17.000000000 +1000
+++ pcp-5.0.2/src/pmproxy/pmproxy.conf 2020-02-03 13:36:03.948721355 +1100
@@ -43,6 +43,11 @@ secure.enabled = true
# propogate archives from pmlogger(1) into Redis querying
enabled = true
+# metrics name regex to skip during discovery (eg due to high volume)
+exclude.metrics = proc.*
+
+# comma-separated list of instance domains to skip during discovery
+exclude.indoms = 3.9,79.7
#####################################################################
## settings for fast, scalable time series quering via Redis
diff -Naurp pcp-5.0.2.orig/src/pmproxy/src/redis.c pcp-5.0.2/src/pmproxy/src/redis.c
--- pcp-5.0.2.orig/src/pmproxy/src/redis.c 2019-12-02 16:39:33.000000000 +1100
+++ pcp-5.0.2/src/pmproxy/src/redis.c 2020-02-03 13:36:13.585620539 +1100
@@ -145,11 +145,11 @@ setup_redis_module(struct proxy *proxy)
proxy->slots = redisSlotsConnect(proxy->config,
flags, proxylog, on_redis_connected,
proxy, proxy->events, proxy);
- if (archive_discovery)
+ if (archive_discovery && series_queries)
pmDiscoverSetSlots(&redis_discover.module, proxy->slots);
}
- if (archive_discovery) {
+ if (archive_discovery && series_queries) {
pmDiscoverSetEventLoop(&redis_discover.module, proxy->events);
pmDiscoverSetConfiguration(&redis_discover.module, proxy->config);
pmDiscoverSetMetricRegistry(&redis_discover.module, metric_registry);
Reference in New Issue View Git Blame Copy Permalink