diff -Naurp pcp-5.0.2.orig/qa/1211.out pcp-5.0.2/qa/1211.out --- pcp-5.0.2.orig/qa/1211.out 2019-12-06 15:18:26.000000000 +1100 +++ pcp-5.0.2/qa/1211.out 2020-02-03 13:23:15.258762963 +1100 @@ -144,6 +144,9 @@ kernel.uname.nodename kernel.uname.release kernel.uname.sysname kernel.uname.version +pmcd.pmlogger.archive +pmcd.pmlogger.host +pmcd.pmlogger.port proc.fd.count proc.id.egid proc.id.egid_nm @@ -267,6 +270,7 @@ List all instance names ... 030016 pmlogger -P -c config.default 20110930.17.20 1 minute 15 minute +2950 5 minute cpu0 cpu1 @@ -398,10 +402,10 @@ fecd5a4b4c6e1273eaa001287a6dd57b7bbd19f7 Values fetch for a single-valued query ... d51624d12da45900bfee2fd73f1e23f3ccabb784 - [Mon Oct 3 09:10:22.959242000 2011] 172598244 - [Mon Oct 3 09:10:23.300460000 2011] 172598364 - [Mon Oct 3 09:10:23.802930000 2011] 172598481 [Mon Oct 3 09:10:24.305845000 2011] 172598559 + [Mon Oct 3 09:10:23.802930000 2011] 172598481 + [Mon Oct 3 09:10:23.300460000 2011] 172598364 + [Mon Oct 3 09:10:22.959242000 2011] 172598244 Values fetch with a one-second interval ... @@ -420,15 +424,18 @@ d51624d12da45900bfee2fd73f1e23f3ccabb784 Values fetch for a multi-valued query ... fecd5a4b4c6e1273eaa001287a6dd57b7bbd19f7 - [Mon Oct 3 09:10:23.300460000 2011] 0.000000e+00 59181b1de54ff2b383cfd1cdd8636f86c880b69b - [Mon Oct 3 09:10:23.300460000 2011] 2.000000e-02 ab010c7d45145aa33c8f8fa681a68c9d4102ae19 - [Mon Oct 3 09:10:23.300460000 2011] 5.000000e-02 9d418095c9f971ff4fd44d6828ead27f9d021dc3 - [Mon Oct 3 09:10:23.802930000 2011] 0.000000e+00 59181b1de54ff2b383cfd1cdd8636f86c880b69b - [Mon Oct 3 09:10:23.802930000 2011] 2.000000e-02 ab010c7d45145aa33c8f8fa681a68c9d4102ae19 - [Mon Oct 3 09:10:23.802930000 2011] 5.000000e-02 9d418095c9f971ff4fd44d6828ead27f9d021dc3 [Mon Oct 3 09:10:24.305845000 2011] 0.000000e+00 59181b1de54ff2b383cfd1cdd8636f86c880b69b [Mon Oct 3 09:10:24.305845000 2011] 2.000000e-02 ab010c7d45145aa33c8f8fa681a68c9d4102ae19 [Mon Oct 3 09:10:24.305845000 2011] 5.000000e-02 9d418095c9f971ff4fd44d6828ead27f9d021dc3 + [Mon Oct 3 09:10:23.802930000 2011] 0.000000e+00 59181b1de54ff2b383cfd1cdd8636f86c880b69b + [Mon Oct 3 09:10:23.802930000 2011] 2.000000e-02 ab010c7d45145aa33c8f8fa681a68c9d4102ae19 + [Mon Oct 3 09:10:23.802930000 2011] 5.000000e-02 9d418095c9f971ff4fd44d6828ead27f9d021dc3 + [Mon Oct 3 09:10:23.300460000 2011] 0.000000e+00 59181b1de54ff2b383cfd1cdd8636f86c880b69b + [Mon Oct 3 09:10:23.300460000 2011] 2.000000e-02 ab010c7d45145aa33c8f8fa681a68c9d4102ae19 + [Mon Oct 3 09:10:23.300460000 2011] 5.000000e-02 9d418095c9f971ff4fd44d6828ead27f9d021dc3 + [Mon Oct 3 09:10:22.959242000 2011] 0.000000e+00 59181b1de54ff2b383cfd1cdd8636f86c880b69b + [Mon Oct 3 09:10:22.959242000 2011] 2.000000e-02 ab010c7d45145aa33c8f8fa681a68c9d4102ae19 + [Mon Oct 3 09:10:22.959242000 2011] 5.000000e-02 9d418095c9f971ff4fd44d6828ead27f9d021dc3 Multi-series lookups from a multi-series query ... 2db1da4d276d81c42c578c2829e99188ae7cc898 diff -Naurp pcp-5.0.2.orig/qa/1573 pcp-5.0.2/qa/1573 --- pcp-5.0.2.orig/qa/1573 1970-01-01 10:00:00.000000000 +1000 +++ pcp-5.0.2/qa/1573 2020-02-03 13:36:17.288581801 +1100 @@ -0,0 +1,103 @@ +#!/bin/sh +# PCP QA Test No. 1573 +# Exercise libpcp_web memory leak without a redis-server. +# +# Copyright (c) 2020 Red Hat. +# + +seq=`basename $0` +echo "QA output created by $seq" + +# get standard environment, filters and checks +. ./common.product +. ./common.filter +. ./common.check + +_check_series + +_cleanup() +{ + cd $here + if $need_restore + then + need_restore=false + _service pmlogger stop >/dev/null + $sudo rm -rf $PCP_LOG_DIR/pmlogger + $sudo mv $PCP_LOG_DIR/pmlogger.$seq $PCP_LOG_DIR/pmlogger + _restore_config $PCP_ETC_DIR/pcp/pmlogger + _service pcp restart 2>&1 | _filter_pcp_stop | _filter_pcp_start + _wait_for_pmcd + _wait_for_pmlogger + echo === restarting pmproxy + _restore_config $PCP_SYSCONF_DIR/pmproxy + _service pmproxy restart 2>&1 | _filter_pcp_start + _wait_for_pmproxy + fi + $sudo rm -rf $tmp $tmp.* +} + +status=1 # failure is the default! +need_restore=false +$sudo rm -rf $tmp $tmp.* $seq.full +trap "_cleanup; exit \$status" 0 1 2 3 15 + +# real QA test starts here +_save_config $PCP_SYSCONF_DIR/pmproxy +need_restore=true + +# only want the primary logger running +_save_config $PCP_ETC_DIR/pcp/pmlogger +_restore_pmlogger_control + +#$sudo rm -f $PCP_SYSCONF_DIR/pmproxy/* +echo "[pmproxy]" > $tmp.conf +echo "pcp.enabled = true" >> $tmp.conf +echo "http.enabled = true" >> $tmp.conf +echo "redis.enabled = true" >> $tmp.conf +echo "[discover]" >> $tmp.conf +echo "enabled = true" >> $tmp.conf +echo "[pmseries]" >> $tmp.conf +echo "enabled = false" >> $tmp.conf +$sudo cp $tmp.conf $PCP_SYSCONF_DIR/pmproxy/pmproxy.conf + +_service pmlogger stop >/dev/null + +# move aside existing logs so we can measure base memory footprint +[ -d $PCP_LOG_DIR/pmlogger.$seq ] && $sudo mv $PCP_LOG_DIR/pmlogger.$seq $PCP_LOG_DIR/pmlogger.$seq.saved +$sudo mv $PCP_LOG_DIR/pmlogger $PCP_LOG_DIR/pmlogger.$seq +$sudo mkdir -p $PCP_LOG_DIR/pmlogger +$sudo chmod 775 $PCP_LOG_DIR/pmlogger +$sudo chown $PCP_USER:$PCP_USER $PCP_LOG_DIR/pmlogger + +_service pmproxy restart 2>&1 | _filter_pcp_stop | _filter_pcp_start +_wait_for_pmproxy + +pmproxy_pid=`_get_pids_by_name -a pmproxy` +[ -z "$pmproxy_pid" ] && echo === pmproxy not running && status=1 && exit 1 +echo === extract initial rss +pmproxy_rss1=`pminfo -f proc.memory.rss | + $PCP_AWK_PROG '{ if ($2 == "['$pmproxy_pid'") { print $NF} }'` + +echo === restarting pmlogger # primary only +_service pmlogger restart 2>&1 | _filter_pcp_start +_wait_for_pmlogger + +echo === wait for pmproxy to process filesystem events +pmsleep 4.2 + +echo === extract updated rss +pmproxy_rss2=`pminfo -f proc.memory.rss | + $PCP_AWK_PROG '{ if ($2 == "['$pmproxy_pid'") { print $NF} }'` + +echo === checking rss within tolerance +_within_tolerance "rss" $pmproxy_rss1 $pmproxy_rss2 10% +[ $pmproxy_rss2 -gt 10000 ] && echo "Unexpected pmproxy RSS: $pmproxy_rss2, was initially $pmproxy_rss1" + +echo "RSS1 for PID $pmproxy_pid is $pmproxy_rss1" >> $here/$seq.full +echo "RSS2 for PID $pmproxy_pid is $pmproxy_rss2" >> $here/$seq.full +cat $PCP_LOG_DIR/pmproxy/pmproxy.log >>$seq.full +echo === see $seq.full for pmproxy rss and logs + +# success, all done +status=0 +exit diff -Naurp pcp-5.0.2.orig/qa/1573.out pcp-5.0.2/qa/1573.out --- pcp-5.0.2.orig/qa/1573.out 1970-01-01 10:00:00.000000000 +1000 +++ pcp-5.0.2/qa/1573.out 2020-02-03 13:23:15.259762953 +1100 @@ -0,0 +1,8 @@ +QA output created by 1573 +=== extract initial rss +=== restarting pmlogger +=== wait for pmproxy to process filesystem events +=== extract updated rss +=== checking rss within tolerance +=== see 1573.full for pmproxy rss and logs +=== restarting pmproxy diff -Naurp pcp-5.0.2.orig/qa/1600 pcp-5.0.2/qa/1600 --- pcp-5.0.2.orig/qa/1600 2019-12-10 17:49:05.000000000 +1100 +++ pcp-5.0.2/qa/1600 2020-02-03 13:23:15.260762942 +1100 @@ -82,7 +82,11 @@ _filter_values() _filter_label_values() { sed \ + -e "s/^domainname: \"${domainname}\"/domainname: \"DOMAIN\"/g" \ + -e "s/^machineid: \"${machineid}\"/machineid: \"MACHINE\"/g" \ -e "s/^hostname: \"${hostname}\"/hostname: \"HOSTNAME\"/g" \ + -e "s/^groupid: $groupid/groupid: GID/g" \ + -e "s/^userid: $userid/userid: UID/g" \ -e "s/changed: false, true/changed: false/g" \ -e "/metric_label: null/d" \ #end diff -Naurp pcp-5.0.2.orig/qa/1600.out pcp-5.0.2/qa/1600.out --- pcp-5.0.2.orig/qa/1600.out 2019-12-10 10:46:20.000000000 +1100 +++ pcp-5.0.2/qa/1600.out 2020-02-03 13:23:15.260762942 +1100 @@ -27,15 +27,15 @@ TIMESERIES == verify metric labels TIMESERIES - inst [100 or "bin-100"] labels {"agent":"sample","hostname":"HOST","role":"testing"} - inst [200 or "bin-200"] labels {"agent":"sample","hostname":"HOST","role":"testing"} - inst [300 or "bin-300"] labels {"agent":"sample","hostname":"HOST","role":"testing"} - inst [400 or "bin-400"] labels {"agent":"sample","hostname":"HOST","role":"testing"} - inst [500 or "bin-500"] labels {"agent":"sample","hostname":"HOST","role":"testing"} - inst [600 or "bin-600"] labels {"agent":"sample","hostname":"HOST","role":"testing"} - inst [700 or "bin-700"] labels {"agent":"sample","hostname":"HOST","role":"testing"} - inst [800 or "bin-800"] labels {"agent":"sample","hostname":"HOST","role":"testing"} - inst [900 or "bin-900"] labels {"agent":"sample","hostname":"HOST","role":"testing"} + inst [100 or "bin-100"] labels {"agent":"sample","bin":100,"domainname":"DOMAIN","groupid":GID,"hostname":"HOST","latitude":-25.28496,"longitude":152.87886,"machineid":"MACHINE","role":"testing","userid":UID} + inst [200 or "bin-200"] labels {"agent":"sample","bin":200,"domainname":"DOMAIN","groupid":GID,"hostname":"HOST","latitude":-25.28496,"longitude":152.87886,"machineid":"MACHINE","role":"testing","userid":UID} + inst [300 or "bin-300"] labels {"agent":"sample","bin":300,"domainname":"DOMAIN","groupid":GID,"hostname":"HOST","latitude":-25.28496,"longitude":152.87886,"machineid":"MACHINE","role":"testing","userid":UID} + inst [400 or "bin-400"] labels {"agent":"sample","bin":400,"domainname":"DOMAIN","groupid":GID,"hostname":"HOST","latitude":-25.28496,"longitude":152.87886,"machineid":"MACHINE","role":"testing","userid":UID} + inst [500 or "bin-500"] labels {"agent":"sample","bin":500,"domainname":"DOMAIN","groupid":GID,"hostname":"HOST","latitude":-25.28496,"longitude":152.87886,"machineid":"MACHINE","role":"testing","userid":UID} + inst [600 or "bin-600"] labels {"agent":"sample","bin":600,"domainname":"DOMAIN","groupid":GID,"hostname":"HOST","latitude":-25.28496,"longitude":152.87886,"machineid":"MACHINE","role":"testing","userid":UID} + inst [700 or "bin-700"] labels {"agent":"sample","bin":700,"domainname":"DOMAIN","groupid":GID,"hostname":"HOST","latitude":-25.28496,"longitude":152.87886,"machineid":"MACHINE","role":"testing","userid":UID} + inst [800 or "bin-800"] labels {"agent":"sample","bin":800,"domainname":"DOMAIN","groupid":GID,"hostname":"HOST","latitude":-25.28496,"longitude":152.87886,"machineid":"MACHINE","role":"testing","userid":UID} + inst [900 or "bin-900"] labels {"agent":"sample","bin":900,"domainname":"DOMAIN","groupid":GID,"hostname":"HOST","latitude":-25.28496,"longitude":152.87886,"machineid":"MACHINE","role":"testing","userid":UID} == verify metric values TIMESERIES @@ -43,15 +43,24 @@ TIMESERIES [TIMESTAMP] VALUE == verify label names and values agent: "mmv", "sample", "pmcd" +bin: 100, 200, 300, 400, 500, 600, 700, 800, 900 changed: false clan: "mcdonell" cluster: "zero" +domainname: "DOMAIN" +groupid: GID hostname: "HOSTNAME" indom_label: 42.001 +latitude: -25.28496 +longitude: 152.87886 +machineid: "MACHINE" measure: "speed" model: "RGB" +registry_label: "string" role: "testing" +transient: false, true units: "metres per second" unitsystem: "SI" +userid: UID == verify archive removal == all done diff -Naurp pcp-5.0.2.orig/qa/1601.out pcp-5.0.2/qa/1601.out --- pcp-5.0.2.orig/qa/1601.out 2019-11-27 16:01:34.000000000 +1100 +++ pcp-5.0.2/qa/1601.out 2020-02-03 13:23:15.261762932 +1100 @@ -131,7 +131,7 @@ Using series 01d8bc7fa75aaff98a08aa0b1c0 { "series": "605fc77742cd0317597291329561ac4e50c0dd12", "instance": "c3795d8b757506a2901c6b08b489ba56cae7f0d4", - "timestamp": 1317633023300.460, + "timestamp": 1317633024305.845, "value": "71661" }, { @@ -147,7 +147,7 @@ Using series 01d8bc7fa75aaff98a08aa0b1c0 { "series": "605fc77742cd0317597291329561ac4e50c0dd12", "instance": "c3795d8b757506a2901c6b08b489ba56cae7f0d4", - "timestamp": 1317633023300.460, + "timestamp": 1317633024305.845, "value": "71661" }, { @@ -163,7 +163,7 @@ Using series 01d8bc7fa75aaff98a08aa0b1c0 { "series": "605fc77742cd0317597291329561ac4e50c0dd12", "instance": "c3795d8b757506a2901c6b08b489ba56cae7f0d4", - "timestamp": 1317633023300.460, + "timestamp": 1317633024305.845, "value": "71661" }, { @@ -179,7 +179,7 @@ Using series 01d8bc7fa75aaff98a08aa0b1c0 { "series": "605fc77742cd0317597291329561ac4e50c0dd12", "instance": "c3795d8b757506a2901c6b08b489ba56cae7f0d4", - "timestamp": 1317633023300.460, + "timestamp": 1317633024305.845, "value": "71661" }, { diff -Naurp pcp-5.0.2.orig/qa/1661 pcp-5.0.2/qa/1661 --- pcp-5.0.2.orig/qa/1661 2019-12-10 17:04:20.000000000 +1100 +++ pcp-5.0.2/qa/1661 2020-02-03 13:23:15.261762932 +1100 @@ -41,8 +41,7 @@ _restore_pmlogger_control echo;echo === restarting pmproxy service to ensure sane starting condition _service pmlogger stop 2>&1 | _filter_pcp_stop _service pmproxy restart 2>&1 | _filter_pcp_stop | _filter_pcp_start -# give pmproxy a chance to startup -pmsleep 2; _wait_for_pmproxy +_wait_for_pmproxy pmproxy_pid=`_get_pids_by_name -a pmproxy` [ -z "$pmproxy_pid" ] && echo === pmproxy not running && status=1 && exit 1 diff -Naurp pcp-5.0.2.orig/qa/group pcp-5.0.2/qa/group --- pcp-5.0.2.orig/qa/group 2019-12-11 14:06:06.000000000 +1100 +++ pcp-5.0.2/qa/group 2020-02-03 13:23:15.261762932 +1100 @@ -1688,6 +1688,7 @@ BAD 1545 pcp2xml python pcp2xxx local 1546 pmrep python local 1547 pmrep python local +1573 pmproxy libpcp_web pmlogger local 1588 python pmiostat local 1598 pmda.statsd local 1599 pmda.statsd local diff -Naurp pcp-5.0.2.orig/src/include/pcp/libpcp.h pcp-5.0.2/src/include/pcp/libpcp.h --- pcp-5.0.2.orig/src/include/pcp/libpcp.h 2019-09-24 17:23:36.000000000 +1000 +++ pcp-5.0.2/src/include/pcp/libpcp.h 2020-02-03 13:23:15.261762932 +1100 @@ -7,7 +7,7 @@ * remain fixed across releases, and they may not work, or may * provide different semantics at some point in the future. * - * Copyright (c) 2012-2019 Red Hat. + * Copyright (c) 2012-2020 Red Hat. * Copyright (c) 2008-2009 Aconex. All Rights Reserved. * Copyright (c) 1995-2002 Silicon Graphics, Inc. All Rights Reserved. * @@ -846,6 +846,13 @@ PCP_CALL extern int __pmLogPutText(__pmA PCP_CALL extern int __pmLogWriteLabel(__pmFILE *, const __pmLogLabel *); PCP_CALL extern int __pmLogLoadLabel(__pmArchCtl *, const char *); PCP_CALL extern int __pmLogLoadMeta(__pmArchCtl *); +PCP_CALL extern int __pmLogAddDesc(__pmArchCtl *, const pmDesc *); +PCP_CALL extern int __pmLogAddInDom(__pmArchCtl *, const pmTimespec *, const pmInResult *, int *, int); +PCP_CALL extern int __pmLogAddPMNSNode(__pmArchCtl *, pmID, const char *); +PCP_CALL extern int __pmLogAddLabelSets(__pmArchCtl *, const pmTimespec *, unsigned int, unsigned int, int, pmLabelSet *); +PCP_CALL extern int __pmLogAddText(__pmArchCtl *, unsigned int, unsigned int, const char *); +PCP_CALL extern int __pmLogAddVolume(__pmArchCtl *, unsigned int); + #define PMLOGREAD_NEXT 0 #define PMLOGREAD_TO_EOF 1 PCP_CALL extern int __pmLogRead(__pmArchCtl *, int, __pmFILE *, pmResult **, int); @@ -862,7 +869,9 @@ PCP_CALL extern int __pmLogLookupText(__ PCP_CALL extern int __pmLogNameInDom(__pmArchCtl *, pmInDom, pmTimeval *, int, char **); PCP_CALL extern const char *__pmLogLocalSocketDefault(int, char *buf, size_t bufSize); PCP_CALL extern const char *__pmLogLocalSocketUser(int, char *buf, size_t bufSize); +PCP_CALL extern int __pmLogCompressedSuffix(const char *); PCP_CALL extern char *__pmLogBaseName(char *); +PCP_CALL extern char *__pmLogBaseNameVol(char *, int *); PCP_DATA extern int __pmLogReads; /* Convert opaque context handle to __pmContext pointer */ diff -Naurp pcp-5.0.2.orig/src/libpcp/src/exports.master pcp-5.0.2/src/libpcp/src/exports.master --- pcp-5.0.2.orig/src/libpcp/src/exports.master 2019-10-02 14:40:30.000000000 +1000 +++ pcp-5.0.2/src/libpcp/src/exports.master 2020-02-03 13:23:15.262762921 +1100 @@ -683,3 +683,15 @@ PCP_3.26 { global: __pmDupLabelSets; } PCP_3.25; + +PCP_3.26_1 { + global: + __pmLogAddDesc; + __pmLogAddInDom; + __pmLogAddPMNSNode; + __pmLogAddLabelSets; + __pmLogAddText; + __pmLogAddVolume; + __pmLogCompressedSuffix; + __pmLogBaseNameVol; +} PCP_3.26; diff -Naurp pcp-5.0.2.orig/src/libpcp/src/io.c pcp-5.0.2/src/libpcp/src/io.c --- pcp-5.0.2.orig/src/libpcp/src/io.c 2018-06-09 11:43:34.000000000 +1000 +++ pcp-5.0.2/src/libpcp/src/io.c 2020-02-03 13:23:15.262762921 +1100 @@ -1,5 +1,5 @@ /* - * Copyright (c) 2017-2018 Red Hat. + * Copyright (c) 2017-2018,2020 Red Hat. * * This library is free software; you can redistribute it and/or modify it * under the terms of the GNU Lesser General Public License as published @@ -47,7 +47,7 @@ extern __pm_fops __pm_xz; #endif static const struct { - const char *suff; + const char *suffix; const int appl; __pm_fops *handler; } compress_ctl[] = { @@ -61,40 +61,43 @@ static const struct { }; static const int ncompress = sizeof(compress_ctl) / sizeof(compress_ctl[0]); +int +__pmLogCompressedSuffix(const char *suffix) +{ + int i; + + for (i = 0; i < ncompress; i++) + if (strcmp(suffix, compress_ctl[i].suffix) == 0) + return 1; + return 0; +} + /* - * If name contains '.' and the suffix is "index", "meta" or a string of - * digits, all optionally followed by one of the compression suffixes, - * strip the suffix. - * - * Modifications are performed on the argument string in-place. If modifications - * are made, a pointer to the start of the modified string is returned. - * Otherwise, NULL is returned. + * Variant of __pmLogBaseName() - see below that also returns log + * the volume number if the file name is an archive log volume. + * If the vol argument is NULL it will be ignored. */ char * -__pmLogBaseName(char *name) +__pmLogBaseNameVol(char *name, int *vol) { - char *q; - int strip; - int i; + char *q, *q2; + int strip = 0; - strip = 0; + if (vol) + *vol = -1; if ((q = strrchr(name, '.')) != NULL) { - for (i = 0; i < ncompress; i++) { - if (strcmp(q, compress_ctl[i].suff) == 0) { - char *q2; - /* - * The name ends with one of the supported compressed file - * suffixes. Strip it before checking for other known suffixes. - */ - *q = '\0'; - if ((q2 = strrchr(name, '.')) == NULL) { - /* no . to the left of the suffix */ - *q = '.'; - goto done; - } - q = q2; - break; + if (__pmLogCompressedSuffix(q)) { + /* + * The name ends with one of the supported compressed file + * suffixes. Strip it before checking for other known suffixes. + */ + *q = '\0'; + if ((q2 = strrchr(name, '.')) == NULL) { + /* no . to the left of the suffix */ + *q = '.'; + goto done; } + q = q2; } if (strcmp(q, ".index") == 0) { strip = 1; @@ -109,16 +112,10 @@ __pmLogBaseName(char *name) */ if (q[1] != '\0') { char *end; - /* - * Below we don't care about the value from strtol(), - * we're interested in updating the pointer "end". - * The messiness is thanks to gcc and glibc ... strtol() - * is marked __attribute__((warn_unused_result)) ... - * to avoid warnings on all platforms, assign to a - * dummy variable that is explicitly marked unused. - */ - long tmpl __attribute__((unused)); + long tmpl; tmpl = strtol(q+1, &end, 10); + if (vol) + *vol = tmpl; if (*end == '\0') strip = 1; } } @@ -131,6 +128,21 @@ done: return NULL; /* not the name of an archive file. */ } +/* + * If name contains '.' and the suffix is "index", "meta" or a string of + * digits, all optionally followed by one of the compression suffixes, + * strip the suffix. + * + * Modifications are performed on the argument string in-place. If modifications + * are made, a pointer to the start of the modified string is returned. + * Otherwise, NULL is returned. + */ +char * +__pmLogBaseName(char *name) +{ + return __pmLogBaseNameVol(name, NULL); +} + static int popen_uncompress(const char *cmd, const char *arg, const char *fname, int fd) { @@ -319,7 +331,7 @@ __pmCompressedFileIndex(char *fname, siz char tmpname[MAXPATHLEN]; for (i = 0; i < ncompress; i++) { - suffix = compress_ctl[i].suff; + suffix = compress_ctl[i].suffix; pmsprintf(tmpname, sizeof(tmpname), "%s%s", fname, suffix); sts = access(tmpname, R_OK); if (sts == 0 || (errno != ENOENT && errno != ENOTDIR)) { @@ -358,7 +370,7 @@ index_compress(char *fname, size_t flen) suffix = strrchr(fname, '.'); if (suffix != NULL) { for (i = 0; i < ncompress; i++) { - if (strcmp(suffix, compress_ctl[i].suff) == 0) + if (strcmp(suffix, compress_ctl[i].suffix) == 0) return i; } } @@ -731,7 +743,7 @@ compress_suffix_list(void) const char *q; for (i = 0; i < ncompress; i++) { - q = compress_ctl[i].suff; + q = compress_ctl[i].suffix; if (i > 0) *p++ = ' '; while (*q) { diff -Naurp pcp-5.0.2.orig/src/libpcp/src/logmeta.c pcp-5.0.2/src/libpcp/src/logmeta.c --- pcp-5.0.2.orig/src/libpcp/src/logmeta.c 2018-09-14 10:22:56.000000000 +1000 +++ pcp-5.0.2/src/libpcp/src/logmeta.c 2020-02-03 13:23:15.262762921 +1100 @@ -1,5 +1,5 @@ /* - * Copyright (c) 2013-2018 Red Hat. + * Copyright (c) 2013-2018, 2020 Red Hat. * Copyright (c) 1995-2002 Silicon Graphics, Inc. All Rights Reserved. * * This library is free software; you can redistribute it and/or modify it @@ -490,7 +490,7 @@ check_dup_labels(const __pmArchCtl *acp) } static int -addtext(__pmArchCtl *acp, unsigned int ident, unsigned int type, char *buffer) +addtext(__pmArchCtl *acp, unsigned int ident, unsigned int type, const char *buffer) { __pmLogCtl *lcp = acp->ac_log; __pmHashNode *hp; @@ -553,6 +553,92 @@ PM_FAULT_POINT("libpcp/" __FILE__ ":15", return sts; } +int +__pmLogAddDesc(__pmArchCtl *acp, const pmDesc *newdp) +{ + __pmHashNode *hp; + __pmLogCtl *lcp = acp->ac_log; + pmDesc *dp, *olddp; + + if ((hp = __pmHashSearch((int)newdp->pmid, &lcp->l_hashpmid)) != NULL) { + /* PMID is already in the hash table - check for conflicts. */ + olddp = (pmDesc *)hp->data; + if (newdp->type != olddp->type) + return PM_ERR_LOGCHANGETYPE; + if (newdp->sem != olddp->sem) + return PM_ERR_LOGCHANGESEM; + if (newdp->indom != olddp->indom) + return PM_ERR_LOGCHANGEINDOM; + if (newdp->units.dimSpace != olddp->units.dimSpace || + newdp->units.dimTime != olddp->units.dimTime || + newdp->units.dimCount != olddp->units.dimCount || + newdp->units.scaleSpace != olddp->units.scaleSpace || + newdp->units.scaleTime != olddp->units.scaleTime || + newdp->units.scaleCount != olddp->units.scaleCount) + return PM_ERR_LOGCHANGEUNITS; + + /* PMID is already known and checks out - we're done here. */ + return 0; + } + + /* Add a copy of the descriptor into the PMID:desc hash table. */ +PM_FAULT_POINT("libpcp/" __FILE__ ":2", PM_FAULT_ALLOC); + if ((dp = (pmDesc *)malloc(sizeof(pmDesc))) == NULL) + return -oserror(); + *dp = *newdp; + + return __pmHashAdd((int)dp->pmid, (void *)dp, &lcp->l_hashpmid); +} + +int +__pmLogAddPMNSNode(__pmArchCtl *acp, pmID pmid, const char *name) +{ + __pmLogCtl *lcp = acp->ac_log; + int sts; + + /* + * If we see a duplicate name with a different PMID, its a + * recoverable error. + * We wont be able to see all of the data in the log, but + * its better to provide access to some rather than none, + * esp. when only one or two metric IDs may be corrupted + * in this way (which we may not be interested in anyway). + */ + sts = __pmAddPMNSNode(lcp->l_pmns, pmid, name); + if (sts == PM_ERR_PMID) + sts = 0; + return sts; +} + +int +__pmLogAddInDom(__pmArchCtl *acp, const pmTimespec *when, const pmInResult *in, + int *tbuf, int allinbuf) +{ + pmTimeval tv; + + tv.tv_sec = when->tv_sec; + tv.tv_usec = when->tv_nsec / 1000; + return addindom(acp->ac_log, in->indom, &tv, + in->numinst, in->instlist, in->namelist, tbuf, allinbuf); +} + +int +__pmLogAddLabelSets(__pmArchCtl *acp, const pmTimespec *when, unsigned int type, + unsigned int ident, int nsets, pmLabelSet *labelsets) +{ + pmTimeval tv; + + tv.tv_sec = when->tv_sec; + tv.tv_usec = when->tv_nsec / 1000; + return addlabel(acp, type, ident, nsets, labelsets, &tv); +} + +int +__pmLogAddText(__pmArchCtl *acp, unsigned int ident, unsigned int type, const char *buffer) +{ + return addtext(acp, ident, type, buffer); +} + /* * Load _all_ of the hashed pmDesc and __pmLogInDom structures from the metadata * log file -- used at the initialization (NewContext) of an archive. @@ -563,11 +649,8 @@ int __pmLogLoadMeta(__pmArchCtl *acp) { __pmLogCtl *lcp = acp->ac_log; - __pmHashNode *hp; int rlen; int check; - pmDesc *dp; - pmDesc *olddp; int sts = 0; __pmLogHdr h; __pmFILE *f = lcp->l_mdfp; @@ -615,13 +698,10 @@ __pmLogLoadMeta(__pmArchCtl *acp) } rlen = h.len - (int)sizeof(__pmLogHdr) - (int)sizeof(int); if (h.type == TYPE_DESC) { + pmDesc desc; + numpmid++; -PM_FAULT_POINT("libpcp/" __FILE__ ":2", PM_FAULT_ALLOC); - if ((dp = (pmDesc *)malloc(sizeof(pmDesc))) == NULL) { - sts = -oserror(); - goto end; - } - if ((n = (int)__pmFread(dp, 1, sizeof(pmDesc), f)) != sizeof(pmDesc)) { + if ((n = (int)__pmFread(&desc, 1, sizeof(pmDesc), f)) != sizeof(pmDesc)) { if (pmDebugOptions.logmeta) { fprintf(stderr, "__pmLogLoadMeta: pmDesc read -> %d: expected: %d\n", n, (int)sizeof(pmDesc)); @@ -632,67 +712,25 @@ PM_FAULT_POINT("libpcp/" __FILE__ ":2", } else sts = PM_ERR_LOGREC; - free(dp); goto end; } - else { - /* swab desc */ - dp->type = ntohl(dp->type); - dp->sem = ntohl(dp->sem); - dp->indom = __ntohpmInDom(dp->indom); - dp->units = __ntohpmUnits(dp->units); - dp->pmid = __ntohpmID(dp->pmid); - } - /* Add it to the hash pmid hash table. */ - if ((hp = __pmHashSearch((int)dp->pmid, &lcp->l_hashpmid)) != NULL) { - /* - * This pmid is already in the hash table. Check for conflicts. - */ - olddp = (pmDesc *)hp->data; - if (dp->type != olddp->type) { - sts = PM_ERR_LOGCHANGETYPE; - free(dp); - goto end; - } - if (dp->sem != olddp->sem) { - sts = PM_ERR_LOGCHANGESEM; - free(dp); - goto end; - } - if (dp->indom != olddp->indom) { - sts = PM_ERR_LOGCHANGEINDOM; - free(dp); - goto end; - } - if (dp->units.dimSpace != olddp->units.dimSpace || - dp->units.dimTime != olddp->units.dimTime || - dp->units.dimCount != olddp->units.dimCount || - dp->units.scaleSpace != olddp->units.scaleSpace || - dp->units.scaleTime != olddp->units.scaleTime || - dp->units.scaleCount != olddp->units.scaleCount) { - sts = PM_ERR_LOGCHANGEUNITS; - free(dp); - goto end; - } - /* - * This pmid is already known, and matches. We can free the newly - * read copy and use the one in the hash table. - */ - free(dp); - dp = olddp; - } - else if ((sts = __pmHashAdd((int)dp->pmid, (void *)dp, &lcp->l_hashpmid)) < 0) { - free(dp); + /* swab desc */ + desc.type = ntohl(desc.type); + desc.sem = ntohl(desc.sem); + desc.indom = __ntohpmInDom(desc.indom); + desc.units = __ntohpmUnits(desc.units); + desc.pmid = __ntohpmID(desc.pmid); + + if ((sts = __pmLogAddDesc(acp, &desc)) < 0) goto end; - } /* read in the names & store in PMNS tree ... */ if ((n = (int)__pmFread(&numnames, 1, sizeof(numnames), f)) != sizeof(numnames)) { if (pmDebugOptions.logmeta) { - fprintf(stderr, "__pmLogLoadMeta: numnames read -> %d: expected: %d\n", - n, (int)sizeof(numnames)); + fprintf(stderr, "%s: numnames read -> %d: expected: %d\n", + "__pmLogLoadMeta", n, (int)sizeof(numnames)); } if (__pmFerror(f)) { __pmClearerr(f); @@ -711,8 +749,8 @@ PM_FAULT_POINT("libpcp/" __FILE__ ":2", if ((n = (int)__pmFread(&len, 1, sizeof(len), f)) != sizeof(len)) { if (pmDebugOptions.logmeta) { - fprintf(stderr, "__pmLogLoadMeta: len name[%d] read -> %d: expected: %d\n", - i, n, (int)sizeof(len)); + fprintf(stderr, "%s: len name[%d] read -> %d: expected: %d\n", + "__pmLogLoadMeta", i, n, (int)sizeof(len)); } if (__pmFerror(f)) { __pmClearerr(f); @@ -729,8 +767,8 @@ PM_FAULT_POINT("libpcp/" __FILE__ ":2", if ((n = (int)__pmFread(name, 1, len, f)) != len) { if (pmDebugOptions.logmeta) { - fprintf(stderr, "__pmLogLoadMeta: name[%d] read -> %d: expected: %d\n", - i, n, len); + fprintf(stderr, "%s: name[%d] read -> %d: expected: %d\n", + "__pmLogLoadMeta", i, n, len); } if (__pmFerror(f)) { __pmClearerr(f); @@ -743,36 +781,23 @@ PM_FAULT_POINT("libpcp/" __FILE__ ":2", name[len] = '\0'; if (pmDebugOptions.logmeta) { char strbuf[20]; - fprintf(stderr, "__pmLogLoadMeta: PMID: %s name: %s\n", - pmIDStr_r(dp->pmid, strbuf, sizeof(strbuf)), name); + fprintf(stderr, "%s: PMID: %s name: %s\n", + "__pmLogLoadMeta", + pmIDStr_r(desc.pmid, strbuf, sizeof(strbuf)), name); } - /* Add the new PMNS node */ - if ((sts = __pmAddPMNSNode(lcp->l_pmns, dp->pmid, name)) < 0) { - /* - * If we see a duplicate name with a different PMID, its a - * recoverable error. - * We wont be able to see all of the data in the log, but - * its better to provide access to some rather than none, - * esp. when only one or two metric IDs may be corrupted - * in this way (which we may not be interested in anyway). - */ - if (sts != PM_ERR_PMID) - goto end; - } + + /* Add the new PMNS node into this context */ + if ((sts = __pmLogAddPMNSNode(acp, desc.pmid, name)) < 0) + goto end; }/*for*/ } else if (h.type == TYPE_INDOM) { - int *tbuf; - pmInDom indom; - pmTimeval *when; - int numinst; - int *instlist; - char **namelist; + pmTimeval *tv; + pmTimespec when; + pmInResult in; char *namebase; - int *stridx; - int i; - int k; - int allinbuf = 0; + int *tbuf, *stridx; + int i, k, allinbuf = 0; PM_FAULT_POINT("libpcp/" __FILE__ ":3", PM_FAULT_ALLOC); if ((tbuf = (int *)malloc(rlen)) == NULL) { @@ -781,8 +806,8 @@ PM_FAULT_POINT("libpcp/" __FILE__ ":3", } if ((n = (int)__pmFread(tbuf, 1, rlen, f)) != rlen) { if (pmDebugOptions.logmeta) { - fprintf(stderr, "__pmLogLoadMeta: indom read -> %d: expected: %d\n", - n, rlen); + fprintf(stderr, "%s: indom read -> %d: expected: %d\n", + "__pmLogLoadMeta", n, rlen); } if (__pmFerror(f)) { __pmClearerr(f); @@ -795,44 +820,44 @@ PM_FAULT_POINT("libpcp/" __FILE__ ":3", } k = 0; - when = (pmTimeval *)&tbuf[k]; - when->tv_sec = ntohl(when->tv_sec); - when->tv_usec = ntohl(when->tv_usec); - k += sizeof(*when)/sizeof(int); - indom = __ntohpmInDom((unsigned int)tbuf[k++]); - numinst = ntohl(tbuf[k++]); - if (numinst > 0) { - instlist = &tbuf[k]; - k += numinst; + tv = (pmTimeval *)&tbuf[k]; + when.tv_sec = ntohl(tv->tv_sec); + when.tv_nsec = ntohl(tv->tv_usec) * 1000; + k += sizeof(*tv)/sizeof(int); + in.indom = __ntohpmInDom((unsigned int)tbuf[k++]); + in.numinst = ntohl(tbuf[k++]); + if (in.numinst > 0) { + in.instlist = &tbuf[k]; + k += in.numinst; stridx = &tbuf[k]; #if defined(HAVE_32BIT_PTR) - namelist = (char **)stridx; + in.namelist = (char **)stridx; allinbuf = 1; /* allocation is all in tbuf */ #else allinbuf = 0; /* allocation for namelist + tbuf */ /* need to allocate to hold the pointers */ PM_FAULT_POINT("libpcp/" __FILE__ ":4", PM_FAULT_ALLOC); - namelist = (char **)malloc(numinst*sizeof(char*)); - if (namelist == NULL) { + in.namelist = (char **)malloc(in.numinst * sizeof(char*)); + if (in.namelist == NULL) { sts = -oserror(); free(tbuf); goto end; } #endif - k += numinst; + k += in.numinst; namebase = (char *)&tbuf[k]; - for (i = 0; i < numinst; i++) { - instlist[i] = ntohl(instlist[i]); - namelist[i] = &namebase[ntohl(stridx[i])]; + for (i = 0; i < in.numinst; i++) { + in.instlist[i] = ntohl(in.instlist[i]); + in.namelist[i] = &namebase[ntohl(stridx[i])]; } - if ((sts = addindom(lcp, indom, when, numinst, instlist, namelist, tbuf, allinbuf)) < 0) + if ((sts = __pmLogAddInDom(acp, &when, &in, tbuf, allinbuf)) < 0) goto end; /* If this indom was a duplicate, then we need to free tbuf and namelist, as appropriate. */ if (sts == PMLOGPUTINDOM_DUP) { free(tbuf); - if (namelist != NULL && !allinbuf) - free(namelist); + if (in.namelist != NULL && !allinbuf) + free(in.namelist); } } else { @@ -860,8 +885,8 @@ PM_FAULT_POINT("libpcp/" __FILE__ ":11", } if ((n = (int)__pmFread(tbuf, 1, rlen, f)) != rlen) { if (pmDebugOptions.logmeta) { - fprintf(stderr, "__pmLogLoadMeta: label read -> %d: expected: %d\n", - n, rlen); + fprintf(stderr, "%s: label read -> %d: expected: %d\n", + "__pmLogLoadMeta", n, rlen); } if (__pmFerror(f)) { __pmClearerr(f); @@ -908,7 +933,8 @@ PM_FAULT_POINT("libpcp/" __FILE__ ":11", if (jsonlen < 0 || jsonlen > PM_MAXLABELJSONLEN) { if (pmDebugOptions.logmeta) - fprintf(stderr, "__pmLogLoadMeta: corrupted json in labelset. jsonlen=%d\n", jsonlen); + fprintf(stderr, "%s: corrupted json in labelset. jsonlen=%d\n", + "__pmLogLoadMeta", jsonlen); sts = PM_ERR_LOGREC; free(labelsets); free(tbuf); @@ -935,7 +961,8 @@ PM_FAULT_POINT("libpcp/" __FILE__ ":11", if (nlabels > PM_MAXLABELS || k + nlabels * sizeof(pmLabel) > rlen) { /* corrupt archive metadata detected. GH #475 */ if (pmDebugOptions.logmeta) - fprintf(stderr, "__pmLogLoadMeta: corrupted labelset. nlabels=%d\n", nlabels); + fprintf(stderr, "%s: corrupted labelset. nlabels=%d\n", + "__pmLogLoadMeta", nlabels); sts = PM_ERR_LOGREC; free(labelsets); free(tbuf); @@ -975,8 +1002,8 @@ PM_FAULT_POINT("libpcp/" __FILE__ ":16", } if ((n = (int)__pmFread(tbuf, 1, rlen, f)) != rlen) { if (pmDebugOptions.logmeta) { - fprintf(stderr, "__pmLogLoadMeta: text read -> %d: expected: %d\n", - n, rlen); + fprintf(stderr, "%s: text read -> %d: expected: %d\n", + "__pmLogLoadMeta", n, rlen); } if (__pmFerror(f)) { __pmClearerr(f); @@ -1005,8 +1032,8 @@ PM_FAULT_POINT("libpcp/" __FILE__ ":16", ident = __ntohpmID(*((unsigned int *)&tbuf[k])); else { if (pmDebugOptions.logmeta) { - fprintf(stderr, "__pmLogLoadMeta: bad text ident -> %x\n", - type); + fprintf(stderr, "%s: bad text ident -> %x\n", + "__pmLogLoadMeta", type); } free(tbuf); continue; @@ -1024,8 +1051,9 @@ PM_FAULT_POINT("libpcp/" __FILE__ ":16", check = ntohl(check); if (n != sizeof(check) || h.len != check) { if (pmDebugOptions.logmeta) { - fprintf(stderr, "__pmLogLoadMeta: trailer read -> %d or len=%d: expected %d @ offset=%d\n", - n, check, h.len, (int)(__pmFtell(f) - sizeof(check))); + fprintf(stderr, "%s: trailer read -> %d or len=%d: " + "expected %d @ offset=%d\n", "__pmLogLoadMeta", + n, check, h.len, (int)(__pmFtell(f) - sizeof(check))); } if (__pmFerror(f)) { __pmClearerr(f); @@ -1046,7 +1074,7 @@ end: if (sts == 0) { if (numpmid == 0) { if (pmDebugOptions.logmeta) { - fprintf(stderr, "__pmLogLoadMeta: no metrics found?\n"); + fprintf(stderr, "%s: no metrics found?\n", "__pmLogLoadMeta"); } sts = PM_ERR_LOGREC; } diff -Naurp pcp-5.0.2.orig/src/libpcp/src/logutil.c pcp-5.0.2/src/libpcp/src/logutil.c --- pcp-5.0.2.orig/src/libpcp/src/logutil.c 2018-07-08 10:58:08.000000000 +1000 +++ pcp-5.0.2/src/libpcp/src/logutil.c 2020-02-03 13:23:15.263762911 +1100 @@ -1,5 +1,5 @@ /* - * Copyright (c) 2012-2017 Red Hat. + * Copyright (c) 2012-2017,2020 Red Hat. * Copyright (c) 1995-2002,2004 Silicon Graphics, Inc. All Rights Reserved. * * This library is free software; you can redistribute it and/or modify it @@ -764,6 +764,22 @@ __pmLogClose(__pmArchCtl *acp) } int +__pmLogAddVolume(__pmArchCtl *acp, unsigned int vol) +{ + __pmLogCtl *lcp = acp->ac_log; + + if (lcp->l_minvol == -1) { + lcp->l_minvol = vol; + lcp->l_maxvol = vol; + } else if (vol < lcp->l_minvol) { + lcp->l_minvol = vol; + } else if (vol > lcp->l_maxvol) { + lcp->l_maxvol = vol; + } + return 0; +} + +int __pmLogLoadLabel(__pmArchCtl *acp, const char *name) { __pmLogCtl *lcp = acp->ac_log; @@ -876,21 +892,14 @@ __pmLogLoadLabel(__pmArchCtl *acp, const } } else { - char *q; - int vol; - vol = (int)strtol(tp, &q, 10); + char *q; + unsigned int vol; + + vol = (unsigned int)strtoul(tp, &q, 10); if (*q == '\0') { exists = 1; - if (lcp->l_minvol == -1) { - lcp->l_minvol = vol; - lcp->l_maxvol = vol; - } - else { - if (vol < lcp->l_minvol) - lcp->l_minvol = vol; - if (vol > lcp->l_maxvol) - lcp->l_maxvol = vol; - } + if ((sts = __pmLogAddVolume(acp, vol)) < 0) + goto cleanup; } } } @@ -2282,7 +2291,7 @@ __pmLogSetTime(__pmContext *ctxp) int match = 0; int vol; int numti = lcp->l_numti; - __pmFILE *f; + __pmFILE *f; __pmLogTI *tip = lcp->l_ti; double t_lo; struct stat sbuf; diff -Naurp pcp-5.0.2.orig/src/libpcp_web/src/discover.c pcp-5.0.2/src/libpcp_web/src/discover.c --- pcp-5.0.2.orig/src/libpcp_web/src/discover.c 2019-12-10 17:04:20.000000000 +1100 +++ pcp-5.0.2/src/libpcp_web/src/discover.c 2020-02-03 13:36:11.958637560 +1100 @@ -1,5 +1,5 @@ /* - * Copyright (c) 2018-2019 Red Hat. + * Copyright (c) 2018-2020 Red Hat. * * This library is free software; you can redistribute it and/or modify it * under the terms of the GNU Lesser General Public License as published @@ -14,6 +14,8 @@ #include "discover.h" #include "slots.h" #include "util.h" +#include +#include /* Decode various archive metafile records (desc, indom, labels, helptext) */ static int pmDiscoverDecodeMetaDesc(uint32_t *, int, pmDesc *, int *, char ***); @@ -24,11 +26,15 @@ static int pmDiscoverDecodeMetaLabelSet( /* array of registered callbacks, see pmDiscoverSetup() */ static int discoverCallBackTableSize; static pmDiscoverCallBacks **discoverCallBackTable; +static char *pmDiscoverFlagsStr(pmDiscover *); /* internal hash table of discovered paths */ -#define PM_DISCOVER_HASHTAB_SIZE 64 +#define PM_DISCOVER_HASHTAB_SIZE 16 static pmDiscover *discover_hashtable[PM_DISCOVER_HASHTAB_SIZE]; +/* pmlogger_daily log-roll lock count */ +static int lockcnt = 0; + /* FNV string hash algorithm. Return unsigned in range 0 .. limit-1 */ static unsigned int strhash(const char *s, unsigned int limit) @@ -43,18 +49,38 @@ strhash(const char *s, unsigned int limi return h % limit; } +/* ctime string - note static buf is returned */ +static char * +stamp(void) +{ + time_t now = time(NULL); + char *p, *c = ctime(&now); + + if ((p = strrchr(c, '\n')) != NULL) + *p = '\0'; + return c; +} + /* - * Lookup or Add a discovered file path (directory or PCP archive file) + * Lookup or Add a discovered file path (directory or PCP archive file). + * Note: the fullpath suffix (.meta, .[0-9]+) should already be stripped. * Return path table entry (new or existing). */ static pmDiscover * -pmDiscoverLookupAdd(const char *path, pmDiscoverModule *module, void *arg) +pmDiscoverLookupAdd(const char *fullpath, pmDiscoverModule *module, void *arg) { pmDiscover *p, *h; - unsigned int k = strhash(path, PM_DISCOVER_HASHTAB_SIZE); + unsigned int k; + sds name; + + name = sdsnew(fullpath); + k = strhash(name, PM_DISCOVER_HASHTAB_SIZE); + + if (pmDebugOptions.discovery) + fprintf(stderr, "pmDiscoverLookupAdd: name=%s\n", name); for (p = NULL, h = discover_hashtable[k]; h != NULL; p = h, h = h->next) { - if (strcmp(h->context.name, path) == 0) + if (sdscmp(h->context.name, name) == 0) break; } @@ -65,14 +91,24 @@ pmDiscoverLookupAdd(const char *path, pm h->ctx = -1; /* no PMAPI context initially */ h->flags = PM_DISCOVER_FLAGS_NEW; h->context.type = PM_CONTEXT_ARCHIVE; - h->context.name = sdsnew(path); + h->context.name = name; h->module = module; h->data = arg; if (p == NULL) discover_hashtable[k] = h; else p->next = h; + if (pmDebugOptions.discovery) + fprintf(stderr, "pmDiscoverLookupAdd: --> new entry %s\n", name); + + } + else { + /* already in hash table, so free the buffer */ + if (pmDebugOptions.discovery) + fprintf(stderr, "pmDiscoverLookupAdd: --> existing entry %s\n", name); + sdsfree(name); } + return h; } @@ -82,12 +118,6 @@ pmDiscoverLookup(const char *path) return pmDiscoverLookupAdd(path, NULL, NULL); } -static pmDiscover * -pmDiscoverAdd(const char *path, pmDiscoverModule *module, void *arg) -{ - return pmDiscoverLookupAdd(path, module, arg); -} - static void pmDiscoverFree(pmDiscover *p) { @@ -101,39 +131,42 @@ pmDiscoverFree(pmDiscover *p) sdsfree(p->context.source); if (p->context.labelset) pmFreeLabelSets(p->context.labelset, 1); + if (p->event_handle) { + uv_fs_event_stop(p->event_handle); + free(p->event_handle); + p->event_handle = NULL; + } + memset(p, 0, sizeof(*p)); free(p); } /* - * Delete tracking of a previously discovered path. Frees resources and - * destroy PCP context (if any). + * Traverse and invoke callback for all paths matching any bit + * in the flags bitmap. Callback can be NULL to just get a count. + * Return count of matching paths, may be 0. */ -static void -pmDiscoverDelete(sds path) +static int +pmDiscoverTraverse(unsigned int flags, void (*callback)(pmDiscover *)) { - pmDiscover *p, *h; - unsigned int k = strhash(path, PM_DISCOVER_HASHTAB_SIZE); + int count = 0, i; + pmDiscover *p; - for (p = NULL, h = discover_hashtable[k]; h != NULL; p = h, h = h->next) { - if (sdscmp(h->context.name, path) == 0) { - if (p == NULL) - discover_hashtable[k] = NULL; - else - p->next = h->next; - pmDiscoverFree(h); - break; + for (i = 0; i < PM_DISCOVER_HASHTAB_SIZE; i++) { + for (p = discover_hashtable[i]; p; p = p->next) { + if (p->flags & flags) { + if (callback) + callback(p); + count++; + } } } + return count; } -/* - * Traverse and invoke callback for all paths matching any bit - * in the flags bitmap. Callback can be NULL to just get a count. - * Return count of matching paths, may be 0. - */ +/* as above, but with an extra (void *)arg passed to the cb */ static int -pmDiscoverTraverse(unsigned int flags, void (*callback)(pmDiscover *)) +pmDiscoverTraverseArg(unsigned int flags, void (*callback)(pmDiscover *, void *), void *arg) { int count = 0, i; pmDiscover *p; @@ -142,7 +175,7 @@ pmDiscoverTraverse(unsigned int flags, v for (p = discover_hashtable[i]; p; p = p->next) { if (p->flags & flags) { if (callback) - callback(p); + callback(p, arg); count++; } } @@ -150,6 +183,7 @@ pmDiscoverTraverse(unsigned int flags, v return count; } + /* * Traverse and purge deleted entries * Return count of purged entries. @@ -173,6 +207,9 @@ pmDiscoverPurgeDeleted(void) prev->next = next; else discover_hashtable[i] = next; + if (pmDebugOptions.discovery) + fprintf(stderr, "pmDiscoverPurgeDeleted: deleted %s %s\n", + p->context.name, pmDiscoverFlagsStr(p)); pmDiscoverFree(p); count++; } @@ -180,14 +217,32 @@ pmDiscoverPurgeDeleted(void) } } - if (pmDebugOptions.discovery) - fprintf(stderr, "%s: purged %d entries\n", - "pmDiscoverPurgeDeleted", count); - return count; } /* + * if string ends with given suffix then return pointer + * to start of suffix in string, else NULL + */ +static char * +strsuffix(char *s, const char *suffix) +{ + int slen, suflen; + char *ret = NULL; + + if (s && suffix) { + slen = strlen(s); + suflen = strlen(suffix); + if (slen >= suflen) { + ret = s + (slen - suflen); + if (strncmp(ret, suffix, suflen) != 0) + ret = NULL; + } + } + return ret; +} + +/* * Discover dirs and archives - add new entries or refresh existing. * Call this for each top-level directory. Discovered paths are not * automatically monitored. After discovery, need to traverse and @@ -196,44 +251,88 @@ pmDiscoverPurgeDeleted(void) static int pmDiscoverArchives(const char *dir, pmDiscoverModule *module, void *arg) { - uv_fs_t sreq, req; - uv_dirent_t dent; - uv_stat_t *s; + DIR *dirp; + struct dirent *dent; + struct stat *s; + struct stat statbuf; pmDiscover *a; + char *suffix; char path[MAXNAMELEN]; - char basepath[MAXNAMELEN]; int sep = pmPathSeparator(); + int vol; + + /* + * note: pmDiscoverLookupAdd sets PM_DISCOVER_FLAGS_NEW + * if this is a newly discovered archive or directory + */ + a = pmDiscoverLookupAdd(dir, module, arg); + a->flags |= PM_DISCOVER_FLAGS_DIRECTORY; - if (uv_fs_scandir(NULL, &req, dir, 0, NULL) < 0) + if ((dirp = opendir(dir)) == NULL) { + if (pmDebugOptions.discovery) + fprintf(stderr, "pmDiscoverArchives: opendir %s failed %s: err %d\n", dir, path, errno); return -ESRCH; + } - a = pmDiscoverAdd(dir, module, arg); - a->flags |= PM_DISCOVER_FLAGS_DIRECTORY; + while ((dent = readdir(dirp)) != NULL) { + if (dent->d_name[0] == '.') + continue; + pmsprintf(path, sizeof(path), "%s%c%s", dir, sep, dent->d_name); + + if (pmDebugOptions.discovery) + fprintf(stderr, "pmDiscoverArchives: readdir found %s\n", path); - while (uv_fs_scandir_next(&req, &dent) != UV_EOF) { - pmsprintf(path, sizeof(path), "%s%c%s", dir, sep, dent.name); - if (uv_fs_stat(NULL, &sreq, path, NULL) < 0) + if (stat(path, &statbuf) < 0) { + if (pmDebugOptions.discovery) + fprintf(stderr, "pmDiscoverArchives: stat failed %s, err %d\n", path, errno); continue; - s = &sreq.statbuf; - strncpy(basepath, path, sizeof(basepath)); /* __pmLogBaseName modifies it's argument */ - if (S_ISREG(s->st_mode) && __pmLogBaseName(basepath) != NULL) { - /* - * An archive file (index, meta or data vol). If compressed, then - * it is read-only and we don't have to monitor it for growth. - */ - a = pmDiscoverAdd(path, module, arg); - a->flags &= ~PM_DISCOVER_FLAGS_DELETED; + } - if (strstr(path, ".meta")) - a->flags |= PM_DISCOVER_FLAGS_META; - else if (strstr(path, ".index")) - a->flags |= PM_DISCOVER_FLAGS_INDEX; - else - a->flags |= PM_DISCOVER_FLAGS_DATAVOL; - - /* compare to libpcp io.c for suffix list */ - if (strstr(path, ".xz") || strstr(path, ".gz")) - a->flags |= PM_DISCOVER_FLAGS_COMPRESSED; + s = &statbuf; + if (S_ISREG(s->st_mode)) { + if ((suffix = strsuffix(path, ".meta")) != NULL) { + /* + * An uncompressed PCP archive meta file. Track the meta + * file - the matching logvol filename varies because logvols + * are periodically rolled by pmlogger. Importantly, process all + * available metadata to EOF before processing any logvol data. + */ + *suffix = '\0'; /* strip suffix from path giving archive name */ + a = pmDiscoverLookupAdd(path, module, arg); + + /* + * note: pmDiscoverLookupAdd sets PM_DISCOVER_FLAGS_NEW + * if this is a newly discovered archive, otherwise we're + * already tracking this archive. + */ + a->flags |= PM_DISCOVER_FLAGS_META; + } + else if ((suffix = __pmLogBaseNameVol(path, &vol)) != NULL && vol >= 0) { + /* + * An archive logvol. This logvol may have been created since + * the context was first opened. Update the context maxvol + * to be sure pmFetchArchive can switch to it in due course. + */ + if ((a = pmDiscoverLookup(path)) != NULL) { + a->flags |= PM_DISCOVER_FLAGS_DATAVOL; + /* ensure archive context knows about this volume */ + if (pmDebugOptions.discovery) + fprintf(stderr, "pmDiscoverArchives: found logvol %s %s vol=%d\n", + a->context.name, pmDiscoverFlagsStr(a), vol); + if (a->ctx >= 0 && vol >= 0) { + __pmContext *ctxp = __pmHandleToPtr(a->ctx); + __pmArchCtl *acp = ctxp->c_archctl; + + __pmLogAddVolume(acp, vol); + PM_UNLOCK(ctxp->c_lock); + } + if (pmDebugOptions.discovery) + fprintf(stderr, "pmDiscoverArchives: added logvol %s %s vol=%d\n", + a->context.name, pmDiscoverFlagsStr(a), vol); + } + } else if (pmDebugOptions.discovery) { + fprintf(stderr, "pmDiscoverArchives: ignored regular file %s\n", path); + } } else if (S_ISDIR(s->st_mode)) { /* @@ -241,29 +340,117 @@ pmDiscoverArchives(const char *dir, pmDi */ pmDiscoverArchives(path, module, arg); } - uv_fs_req_cleanup(&sreq); } - uv_fs_req_cleanup(&req); + if (dirp) + closedir(dirp); /* success */ return 0; } +/* + * Return 1 if monitored path has been deleted. + * For archives, we only check the meta file because + * a logvol can be deleted (e.g. via compression when + * the logvol is rolled to a new volume) without + * actually deleting the archive. + */ +static int +is_deleted(pmDiscover *p, struct stat *sbuf) +{ + int ret = 0; + + if (p->flags & PM_DISCOVER_FLAGS_DIRECTORY) { + if (stat(p->context.name, sbuf) < 0) + ret = 1; /* directory has been deleted */ + } + + if (p->flags & (PM_DISCOVER_FLAGS_META|PM_DISCOVER_FLAGS_DATAVOL)) { + sds meta = sdsnew(p->context.name); + meta = sdscat(meta, ".meta"); + if (stat(meta, sbuf) < 0) { + /* + * Archive metadata file has been deleted (or compressed) + * hence consider the archive to be deleted because there + * is no more data to logtail. + */ + ret = 1; + } + sdsfree(meta); + } + + if (pmDebugOptions.discovery) { + fprintf(stderr, "is_deleted: checking %s (%s) -> %s\n", + p->context.name, pmDiscoverFlagsStr(p), ret ? "DELETED" : "no"); + } + + return ret; +} + +static void +logdir_is_locked_callBack(pmDiscover *p, void *arg) +{ + int *cntp = (int *)arg; + char sep = pmPathSeparator(); + char path[MAXNAMELEN]; + + pmsprintf(path, sizeof(path), "%s%c%s", p->context.name, sep, "lock"); + if (access(path, F_OK) == 0) + (*cntp)++; +} + +static void +check_deleted(pmDiscover *p) +{ + struct stat sbuf; + if (!(p->flags & PM_DISCOVER_FLAGS_DELETED) && is_deleted(p, &sbuf)) + p->flags |= PM_DISCOVER_FLAGS_DELETED; +} + static void fs_change_callBack(uv_fs_event_t *handle, const char *filename, int events, int status) { char buffer[MAXNAMELEN]; size_t bytes = sizeof(buffer) - 1; pmDiscover *p; - uv_fs_t sreq; + char *s; sds path; - int path_changed = 0; + int count = 0; + struct stat statbuf; + + /* + * check if logs are currently being rolled by pmlogger_daily et al + * in any of the directories we are tracking. For mutex, the log control + * scripts use a 'lock' file in each directory as it is processed. + */ + pmDiscoverTraverseArg(PM_DISCOVER_FLAGS_DIRECTORY, + logdir_is_locked_callBack, (void *)&count); + + if (lockcnt == 0 && count > 0) { + /* log-rolling has started */ + fprintf(stderr, "%s discovery callback ignored: log-rolling is now in progress\n", stamp()); + lockcnt = count; + return; + } + + if (lockcnt > 0 && count > 0) { + /* log-rolling is still in progress */ + lockcnt = count; + return; + } + + if (lockcnt > 0 && count == 0) { + /* log-rolling is finished: check what got deleted, and then purge */ + fprintf(stderr, "%s discovery callback: finished log-rolling\n", stamp()); + pmDiscoverTraverse(PM_DISCOVER_FLAGS_META|PM_DISCOVER_FLAGS_DATAVOL, check_deleted); + } + lockcnt = count; uv_fs_event_getpath(handle, buffer, &bytes); path = sdsnewlen(buffer, bytes); if (pmDebugOptions.discovery) { - fprintf(stderr, "%s: event on %s -", "fs_change_callBack", path); + fprintf(stderr, "fs_change_callBack: event on %s -", path); if (events & UV_RENAME) fprintf(stderr, " renamed"); if (events & UV_CHANGE) @@ -271,38 +458,40 @@ fs_change_callBack(uv_fs_event_t *handle fputc('\n', stderr); } + /* - * Lookup the path, stat and update it's flags accordingly. If the - * path has been deleted, stop it's event monitor and free the req buffer. - * Then call the pmDiscovery callback. + * Strip ".meta" suffix (if any) and lookup the path. stat and update it's + * flags accordingly. If the path has been deleted, stop it's event monitor + * and free the req buffer, else call the pmDiscovery callback. */ - if ((p = pmDiscoverLookup(path)) == NULL) { + if ((s = strsuffix(path, ".meta")) != NULL) + *s = '\0'; + + p = pmDiscoverLookup(path); + if (p && pmDebugOptions.discovery) { + fprintf(stderr, "fs_change_callBack: ---> found entry %s (%s)\n", + p->context.name, pmDiscoverFlagsStr(p)); + } + + if (p == NULL) { if (pmDebugOptions.discovery) - fprintf(stderr, "%s: filename %s lookup failed\n", - "fs_change_callBack", filename); + fprintf(stderr, "fs_change_callBack: %s lookup failed\n", filename); } - else if (uv_fs_stat(NULL, &sreq, p->context.name, NULL) < 0) { - p->flags |= PM_DISCOVER_FLAGS_DELETED; - if (p->event_handle) { - uv_fs_event_stop(p->event_handle); - free(p->event_handle); - p->event_handle = NULL; - } + else if (is_deleted(p, &statbuf)) { /* path has been deleted. statbuf is invalid */ + p->flags |= PM_DISCOVER_FLAGS_DELETED; memset(&p->statbuf, 0, sizeof(p->statbuf)); - path_changed = 1; - } - else { - /* avoid spurious events. only call the callBack if it really changed */ - if (p->statbuf.st_mtim.tv_sec != sreq.statbuf.st_mtim.tv_sec || - p->statbuf.st_mtim.tv_nsec != sreq.statbuf.st_mtim.tv_nsec) - path_changed = 1; - p->statbuf = sreq.statbuf; /* struct copy */ - uv_fs_req_cleanup(&sreq); + if (pmDebugOptions.discovery) + fprintf(stderr, "fs_change_callBack: %s (%s) has been deleted", + p->context.name, pmDiscoverFlagsStr(p)); } - if (p && p->changed && path_changed && !(p->flags & PM_DISCOVER_FLAGS_DELETED)) - p->changed(p); + /* + * Something in the directory changed - new or deleted archive, or + * a tracked archive meta data file or logvolume grew + */ + if (p) + p->changed(p); /* returns immediately if PM_DISCOVER_FLAGS_DELETED */ sdsfree(path); } @@ -316,9 +505,14 @@ pmDiscoverMonitor(sds path, void (*callb { discoverModuleData *data; pmDiscover *p; + sds eventfilename; - if ((p = pmDiscoverLookup(path)) == NULL) + if ((p = pmDiscoverLookup(path)) == NULL) { + if (pmDebugOptions.discovery) { + fprintf(stderr, "pmDiscoverMonitor: lookup failed for %s\n", path); + } return -ESRCH; + } data = getDiscoverModuleData(p->module); /* save the discovery callback to be invoked */ @@ -330,9 +524,29 @@ pmDiscoverMonitor(sds path, void (*callb * Start monitoring, using given uv loop. Up to the caller to create * a PCP PMAPI context and to fetch/logtail in the changed callback. */ + eventfilename = sdsnew(p->context.name); uv_fs_event_init(data->events, p->event_handle); - uv_fs_event_start(p->event_handle, fs_change_callBack, p->context.name, + + if (p->flags & PM_DISCOVER_FLAGS_DIRECTORY) { + uv_fs_event_start(p->event_handle, fs_change_callBack, eventfilename, + UV_FS_EVENT_WATCH_ENTRY); + } + else { + /* + * Monitor an archive file. This tracks the archive meta file + * but the change callback processes both meta and logvol on + * every callback (meta before logvol). + */ + eventfilename = sdscat(eventfilename, ".meta"); + uv_fs_event_start(p->event_handle, fs_change_callBack, eventfilename, UV_FS_EVENT_WATCH_ENTRY); + } + + if (pmDebugOptions.discovery) { + fprintf(stderr, "pmDiscoverMonitor: added event for %s (%s)\n", + eventfilename, pmDiscoverFlagsStr(p)); + } + sdsfree(eventfilename); } return 0; @@ -411,41 +625,23 @@ static void changed_callback(pmDiscover static void created_callback(pmDiscover *p) { + if (p->flags & (PM_DISCOVER_FLAGS_COMPRESSED|PM_DISCOVER_FLAGS_INDEX)) + return; /* compressed archives don't grow and we ignore archive index files */ + if (pmDebugOptions.discovery) fprintf(stderr, "CREATED %s, %s\n", p->context.name, pmDiscoverFlagsStr(p)); - p->flags &= ~PM_DISCOVER_FLAGS_NEW; - - if (p->flags & PM_DISCOVER_FLAGS_COMPRESSED) - return; /* compressed archives don't grow */ - if (p->flags & PM_DISCOVER_FLAGS_DIRECTORY) { if (pmDebugOptions.discovery) fprintf(stderr, "MONITOR directory %s\n", p->context.name); pmDiscoverMonitor(p->context.name, changed_callback); } - - if (p->flags & PM_DISCOVER_FLAGS_DATAVOL) { + else if (p->flags & (PM_DISCOVER_FLAGS_META|PM_DISCOVER_FLAGS_DATAVOL)) { if (pmDebugOptions.discovery) - fprintf(stderr, "MONITOR logvol %s\n", p->context.name); + fprintf(stderr, "MONITOR archive %s\n", p->context.name); pmDiscoverMonitor(p->context.name, changed_callback); } - - if (p->flags & PM_DISCOVER_FLAGS_META) { - if (pmDebugOptions.discovery) - fprintf(stderr, "MONITOR metadata %s\n", p->context.name); - pmDiscoverMonitor(p->context.name, changed_callback); - } -} - -static void -deleted_callback(pmDiscover *p) -{ - if (pmDebugOptions.discovery) - fprintf(stderr, "DELETED %s (%s)\n", p->context.name, - pmDiscoverFlagsStr(p)); - pmDiscoverDelete(p->context.name); - /* p is now no longer valid */ + p->flags &= ~PM_DISCOVER_FLAGS_NEW; } static void @@ -509,37 +705,84 @@ static void pmDiscoverInvokeMetricCallBacks(pmDiscover *p, pmTimespec *ts, pmDesc *desc, int numnames, char **names) { + discoverModuleData *data = getDiscoverModuleData(p->module); pmDiscoverCallBacks *callbacks; pmDiscoverEvent event; char buf[32]; - int i; + int i, sts; if (pmDebugOptions.discovery) { fprintf(stderr, "%s[%s]: %s name%s", "pmDiscoverInvokeMetricCallBacks", timespec_str(ts, buf, sizeof(buf)), p->context.source, numnames > 0 ? " " : "(none)\n"); for (i = 0; i < numnames; i++) - printf("\"%s\"%s", names[i], i < numnames - 1 ? ", " : "\n"); + fprintf(stderr, "[%u/%u] \"%s\"%s", i+1, numnames, names[i], + i < numnames - 1 ? ", " : "\n"); pmPrintDesc(stderr, desc); if (pmDebugOptions.labels) fprintf(stderr, "context labels %s\n", p->context.labelset->json); } + if (data->pmids) { + if (dictFind(data->pmids, &desc->pmid) != NULL) + goto out; /* metric contains an already excluded PMID */ + for (i = 0; i < numnames; i++) { + if (regexec(&data->exclude_names, names[i], 0, NULL, 0) == 0) + break; + } + if (i != numnames) { + if (pmDebugOptions.discovery) + fprintf(stderr, "%s: excluding metric %s\n", + "pmDiscoverInvokeMetricCallBacks", names[i]); + /* add this pmid to the exclusion list and return early */ + dictAdd(data->pmids, &desc->pmid, NULL); + goto out; + } + } + if (data->indoms) { + if (dictFind(data->indoms, &desc->indom) != NULL) + goto out; /* metric contains an already excluded InDom */ + } + + if (p->ctx >= 0 && p->context.type == PM_CONTEXT_ARCHIVE) { + __pmContext *ctxp = __pmHandleToPtr(p->ctx); + __pmArchCtl *acp = ctxp->c_archctl; + char idstr[32]; + + if ((sts = __pmLogAddDesc(acp, desc)) < 0) + fprintf(stderr, "%s: failed to add metric descriptor for %s\n", + "pmDiscoverInvokeMetricCallBacks", + pmIDStr_r(desc->pmid, idstr, sizeof(idstr))); + for (i = 0; i < numnames; i++) { + if ((sts = __pmLogAddPMNSNode(acp, desc->pmid, names[i])) < 0) + fprintf(stderr, "%s: failed to add metric name %s for %s\n", + "pmDiscoverInvokeMetricCallBacks", names[i], + pmIDStr_r(desc->pmid, idstr, sizeof(idstr))); + } + PM_UNLOCK(ctxp->c_lock); + } + discover_event_init(p, ts, &event); for (i = 0; i < discoverCallBackTableSize; i++) { if ((callbacks = discoverCallBackTable[i]) && callbacks->on_metric != NULL) callbacks->on_metric(&event, desc, numnames, names, p->data); } + +out: + for (i = 0; i < numnames; i++) + free(names[i]); + free(names); } static void pmDiscoverInvokeInDomCallBacks(pmDiscover *p, pmTimespec *ts, pmInResult *in) { + discoverModuleData *data = getDiscoverModuleData(p->module); pmDiscoverCallBacks *callbacks; pmDiscoverEvent event; char buf[32], inbuf[32]; - int i; + int i, sts = PMLOGPUTINDOM_DUP; /* free after callbacks */ if (pmDebugOptions.discovery) { fprintf(stderr, "%s[%s]: %s numinst %d indom %s\n", @@ -551,22 +794,48 @@ pmDiscoverInvokeInDomCallBacks(pmDiscove fprintf(stderr, "context labels %s\n", p->context.labelset->json); } + if (data->indoms) { + if (dictFind(data->indoms, &in->indom) != NULL) + goto out; /* excluded InDom */ + } + + if (p->ctx >= 0 && p->context.type == PM_CONTEXT_ARCHIVE) { + __pmContext *ctxp = __pmHandleToPtr(p->ctx); + __pmArchCtl *acp = ctxp->c_archctl; + char errmsg[PM_MAXERRMSGLEN]; + + if ((sts = __pmLogAddInDom(acp, ts, in, NULL, 0)) < 0) + fprintf(stderr, "%s: failed to add indom for %s: %s\n", + "pmDiscoverInvokeInDomCallBacks", pmIDStr(in->indom), + pmErrStr_r(sts, errmsg, sizeof(errmsg))); + PM_UNLOCK(ctxp->c_lock); + } + discover_event_init(p, ts, &event); for (i = 0; i < discoverCallBackTableSize; i++) { if ((callbacks = discoverCallBackTable[i]) && callbacks->on_indom != NULL) callbacks->on_indom(&event, in, p->data); } + +out: + if (sts == PMLOGPUTINDOM_DUP) { + for (i = 0; i < in->numinst; i++) + free(in->namelist[i]); + free(in->namelist); + free(in->instlist); + } } static void pmDiscoverInvokeLabelsCallBacks(pmDiscover *p, pmTimespec *ts, int ident, int type, pmLabelSet *sets, int nsets) { + discoverModuleData *data = getDiscoverModuleData(p->module); pmDiscoverCallBacks *callbacks; pmDiscoverEvent event; char buf[32], idbuf[64]; - int i; + int i, sts = -EAGAIN; /* free labelsets after callbacks */ if (pmDebugOptions.discovery) { __pmLabelIdentString(ident, type, idbuf, sizeof(idbuf)); @@ -579,22 +848,48 @@ pmDiscoverInvokeLabelsCallBacks(pmDiscov fprintf(stderr, "context labels %s\n", p->context.labelset->json); } + if ((type & PM_LABEL_ITEM) && data->pmids) { + if (dictFind(data->pmids, &ident) != NULL) + goto out; /* text from an already excluded InDom */ + } + if ((type & (PM_LABEL_INDOM|PM_LABEL_INSTANCES)) && data->indoms) { + if (dictFind(data->indoms, &ident) != NULL) + goto out; /* text from an already excluded InDom */ + } + + if (p->ctx >= 0 && p->context.type == PM_CONTEXT_ARCHIVE) { + __pmContext *ctxp = __pmHandleToPtr(p->ctx); + __pmArchCtl *acp = ctxp->c_archctl; + char errmsg[PM_MAXERRMSGLEN]; + + if ((sts = __pmLogAddLabelSets(acp, ts, type, ident, nsets, sets)) < 0) + fprintf(stderr, "%s: failed to add log labelset: %s\n", + "pmDiscoverInvokeLabelsCallBacks", + pmErrStr_r(sts, errmsg, sizeof(errmsg))); + PM_UNLOCK(ctxp->c_lock); + } + discover_event_init(p, ts, &event); for (i = 0; i < discoverCallBackTableSize; i++) { if ((callbacks = discoverCallBackTable[i]) && callbacks->on_labels != NULL) callbacks->on_labels(&event, ident, type, sets, nsets, p->data); } + +out: + if (sts < 0) + pmFreeLabelSets(sets, nsets); } static void pmDiscoverInvokeTextCallBacks(pmDiscover *p, pmTimespec *ts, int ident, int type, char *text) { + discoverModuleData *data = getDiscoverModuleData(p->module); pmDiscoverCallBacks *callbacks; pmDiscoverEvent event; char buf[32]; - int i; + int i, sts; if (pmDebugOptions.discovery) { fprintf(stderr, "%s[%s]: %s ", "pmDiscoverInvokeTextCallBacks", @@ -612,12 +907,36 @@ pmDiscoverInvokeTextCallBacks(pmDiscover fprintf(stderr, "context labels %s\n", p->context.labelset->json); } + if ((type & PM_TEXT_PMID) && data->pmids) { + if (dictFind(data->pmids, &ident) != NULL) + goto out; /* text from an already excluded InDom */ + } + if ((type & PM_TEXT_INDOM) && data->indoms) { + if (dictFind(data->indoms, &ident) != NULL) + goto out; /* text from an already excluded InDom */ + } + + if (p->ctx >= 0 && p->context.type == PM_CONTEXT_ARCHIVE) { + __pmContext *ctxp = __pmHandleToPtr(p->ctx); + __pmArchCtl *acp = ctxp->c_archctl; + char errmsg[PM_MAXERRMSGLEN]; + + if ((sts = __pmLogAddText(acp, ident, type, text)) < 0) + fprintf(stderr, "%s: failed to add %u text for %u: %s\n", + "pmDiscoverInvokeTextCallBacks", type, ident, + pmErrStr_r(sts, errmsg, sizeof(errmsg))); + PM_UNLOCK(ctxp->c_lock); + } + discover_event_init(p, ts, &event); for (i = 0; i < discoverCallBackTableSize; i++) { if ((callbacks = discoverCallBackTable[i]) && callbacks->on_text != NULL) callbacks->on_text(&event, ident, type, text, p->data); } + +out: + free(text); } static void @@ -645,8 +964,8 @@ pmDiscoverNewSource(pmDiscover *p, int c p->context.labelset = labelset; /* use timestamp from file creation as starting time */ - timestamp.tv_sec = p->statbuf.st_birthtim.tv_sec; - timestamp.tv_nsec = p->statbuf.st_birthtim.tv_nsec; + timestamp.tv_sec = p->statbuf.st_ctim.tv_sec; + timestamp.tv_nsec = p->statbuf.st_ctim.tv_nsec; /* inform utilities that a source has been discovered */ pmDiscoverInvokeSourceCallBacks(p, ×tamp); @@ -664,7 +983,7 @@ process_metadata(pmDiscover *p) pmDesc desc; off_t off; char *buffer; - int e, i, nb, len, nsets; + int e, nb, len, nsets; int type, id; /* pmID or pmInDom */ int nnames; char **names; @@ -674,6 +993,8 @@ process_metadata(pmDiscover *p) __pmLogHdr hdr; sds msg, source; static uint32_t *buf = NULL; + int deleted; + struct stat sbuf; static int buflen = 0; /* @@ -683,14 +1004,17 @@ process_metadata(pmDiscover *p) */ p->flags |= PM_DISCOVER_FLAGS_META_IN_PROGRESS; if (pmDebugOptions.discovery) - fprintf(stderr, "%s: in progress, flags=%s\n", - "process_metadata", pmDiscoverFlagsStr(p)); + fprintf(stderr, "process_metadata: %s in progress %s\n", + p->context.name, pmDiscoverFlagsStr(p)); for (;;) { off = lseek(p->fd, 0, SEEK_CUR); nb = read(p->fd, &hdr, sizeof(__pmLogHdr)); - if (nb <= 0) { - /* we're at EOF or an error. But may still be part way through a record */ + deleted = is_deleted(p, &sbuf); + if (nb <= 0 || deleted) { + /* we're at EOF or an error, or deleted. But may still be part way through a record */ + if (deleted) + p->flags |= PM_DISCOVER_FLAGS_DELETED; break; } @@ -750,10 +1074,6 @@ process_metadata(pmDiscover *p) ts.tv_sec = p->statbuf.st_mtim.tv_sec; ts.tv_nsec = p->statbuf.st_mtim.tv_nsec; pmDiscoverInvokeMetricCallBacks(p, &ts, &desc, nnames, names); - for (i = 0; i < nnames; i++) - free(names[i]); - if (names) - free(names); break; case TYPE_INDOM: @@ -765,12 +1085,6 @@ process_metadata(pmDiscover *p) break; } pmDiscoverInvokeInDomCallBacks(p, &ts, &inresult); - if (inresult.numinst > 0) { - for (i = 0; i < inresult.numinst; i++) - free(inresult.namelist[i]); - free(inresult.namelist); - free(inresult.instlist); - } break; case TYPE_LABEL: @@ -795,13 +1109,13 @@ process_metadata(pmDiscover *p) } else { sdsfree(p->context.source); p->context.source = source; - p->context.labelset = labelset; + if (p->context.labelset) + pmFreeLabelSets(p->context.labelset, 1); + p->context.labelset = __pmDupLabelSets(labelset, 1); pmDiscoverInvokeSourceCallBacks(p, &ts); } } pmDiscoverInvokeLabelsCallBacks(p, &ts, id, type, labelset, nsets); - if (labelset != p->context.labelset) - pmFreeLabelSets(labelset, nsets); break; case TYPE_TEXT: @@ -819,8 +1133,6 @@ process_metadata(pmDiscover *p) ts.tv_sec = p->statbuf.st_mtim.tv_sec; ts.tv_nsec = p->statbuf.st_mtim.tv_nsec; pmDiscoverInvokeTextCallBacks(p, &ts, id, type, buffer); - if (buffer) - free(buffer); break; default: @@ -833,38 +1145,89 @@ process_metadata(pmDiscover *p) } if (partial == 0) - /* flag that all available metadata has been now been read */ + /* flag that all available metadata has now been read */ p->flags &= ~PM_DISCOVER_FLAGS_META_IN_PROGRESS; if (pmDebugOptions.discovery) - fprintf(stderr, "%s : completed, partial=%d flags=%s\n", - "process_metadata", partial, pmDiscoverFlagsStr(p)); + fprintf(stderr, "%s: completed, partial=%d %s %s\n", + "process_metadata", partial, p->context.name, pmDiscoverFlagsStr(p)); } /* - * fetch metric values to EOF and call all registered callbacks + * Fetch metric values to EOF and call all registered callbacks. + * Always process metadata thru to EOF before any logvol data. */ static void -process_logvol_callback(pmDiscover *p) +process_logvol(pmDiscover *p) { + int sts; pmResult *r; pmTimespec ts; + int oldcurvol; + __pmContext *ctxp; + __pmArchCtl *acp; + + for (;;) { + pmUseContext(p->ctx); + ctxp = __pmHandleToPtr(p->ctx); + acp = ctxp->c_archctl; + oldcurvol = acp->ac_curvol; + PM_UNLOCK(ctxp->c_lock); + + if ((sts = pmFetchArchive(&r)) < 0) { + /* err handling to skip to the next vol */ + ctxp = __pmHandleToPtr(p->ctx); + acp = ctxp->c_archctl; + if (oldcurvol < acp->ac_curvol) { + __pmLogChangeVol(acp, acp->ac_curvol); + acp->ac_offset = 0; /* __pmLogFetch will fix it up */ + } + PM_UNLOCK(ctxp->c_lock); + + if (sts == PM_ERR_EOL) { + if (pmDebugOptions.discovery) + fprintf(stderr, "process_logvol: %s end of archive reached\n", + p->context.name); + + /* succesfully processed to current end of log */ + break; + } else { + /* + * This log vol was probably deleted (likely compressed) + * under our feet. Try and skip to the next volume. + * We hold the context lock during error recovery here. + */ + if (pmDebugOptions.discovery) + fprintf(stderr, "process_logvol: %s fetch failed:%s\n", + p->context.name, pmErrStr(sts)); + } - pmUseContext(p->ctx); - while (pmFetchArchive(&r) == 0) { + /* we are done - return and wait for another callback */ + break; + } + + /* + * Fetch succeeded - call the values callback and continue + */ if (pmDebugOptions.discovery) { char tbuf[64], bufs[64]; - fprintf(stderr, "FETCHED @%s [%s] %d metrics\n", - timeval_str(&r->timestamp, tbuf, sizeof(tbuf)), + fprintf(stderr, "process_logvol: %s FETCHED @%s [%s] %d metrics\n", + p->context.name, timeval_str(&r->timestamp, tbuf, sizeof(tbuf)), timeval_stream_str(&r->timestamp, bufs, sizeof(bufs)), r->numpmid); } + + /* + * TODO: persistently save current timestamp, so after being restarted, + * pmproxy can resume where it left off for each archive. + */ ts.tv_sec = r->timestamp.tv_sec; ts.tv_nsec = r->timestamp.tv_usec * 1000; pmDiscoverInvokeValuesCallBack(p, &ts, r); pmFreeResult(r); } + /* datavol is now up-to-date and at EOF */ p->flags &= ~PM_DISCOVER_FLAGS_DATAVOL_READY; } @@ -874,12 +1237,13 @@ pmDiscoverInvokeCallBacks(pmDiscover *p) { int sts; sds msg; + sds metaname; if (p->ctx < 0) { /* * once off initialization on the first event */ - if (p->flags & PM_DISCOVER_FLAGS_DATAVOL) { + if (p->flags & (PM_DISCOVER_FLAGS_DATAVOL | PM_DISCOVER_FLAGS_META)) { struct timeval tvp; /* create the PMAPI context (once off) */ @@ -898,28 +1262,25 @@ pmDiscoverInvokeCallBacks(pmDiscover *p) p->ctx = -1; return; } + /* seek to end of archive for logvol data - see TODO in process_logvol() */ pmSetMode(PM_MODE_FORW, &tvp, 1); - /* note: we do not scan pre-existing logvol data. */ - } - else if (p->flags & PM_DISCOVER_FLAGS_META) { - if ((sts = pmNewContext(p->context.type, p->context.name)) < 0) { - infofmt(msg, "pmNewContext failed for %s: %s\n", - p->context.name, pmErrStr(sts)); - moduleinfo(p->module, PMLOG_ERROR, msg, p->data); - return; - } - pmDiscoverNewSource(p, sts); - /* for archive meta files, p->fd is the direct file descriptor */ - if ((p->fd = open(p->context.name, O_RDONLY)) < 0) { - infofmt(msg, "open failed for %s: %s\n", p->context.name, - osstrerror()); + /* + * For archive meta files, p->fd is the direct file descriptor + * and we pre-scan existing metadata. Note: we do NOT scan + * pre-existing logvol data (see pmSetMode above) + */ + metaname = sdsnew(p->context.name); + metaname = sdscat(metaname, ".meta"); + if ((p->fd = open(metaname, O_RDONLY)) < 0) { + infofmt(msg, "open failed for %s: %s\n", metaname, osstrerror()); moduleinfo(p->module, PMLOG_ERROR, msg, p->data); + sdsfree(metaname); return; } - - /* process all existing metadata */ + /* pre-process all existing metadata */ process_metadata(p); + sdsfree(metaname); } } @@ -943,15 +1304,61 @@ pmDiscoverInvokeCallBacks(pmDiscover *p) } if (p->flags & PM_DISCOVER_FLAGS_META) { - /* process metadata */ + /* process new metadata, if any */ process_metadata(p); } - /* process any unprocessed datavol callbacks */ - pmDiscoverTraverse(PM_DISCOVER_FLAGS_DATAVOL_READY, process_logvol_callback); + if ((p->flags & PM_DISCOVER_FLAGS_META_IN_PROGRESS) == 0) { + /* no metdata read in progress, so process new datavol data, if any */ + process_logvol(p); + } +} + +static void +print_callback(pmDiscover *p) +{ + if (p->flags & PM_DISCOVER_FLAGS_DIRECTORY) { + fprintf(stderr, " DIRECTORY %s %s\n", + p->context.name, pmDiscoverFlagsStr(p)); + } + else { + __pmContext *ctxp; + __pmArchCtl *acp; - /* finally, purge deleted entries, if any */ - pmDiscoverPurgeDeleted(); + if (p->ctx >= 0 && (ctxp = __pmHandleToPtr(p->ctx)) != NULL) { + acp = ctxp->c_archctl; + fprintf(stderr, " ARCHIVE %s fd=%d ctx=%d maxvol=%d ac_curvol=%d ac_offset=%ld %s\n", + p->context.name, p->fd, p->ctx, acp->ac_log->l_maxvol, acp->ac_curvol, + acp->ac_offset, pmDiscoverFlagsStr(p)); + PM_UNLOCK(ctxp->c_lock); + } else { + /* no context yet - probably PM_DISCOVER_FLAGS_NEW */ + fprintf(stderr, " ARCHIVE %s fd=%d ctx=%d %s\n", + p->context.name, p->fd, p->ctx, pmDiscoverFlagsStr(p)); + } + } +} + +/* + * p is a tracked archive and arg is a directory path. + * If p is in the directory, call it's callbacks to + * process metadata and logvol data. This allows better + * scalability because we only process archives in the + * directories that have changed. + */ +static void +directory_changed_cb(pmDiscover *p, void *arg) +{ + char *dirpath = (char *)arg; + int dlen = strlen(dirpath); + + if (strncmp(p->context.name, dirpath, dlen) == 0) { + /* this archive is in this directory - process it's metadata and logvols */ + if (pmDebugOptions.discovery) + fprintf(stderr, "directory_changed_cb: archive %s is in dir %s\n", + p->context.name, dirpath); + pmDiscoverInvokeCallBacks(p); + } } static void @@ -962,27 +1369,46 @@ changed_callback(pmDiscover *p) pmDiscoverFlagsStr(p)); if (p->flags & PM_DISCOVER_FLAGS_DELETED) { - /* path or directory has been deleted - remove from hash table */ - deleted_callback(p); - } - else if (p->flags & PM_DISCOVER_FLAGS_DIRECTORY) { /* - * A changed directory path means a new archive or subdirectory - * has been created - traverse and update the hash table. + * Path has been deleted. Do nothing for now. Will be purged + * in due course by pmDiscoverPurgeDeleted. */ - pmDiscoverArchives(p->context.name, p->module, p->data); - pmDiscoverTraverse(PM_DISCOVER_FLAGS_NEW, created_callback); + return; + } - else if (p->flags & PM_DISCOVER_FLAGS_COMPRESSED) { + + if (p->flags & PM_DISCOVER_FLAGS_COMPRESSED) { /* we do not monitor compressed files - do nothing */ - ; /**/ + return; } - else if (p->flags & (PM_DISCOVER_FLAGS_DATAVOL|PM_DISCOVER_FLAGS_META)) { - /* - * We only monitor uncompressed logvol and metadata paths. Fetch new data - * (metadata or logvol) and call the registered callbacks. + + if (p->flags & PM_DISCOVER_FLAGS_DIRECTORY) { + /* + * A changed directory path means a new archive or subdirectory may have + * been created or deleted - traverse and update the hash table. */ - pmDiscoverInvokeCallBacks(p); + if (pmDebugOptions.discovery) { + fprintf(stderr, "%s DIRECTORY CHANGED %s (%s)\n", + stamp(), p->context.name, pmDiscoverFlagsStr(p)); + } + pmDiscoverArchives(p->context.name, p->module, p->data); + pmDiscoverTraverse(PM_DISCOVER_FLAGS_NEW, created_callback); + + /* + * Walk directory and invoke callbacks for tracked archives in this + * directory that have changed + */ + pmDiscoverTraverseArg(PM_DISCOVER_FLAGS_DATAVOL|PM_DISCOVER_FLAGS_META, + directory_changed_cb, (void *)p->context.name); + + /* finally, purge deleted entries (globally), if any */ + pmDiscoverPurgeDeleted(); + } + + if (pmDebugOptions.discovery) { + fprintf(stderr, "%s -- tracking status\n", stamp()); + pmDiscoverTraverse(PM_DISCOVER_FLAGS_ALL, print_callback); + fprintf(stderr, "--\n"); } } @@ -995,18 +1421,9 @@ dir_callback(pmDiscover *p) static void archive_callback(pmDiscover *p) { - if (p->flags & PM_DISCOVER_FLAGS_COMPRESSED) - return; /* compressed archives don't grow */ - - if (p->flags & PM_DISCOVER_FLAGS_DATAVOL) { - if (pmDebugOptions.discovery) - fprintf(stderr, "DISCOVERED ARCHIVE LOGVOL %s\n", p->context.name); - pmDiscoverMonitor(p->context.name, changed_callback); - } - if (p->flags & PM_DISCOVER_FLAGS_META) { if (pmDebugOptions.discovery) - fprintf(stderr, "DISCOVERED ARCHIVE METADATA %s\n", p->context.name); + fprintf(stderr, "DISCOVERED ARCHIVE %s\n", p->context.name); pmDiscoverMonitor(p->context.name, changed_callback); } } @@ -1048,9 +1465,9 @@ pmDiscoverRegister(const char *dir, pmDi } if (pmDebugOptions.discovery) { - fprintf(stderr, "Now managing %d directories and %d archive files\n", + fprintf(stderr, "Now tracking %d directories and %d archives\n", pmDiscoverTraverse(PM_DISCOVER_FLAGS_DIRECTORY, NULL), - pmDiscoverTraverse(PM_DISCOVER_FLAGS_DATAVOL, NULL)); + pmDiscoverTraverse(PM_DISCOVER_FLAGS_DATAVOL|PM_DISCOVER_FLAGS_META, NULL)); } /* monitor the directories */ diff -Naurp pcp-5.0.2.orig/src/libpcp_web/src/discover.h pcp-5.0.2/src/libpcp_web/src/discover.h --- pcp-5.0.2.orig/src/libpcp_web/src/discover.h 2019-12-10 17:04:20.000000000 +1100 +++ pcp-5.0.2/src/libpcp_web/src/discover.h 2020-02-03 13:36:09.904659047 +1100 @@ -1,5 +1,5 @@ /* - * Copyright (c) 2018-2019 Red Hat. + * Copyright (c) 2018-2020 Red Hat. * * This library is free software; you can redistribute it and/or modify it * under the terms of the GNU Lesser General Public License as published @@ -18,7 +18,9 @@ #include "libpcp.h" #include "mmv_stats.h" #include "slots.h" - +#ifdef HAVE_REGEX_H +#include +#endif #ifdef HAVE_LIBUV #include #else @@ -84,8 +86,8 @@ typedef struct pmDiscover { int fd; /* meta file descriptor */ #ifdef HAVE_LIBUV uv_fs_event_t *event_handle; /* uv fs_notify event handle */ - uv_stat_t statbuf; /* stat buffer from event CB */ #endif + struct stat statbuf; /* stat buffer */ void *baton; /* private internal lib data */ void *data; /* opaque user data pointer */ } pmDiscover; @@ -115,6 +117,10 @@ typedef struct discoverModuleData { struct dict *config; /* configuration dict */ uv_loop_t *events; /* event library loop */ redisSlots *slots; /* server slots data */ + regex_t exclude_names; /* metric names to exclude */ + struct dict *pmids; /* dict of excluded PMIDs */ + unsigned int exclude_indoms; /* exclude instance domains */ + struct dict *indoms; /* dict of excluded InDoms */ void *data; /* user-supplied pointer */ } discoverModuleData; diff -Naurp pcp-5.0.2.orig/src/libpcp_web/src/exports pcp-5.0.2/src/libpcp_web/src/exports --- pcp-5.0.2.orig/src/libpcp_web/src/exports 2019-11-26 16:29:58.000000000 +1100 +++ pcp-5.0.2/src/libpcp_web/src/exports 2020-02-03 13:23:15.264762900 +1100 @@ -178,3 +178,8 @@ PCP_WEB_1.11 { global: pmSeriesLabelValues; } PCP_WEB_1.10; + +PCP_WEB_1.12 { + global: + SDS_NOINIT; +} PCP_WEB_1.11; diff -Naurp pcp-5.0.2.orig/src/libpcp_web/src/load.c pcp-5.0.2/src/libpcp_web/src/load.c --- pcp-5.0.2.orig/src/libpcp_web/src/load.c 2019-12-11 14:01:53.000000000 +1100 +++ pcp-5.0.2/src/libpcp_web/src/load.c 2020-02-03 13:36:03.947721365 +1100 @@ -1,5 +1,5 @@ /* - * Copyright (c) 2017-2019 Red Hat. + * Copyright (c) 2017-2020 Red Hat. * * This library is free software; you can redistribute it and/or modify it * under the terms of the GNU Lesser General Public License as published @@ -112,22 +112,41 @@ load_prepare_metric(const char *name, vo * Iterate over an instance domain and extract names and labels * for each instance. */ -static unsigned int -get_instance_metadata(seriesLoadBaton *baton, pmInDom indom) +static void +get_instance_metadata(seriesLoadBaton *baton, pmInDom indom, int force_refresh) { context_t *cp = &baton->pmapi.context; - unsigned int count = 0; domain_t *dp; indom_t *ip; if (indom != PM_INDOM_NULL) { if ((dp = pmwebapi_add_domain(cp, pmInDom_domain(indom)))) pmwebapi_add_domain_labels(cp, dp); - if ((ip = pmwebapi_add_indom(cp, dp, indom)) && - (count = pmwebapi_add_indom_instances(cp, ip)) > 0) - pmwebapi_add_instances_labels(cp, ip); + if ((ip = pmwebapi_add_indom(cp, dp, indom)) != NULL) { + if (force_refresh) + ip->updated = 1; + if (ip->updated) { + pmwebapi_add_indom_instances(cp, ip); + pmwebapi_add_instances_labels(cp, ip); + } + } } - return count; +} + +static void +get_metric_metadata(seriesLoadBaton *baton, metric_t *metric) +{ + context_t *context = &baton->pmapi.context; + + if (metric->cluster) { + if (metric->cluster->domain) + pmwebapi_add_domain_labels(context, metric->cluster->domain); + pmwebapi_add_cluster_labels(context, metric->cluster); + } + if (metric->indom) + pmwebapi_add_instances_labels(context, metric->indom); + pmwebapi_add_item_labels(context, metric); + pmwebapi_metric_hash(metric); } static metric_t * @@ -140,18 +159,25 @@ new_metric(seriesLoadBaton *baton, pmVal char **nameall = NULL; int count, sts, i; - if ((sts = pmLookupDesc(vsp->pmid, &desc)) < 0) { + if ((sts = pmUseContext(context->context)) < 0) { + fprintf(stderr, "%s: failed to use context for PMID %s: %s\n", + "new_metric", + pmIDStr_r(vsp->pmid, idbuf, sizeof(idbuf)), + pmErrStr_r(sts, errmsg, sizeof(errmsg))); + } else if ((sts = pmLookupDesc(vsp->pmid, &desc)) < 0) { if (sts == PM_ERR_IPC) context->setup = 0; if (pmDebugOptions.series) - fprintf(stderr, "failed to lookup metric %s descriptor: %s", + fprintf(stderr, "%s: failed to lookup metric %s descriptor: %s\n", + "new_metric", pmIDStr_r(vsp->pmid, idbuf, sizeof(idbuf)), pmErrStr_r(sts, errmsg, sizeof(errmsg))); } else if ((sts = count = pmNameAll(vsp->pmid, &nameall)) < 0) { if (sts == PM_ERR_IPC) context->setup = 0; if (pmDebugOptions.series) - fprintf(stderr, "failed to lookup metric %s names: %s", + fprintf(stderr, "%s: failed to lookup metric %s names: %s\n", + "new_metric", pmIDStr_r(vsp->pmid, idbuf, sizeof(idbuf)), pmErrStr_r(sts, errmsg, sizeof(errmsg))); } @@ -160,18 +186,10 @@ new_metric(seriesLoadBaton *baton, pmVal if ((metric = pmwebapi_new_metric(context, NULL, &desc, count, nameall)) == NULL) return NULL; - if (metric->cluster) { - if (metric->cluster->domain) - pmwebapi_add_domain_labels(context, metric->cluster->domain); - pmwebapi_add_cluster_labels(context, metric->cluster); - } - if (metric->indom) - pmwebapi_add_instances_labels(context, metric->indom); - pmwebapi_add_item_labels(context, metric); - pmwebapi_metric_hash(metric); + get_metric_metadata(baton, metric); if (pmDebugOptions.series) { - fprintf(stderr, "new_metric [%s] names:", + fprintf(stderr, "%s [%s] names:\n", "new_metric", pmIDStr_r(vsp->pmid, idbuf, sizeof(idbuf))); for (i = 0; i < count; i++) { pmwebapi_hash_str(metric->names[i].hash, idbuf, sizeof(idbuf)); @@ -409,7 +427,7 @@ pmwebapi_add_valueset(metric_t *metric, } static void -series_cache_update(seriesLoadBaton *baton) +series_cache_update(seriesLoadBaton *baton, struct dict *exclude) { seriesGetContext *context = &baton->pmapi; context_t *cp = &context->context; @@ -418,7 +436,7 @@ series_cache_update(seriesLoadBaton *bat metric_t *metric = NULL; char ts[64]; sds timestamp; - int i, write_meta, write_data; + int i, write_meta, write_inst, write_data; timestamp = sdsnew(timeval_stream_str(&result->timestamp, ts, sizeof(ts))); write_data = (!(baton->flags & PM_SERIES_FLAG_METADATA)); @@ -441,6 +459,12 @@ series_cache_update(seriesLoadBaton *bat dictFetchValue(baton->wanted, &vsp->pmid) == NULL) continue; + /* check if metric to be skipped (optional metric exclusion) */ + if (exclude && (dictFind(exclude, &vsp->pmid)) != NULL) + continue; + + write_meta = write_inst = 0; + /* check if pmid already in hash list */ if ((metric = dictFetchValue(cp->pmids, &vsp->pmid)) == NULL) { /* create a new metric, and add it to load context */ @@ -448,21 +472,22 @@ series_cache_update(seriesLoadBaton *bat continue; write_meta = 1; } else { /* pmid already observed */ - write_meta = 0; + if ((write_meta = metric->cached) == 0) + get_metric_metadata(baton, metric); } /* iterate through result instances and ensure metric_t is complete */ if (metric->error == 0 && vsp->numval < 0) write_meta = 1; if (pmwebapi_add_valueset(metric, vsp) != 0) - write_meta = 1; + write_meta = write_inst = 1; /* record the error code in the cache */ metric->error = (vsp->numval < 0) ? vsp->numval : 0; /* make PMAPI calls to cache metadata */ - if (write_meta && get_instance_metadata(baton, metric->desc.indom) != 0) - continue; + if (write_meta) + get_instance_metadata(baton, metric->desc.indom, write_inst); /* initiate writes to backend caching servers (Redis) */ server_cache_metric(baton, metric, timestamp, write_meta, write_data); @@ -549,7 +574,7 @@ server_cache_window(void *arg) (finish->tv_sec == result->timestamp.tv_sec && finish->tv_usec >= result->timestamp.tv_usec)) { context->done = server_cache_update_done; - series_cache_update(baton); + series_cache_update(baton, NULL); } else { if (pmDebugOptions.series) @@ -1023,7 +1048,7 @@ pmSeriesDiscoverSource(pmDiscoverEvent * sds msg; int i; - if (data == NULL || data->slots == NULL) + if (data == NULL || data->slots == NULL || data->slots->setup == 0) return; baton = (seriesLoadBaton *)calloc(1, sizeof(seriesLoadBaton)); @@ -1032,22 +1057,31 @@ pmSeriesDiscoverSource(pmDiscoverEvent * moduleinfo(module, PMLOG_ERROR, msg, arg); return; } + if ((set = pmwebapi_labelsetdup(p->context.labelset)) == NULL) { + infofmt(msg, "%s: out of memory for labels", "pmSeriesDiscoverSource"); + moduleinfo(module, PMLOG_ERROR, msg, arg); + free(baton); + return; + } + initSeriesLoadBaton(baton, module, 0 /*flags*/, module->on_info, series_discover_done, data->slots, arg); initSeriesGetContext(&baton->pmapi, baton); p->baton = baton; + cp = &baton->pmapi.context; + if (pmDebugOptions.discovery) - fprintf(stderr, "%s: new source %s context=%d\n", - "pmSeriesDiscoverSource", p->context.name, p->ctx); + fprintf(stderr, "%s: new source %s context=%p ctxid=%d\n", + "pmSeriesDiscoverSource", p->context.name, cp, p->ctx); - cp = &baton->pmapi.context; cp->context = p->ctx; cp->type = p->context.type; cp->name.sds = sdsdup(p->context.name); - cp->host = p->context.hostname; - cp->labelset = set = p->context.labelset; + cp->host = sdsdup(p->context.hostname); + cp->labelset = set; + pmwebapi_source_hash(cp->name.hash, set->json, set->jsonlen); pmwebapi_setup_context(cp); set_source_origin(cp); @@ -1095,21 +1129,22 @@ pmSeriesDiscoverLabels(pmDiscoverEvent * sds msg; int i, id; + if (baton == NULL || baton->slots == NULL || baton->slots->setup == 0) + return; + switch (type) { case PM_LABEL_CONTEXT: if (pmDebugOptions.discovery) fprintf(stderr, "%s: context\n", "pmSeriesDiscoverLabels"); if ((labels = pmwebapi_labelsetdup(sets)) != NULL) { -#if 0 /* PCP GH#800 do not free this labelset - it's owned by the discover code */ if (cp->labelset) pmFreeLabelSets(cp->labelset, 1); -#endif cp->labelset = labels; pmwebapi_locate_context(cp); cp->updated = 1; } else { - infofmt(msg, "failed to duplicate label set"); + infofmt(msg, "failed to duplicate %s label set", "context"); moduleinfo(event->module, PMLOG_ERROR, msg, arg); } break; @@ -1125,8 +1160,8 @@ pmSeriesDiscoverLabels(pmDiscoverEvent * pmFreeLabelSets(domain->labelset, 1); domain->labelset = labels; domain->updated = 1; - } else { - infofmt(msg, "failed to duplicate label set"); + } else if (domain) { + infofmt(msg, "failed to duplicate %s label set", "domain"); moduleinfo(event->module, PMLOG_ERROR, msg, arg); } break; @@ -1142,8 +1177,8 @@ pmSeriesDiscoverLabels(pmDiscoverEvent * pmFreeLabelSets(cluster->labelset, 1); cluster->labelset = labels; cluster->updated = 1; - } else { - infofmt(msg, "failed to duplicate label set"); + } else if (cluster) { + infofmt(msg, "failed to duplicate %s label set", "cluster"); moduleinfo(event->module, PMLOG_ERROR, msg, arg); } break; @@ -1159,8 +1194,8 @@ pmSeriesDiscoverLabels(pmDiscoverEvent * pmFreeLabelSets(metric->labelset, 1); metric->labelset = labels; metric->updated = 1; - } else { - infofmt(msg, "failed to duplicate label set"); + } else if (metric) { + infofmt(msg, "failed to duplicate %s label set", "item"); moduleinfo(event->module, PMLOG_ERROR, msg, arg); } break; @@ -1177,8 +1212,8 @@ pmSeriesDiscoverLabels(pmDiscoverEvent * pmFreeLabelSets(indom->labelset, 1); indom->labelset = labels; indom->updated = 1; - } else { - infofmt(msg, "failed to duplicate label set"); + } else if (indom) { + infofmt(msg, "failed to duplicate %s label set", "indom"); moduleinfo(event->module, PMLOG_ERROR, msg, arg); } break; @@ -1196,7 +1231,7 @@ pmSeriesDiscoverLabels(pmDiscoverEvent * if ((instance = dictFetchValue(indom->insts, &id)) == NULL) continue; if ((labels = pmwebapi_labelsetdup(&sets[i])) == NULL) { - infofmt(msg, "failed to dup %s instance labels: %s", + infofmt(msg, "failed to dup indom %s instance label set: %s", pmInDomStr_r(indom->indom, idbuf, sizeof(idbuf)), pmErrStr_r(-ENOMEM, errmsg, sizeof(errmsg))); moduleinfo(event->module, PMLOG_ERROR, msg, arg); @@ -1229,10 +1264,13 @@ pmSeriesDiscoverMetric(pmDiscoverEvent * if (pmDebugOptions.discovery) { for (i = 0; i < numnames; i++) - fprintf(stderr, "pmSeriesDiscoverMetric: [%d/%d] %s - %s\n", + fprintf(stderr, "%s: [%d/%d] %s - %s\n", "pmSeriesDiscoverMetric", i + 1, numnames, pmIDStr(desc->pmid), names[i]); } + if (baton == NULL || baton->slots == NULL || baton->slots->setup == 0) + return; + if ((metric = pmwebapi_add_metric(&baton->pmapi.context, NULL, desc, numnames, names)) == NULL) { infofmt(msg, "%s: failed metric discovery", "pmSeriesDiscoverMetric"); @@ -1244,18 +1282,23 @@ pmSeriesDiscoverMetric(pmDiscoverEvent * void pmSeriesDiscoverValues(pmDiscoverEvent *event, pmResult *result, void *arg) { + pmDiscoverModule *module = event->module; pmDiscover *p = (pmDiscover *)event->data; seriesLoadBaton *baton = p->baton; seriesGetContext *context = &baton->pmapi; + discoverModuleData *data = getDiscoverModuleData(module); if (pmDebugOptions.discovery) fprintf(stderr, "%s: result numpmids=%d\n", "pmSeriesDiscoverValues", result->numpmid); + if (baton == NULL || baton->slots == NULL || baton->slots->setup == 0) + return; + seriesBatonReference(context, "pmSeriesDiscoverValues"); baton->arg = arg; context->result = result; - series_cache_update(baton); + series_cache_update(baton, data->pmids); } void @@ -1271,7 +1314,10 @@ pmSeriesDiscoverInDom(pmDiscoverEvent *e int i; if (pmDebugOptions.discovery) - fprintf(stderr, "pmSeriesDiscoverInDom: %s\n", pmInDomStr(id)); + fprintf(stderr, "%s: %s\n", "pmSeriesDiscoverInDom", pmInDomStr(id)); + + if (baton == NULL || baton->slots == NULL || baton->slots->setup == 0) + return; if ((domain = pmwebapi_add_domain(context, pmInDom_domain(id))) == NULL) { infofmt(msg, "%s: failed indom discovery (domain %u)", @@ -1303,11 +1349,10 @@ pmSeriesDiscoverText(pmDiscoverEvent *ev pmDiscover *p = (pmDiscover *)event->data; seriesLoadBaton *baton = p->baton; - (void)baton; - (void)ident; - (void)type; - (void)text; - (void)arg; + if (pmDebugOptions.discovery) + fprintf(stderr, "%s: ident=%u type=%u arg=%p\n", + "pmSeriesDiscoverText", ident, type, arg); - /* for Redis, help text will need special handling (RediSearch) */ + if (baton == NULL || baton->slots == NULL || baton->slots->setup == 0) + return; } diff -Naurp pcp-5.0.2.orig/src/libpcp_web/src/query.c pcp-5.0.2/src/libpcp_web/src/query.c --- pcp-5.0.2.orig/src/libpcp_web/src/query.c 2019-12-05 17:29:43.000000000 +1100 +++ pcp-5.0.2/src/libpcp_web/src/query.c 2020-02-03 13:23:15.265762890 +1100 @@ -1,5 +1,5 @@ /* - * Copyright (c) 2017-2019 Red Hat. + * Copyright (c) 2017-2020 Red Hat. * * This library is free software; you can redistribute it and/or modify it * under the terms of the GNU Lesser General Public License as published @@ -1243,24 +1243,43 @@ series_prepare_time_reply( series_query_end_phase(baton); } +unsigned int +series_value_count_only(timing_t *tp) +{ + if (tp->window.range || tp->window.delta || + tp->window.start || tp->window.end) + return 0; + return tp->count; +} + static void series_prepare_time(seriesQueryBaton *baton, series_set_t *result) { timing_t *tp = &baton->u.query.timing; unsigned char *series = result->series; seriesGetSID *sid; - char buffer[64]; + char buffer[64], revbuf[64]; sds start, end, key, cmd; - unsigned int i; + unsigned int i, revlen = 0, reverse = 0; + + /* if only 'count' is requested, work back from most recent value */ + if ((reverse = series_value_count_only(tp)) != 0) { + revlen = pmsprintf(revbuf, sizeof(revbuf), "%u", reverse); + start = sdsnew("+"); + } else { + start = sdsnew(timeval_stream_str(&tp->start, buffer, sizeof(buffer))); + } - start = sdsnew(timeval_stream_str(&tp->start, buffer, sizeof(buffer))); if (pmDebugOptions.series) fprintf(stderr, "START: %s\n", start); - if (tp->end.tv_sec) + if (reverse) + end = sdsnew("-"); + else if (tp->end.tv_sec) end = sdsnew(timeval_stream_str(&tp->end, buffer, sizeof(buffer))); else end = sdsnew("+"); /* "+" means "no end" - to the most recent */ + if (pmDebugOptions.series) fprintf(stderr, "END: %s\n", end); @@ -1277,12 +1296,21 @@ series_prepare_time(seriesQueryBaton *ba key = sdscatfmt(sdsempty(), "pcp:values:series:%S", sid->name); - /* XRANGE key t1 t2 */ - cmd = redis_command(4); - cmd = redis_param_str(cmd, XRANGE, XRANGE_LEN); + /* X[REV]RANGE key t1 t2 [count N] */ + if (reverse) { + cmd = redis_command(6); + cmd = redis_param_str(cmd, XREVRANGE, XREVRANGE_LEN); + } else { + cmd = redis_command(4); + cmd = redis_param_str(cmd, XRANGE, XRANGE_LEN); + } cmd = redis_param_sds(cmd, key); cmd = redis_param_sds(cmd, start); cmd = redis_param_sds(cmd, end); + if (reverse) { + cmd = redis_param_str(cmd, "COUNT", sizeof("COUNT")-1); + cmd = redis_param_str(cmd, revbuf, revlen); + } redisSlotsRequest(baton->slots, XRANGE, key, cmd, series_prepare_time_reply, sid); } diff -Naurp pcp-5.0.2.orig/src/libpcp_web/src/schema.c pcp-5.0.2/src/libpcp_web/src/schema.c --- pcp-5.0.2.orig/src/libpcp_web/src/schema.c 2019-11-18 19:35:11.000000000 +1100 +++ pcp-5.0.2/src/libpcp_web/src/schema.c 2020-02-03 13:36:03.948721355 +1100 @@ -1,5 +1,5 @@ /* - * Copyright (c) 2017-2019 Red Hat. + * Copyright (c) 2017-2020 Red Hat. * * This library is free software; you can redistribute it and/or modify it * under the terms of the GNU Lesser General Public License as published @@ -819,7 +819,7 @@ redis_series_metric(redisSlots *slots, m */ /* ensure all metric name strings are mapped */ - for (i = 0; i < metric->numnames; i++) { + for (i = 0; metric->cached == 0 && i < metric->numnames; i++) { assert(metric->names[i].sds != NULL); seriesBatonReference(baton, "redis_series_metric"); redisGetMap(slots, @@ -830,7 +830,8 @@ redis_series_metric(redisSlots *slots, m /* ensure all metric or instance label strings are mapped */ if (metric->desc.indom == PM_INDOM_NULL || metric->u.vlist == NULL) { - series_metric_label_mapping(metric, baton); + if (metric->cached == 0) + series_metric_label_mapping(metric, baton); } else { for (i = 0; i < metric->u.vlist->listcount; i++) { value = &metric->u.vlist->value[i]; @@ -847,7 +848,8 @@ redis_series_metric(redisSlots *slots, m series_name_mapping_callback, baton->info, baton->userdata, baton); - series_instance_label_mapping(metric, instance, baton); + if (instance->cached == 0) + series_instance_label_mapping(metric, instance, baton); } } @@ -941,6 +943,9 @@ redis_series_metadata(context_t *context sds cmd, key; int i; + if (metric->cached) + goto check_instances; + indom = pmwebapi_indom_str(metric, ibuf, sizeof(ibuf)); pmid = pmwebapi_pmid_str(metric, pbuf, sizeof(pbuf)); sem = pmwebapi_semantics_str(metric, sbuf, sizeof(sbuf)); @@ -1000,16 +1005,24 @@ redis_series_metadata(context_t *context cmd = redis_param_sha(cmd, metric->names[i].hash); redisSlotsRequest(slots, SADD, key, cmd, redis_series_source_callback, arg); +check_instances: if (metric->desc.indom == PM_INDOM_NULL || metric->u.vlist == NULL) { - redis_series_labelset(slots, metric, NULL, baton); + if (metric->cached == 0) { + redis_series_labelset(slots, metric, NULL, baton); + metric->cached = 1; + } } else { for (i = 0; i < metric->u.vlist->listcount; i++) { value = &metric->u.vlist->value[i]; if ((instance = dictFetchValue(metric->indom->insts, &value->inst)) == NULL) continue; - redis_series_instance(slots, metric, instance, baton); - redis_series_labelset(slots, metric, instance, baton); + if (instance->cached == 0 || metric->cached == 0) { + redis_series_instance(slots, metric, instance, baton); + redis_series_labelset(slots, metric, instance, baton); + } + instance->cached = 1; } + metric->cached = 1; } } @@ -1210,7 +1223,6 @@ redis_series_stream(redisSlots *slots, s redisSlotsRequest(slots, XADD, key, cmd, redis_series_stream_callback, baton); - key = sdscatfmt(sdsempty(), "pcp:values:series:%s", hash); cmd = redis_command(3); /* EXPIRE key timer */ cmd = redis_param_str(cmd, EXPIRE, EXPIRE_LEN); @@ -1228,9 +1240,6 @@ redis_series_streamed(sds stamp, metric_ char hashbuf[42]; int i; - if (metric->updated == 0) - return; - for (i = 0; i < metric->numnames; i++) { pmwebapi_hash_str(metric->names[i].hash, hashbuf, sizeof(hashbuf)); redis_series_stream(slots, stamp, metric, hashbuf, arg); @@ -1545,7 +1554,10 @@ redis_load_slots_callback( redisSlots *slots = baton->slots; seriesBatonCheckMagic(baton, MAGIC_SLOTS, "redis_load_slots_callback"); + + slots->setup = 1; /* we've received initial response from Redis */ slots->refresh = 0; /* we're processing CLUSTER SLOTS command now */ + /* no cluster redirection checking is needed for this callback */ sdsfree(cmd); @@ -1832,12 +1844,47 @@ pmDiscoverSetup(pmDiscoverModule *module const char fallback[] = "/var/log/pcp"; const char *paths[] = { "pmlogger", "pmmgr" }; const char *logdir = pmGetOptionalConfig("PCP_LOG_DIR"); + struct dict *config; + unsigned int domain, serial; + pmInDom indom; char path[MAXPATHLEN]; char sep = pmPathSeparator(); - int i, sts, count = 0; + sds option, *ids; + int i, sts, nids, count = 0; if (data == NULL) return -ENOMEM; + config = data->config; + + /* double-check that we are supposed to be in here */ + if ((option = pmIniFileLookup(config, "discover", "enabled"))) { + if (strcasecmp(option, "false") == 0) + return 0; + } + + /* prepare for optional metric and indom exclusion */ + if ((option = pmIniFileLookup(config, "discover", "exclude.metrics"))) { + if ((data->pmids = dictCreate(&intKeyDictCallBacks, NULL)) == NULL) + return -ENOMEM; + /* parse regular expression string for matching on metric names */ + regcomp(&data->exclude_names, option, REG_EXTENDED|REG_NOSUB); + } + if ((option = pmIniFileLookup(config, "discover", "exclude.indoms"))) { + if ((data->indoms = dictCreate(&intKeyDictCallBacks, NULL)) == NULL) + return -ENOMEM; + /* parse comma-separated indoms in 'option', convert to pmInDom */ + if ((ids = sdssplitlen(option, sdslen(option), ",", 1, &nids))) { + data->exclude_indoms = nids; + for (i = 0; i < nids; i++) { + if (sscanf(ids[i], "%u.%u", &domain, &serial) == 2) { + indom = pmInDom_build(domain, serial); + dictAdd(data->indoms, &indom, NULL); + } + sdsfree(ids[i]); + } + free(ids); + } + } /* create global EVAL hashes and string map caches */ redisGlobalsInit(data->config); diff -Naurp pcp-5.0.2.orig/src/libpcp_web/src/schema.h pcp-5.0.2/src/libpcp_web/src/schema.h --- pcp-5.0.2.orig/src/libpcp_web/src/schema.h 2019-10-11 17:16:29.000000000 +1100 +++ pcp-5.0.2/src/libpcp_web/src/schema.h 2020-02-03 13:23:15.266762879 +1100 @@ -1,5 +1,5 @@ /* - * Copyright (c) 2017-2018 Red Hat. + * Copyright (c) 2017-2020 Red Hat. * * This library is free software; you can redistribute it and/or modify it * under the terms of the GNU Lesser General Public License as published @@ -51,6 +51,10 @@ #define HSET_LEN (sizeof(HSET)-1) #define HVALS "HVALS" #define HVALS_LEN (sizeof(HVALS)-1) +#define INFO "INFO" +#define INFO_LEN (sizeof(INFO)-1) +#define PING "PING" +#define PING_LEN (sizeof(PING)-1) #define PUBLISH "PUBLISH" #define PUBLISH_LEN (sizeof(PUBLISH)-1) #define SADD "SADD" @@ -63,6 +67,8 @@ #define XADD_LEN (sizeof(XADD)-1) #define XRANGE "XRANGE" #define XRANGE_LEN (sizeof(XRANGE)-1) +#define XREVRANGE "XREVRANGE" +#define XREVRANGE_LEN (sizeof(XREVRANGE)-1) /* create a Redis protocol command (e.g. XADD, SMEMBER) */ static inline sds diff -Naurp pcp-5.0.2.orig/src/libpcp_web/src/slots.c pcp-5.0.2/src/libpcp_web/src/slots.c --- pcp-5.0.2.orig/src/libpcp_web/src/slots.c 2019-10-11 17:16:29.000000000 +1100 +++ pcp-5.0.2/src/libpcp_web/src/slots.c 2020-02-03 13:23:15.266762879 +1100 @@ -356,6 +356,21 @@ redisSlotsRequest(redisSlots *slots, con if (UNLIKELY(pmDebugOptions.desperate)) fputs(cmd, stderr); + if (UNLIKELY(!key && !slots->setup)) { + /* + * First request must be CLUSTER, PING, or similar - must + * not allow regular requests until these have completed. + * This is because the low layers accumulate async requests + * until connection establishment, which might not happen. + * Over time this becomes a memory leak - if we do not ever + * establish an initial connection). + */ + if (strcmp(topic, CLUSTER) != 0 && + strcmp(topic, PING) != 0 && strcmp(topic, INFO) != 0) { + sdsfree(cmd); + return -ENOTCONN; + } + } sts = redisAsyncFormattedCommand(context, callback, cmd, arg); if (key) diff -Naurp pcp-5.0.2.orig/src/libpcp_web/src/slots.h pcp-5.0.2/src/libpcp_web/src/slots.h --- pcp-5.0.2.orig/src/libpcp_web/src/slots.h 2019-04-08 09:11:00.000000000 +1000 +++ pcp-5.0.2/src/libpcp_web/src/slots.h 2020-02-03 13:23:15.266762879 +1100 @@ -44,10 +44,11 @@ typedef struct redisSlotRange { typedef struct redisSlots { unsigned int counter; unsigned int nslots; + unsigned int setup; /* slots info all successfully setup */ + unsigned int refresh; /* do slot refresh whenever possible */ redisSlotRange *slots; /* all instances; e.g. CLUSTER SLOTS */ redisMap *keymap; /* map command names to key position */ dict *contexts; /* async contexts access by hostspec */ - unsigned int refresh; /* do slot refresh whenever possible */ void *events; } redisSlots; diff -Naurp pcp-5.0.2.orig/src/libpcp_web/src/util.c pcp-5.0.2/src/libpcp_web/src/util.c --- pcp-5.0.2.orig/src/libpcp_web/src/util.c 2019-12-10 17:39:49.000000000 +1100 +++ pcp-5.0.2/src/libpcp_web/src/util.c 2020-02-03 13:23:15.266762879 +1100 @@ -1,5 +1,5 @@ /* - * Copyright (c) 2017-2019 Red Hat. + * Copyright (c) 2017-2020 Red Hat. * * This library is free software; you can redistribute it and/or modify it * under the terms of the GNU Lesser General Public License as published @@ -535,6 +535,8 @@ pmwebapi_metric_hash(metric_t *metric) sdsclear(identifier); } sdsfree(identifier); + + metric->cached = 0; } void @@ -574,6 +576,8 @@ pmwebapi_instance_hash(indom_t *ip, inst SHA1Update(&shactx, (unsigned char *)identifier, sdslen(identifier)); SHA1Final(instance->name.hash, &shactx); sdsfree(identifier); + + instance->cached = 0; } sds @@ -1046,7 +1050,6 @@ pmwebapi_add_instance(struct indom *indo instance->name.sds = sdscatlen(instance->name.sds, name, length); pmwebapi_string_hash(instance->name.id, name, length); pmwebapi_instance_hash(indom, instance); - instance->cached = 0; } return instance; } @@ -1202,12 +1205,14 @@ struct metric * pmwebapi_add_metric(context_t *cp, const sds base, pmDesc *desc, int numnames, char **names) { struct metric *metric; - sds name = sdsempty(); + sds name; int i; /* search for a match on any of the given names */ if (base && (metric = dictFetchValue(cp->metrics, base)) != NULL) return metric; + + name = sdsempty(); for (i = 0; i < numnames; i++) { sdsclear(name); name = sdscat(name, names[i]); @@ -1217,6 +1222,7 @@ pmwebapi_add_metric(context_t *cp, const } } sdsfree(name); + return pmwebapi_new_metric(cp, base, desc, numnames, names); } @@ -1230,21 +1236,24 @@ pmwebapi_new_pmid(context_t *cp, const s int sts, numnames; if ((sts = pmUseContext(cp->context)) < 0) { - fprintf(stderr, "failed to use context for PMID %s: %s", + fprintf(stderr, "%s: failed to use context for PMID %s: %s\n", + "pmwebapi_new_pmid", pmIDStr_r(pmid, buffer, sizeof(buffer)), pmErrStr_r(sts, errmsg, sizeof(errmsg))); } else if ((sts = pmLookupDesc(pmid, &desc)) < 0) { if (sts == PM_ERR_IPC) cp->setup = 0; if (pmDebugOptions.series) - fprintf(stderr, "failed to lookup metric %s descriptor: %s", + fprintf(stderr, "%s: failed to lookup metric %s descriptor: %s\n", + "pmwebapi_new_pmid", pmIDStr_r(pmid, buffer, sizeof(buffer)), pmErrStr_r(sts, errmsg, sizeof(errmsg))); } else if ((numnames = sts = pmNameAll(pmid, &names)) < 0) { if (sts == PM_ERR_IPC) cp->setup = 0; if (pmDebugOptions.series) - fprintf(stderr, "failed to lookup metric %s names: %s", + fprintf(stderr, "%s: failed to lookup metric %s names: %s\n", + "pmwebapi_new_pmid", pmIDStr_r(pmid, buffer, sizeof(buffer)), pmErrStr_r(sts, errmsg, sizeof(errmsg))); } else { diff -Naurp pcp-5.0.2.orig/src/pmproxy/pmproxy.conf pcp-5.0.2/src/pmproxy/pmproxy.conf --- pcp-5.0.2.orig/src/pmproxy/pmproxy.conf 2019-08-09 15:50:17.000000000 +1000 +++ pcp-5.0.2/src/pmproxy/pmproxy.conf 2020-02-03 13:36:03.948721355 +1100 @@ -43,6 +43,11 @@ secure.enabled = true # propogate archives from pmlogger(1) into Redis querying enabled = true +# metrics name regex to skip during discovery (eg due to high volume) +exclude.metrics = proc.* + +# comma-separated list of instance domains to skip during discovery +exclude.indoms = 3.9,79.7 ##################################################################### ## settings for fast, scalable time series quering via Redis diff -Naurp pcp-5.0.2.orig/src/pmproxy/src/redis.c pcp-5.0.2/src/pmproxy/src/redis.c --- pcp-5.0.2.orig/src/pmproxy/src/redis.c 2019-12-02 16:39:33.000000000 +1100 +++ pcp-5.0.2/src/pmproxy/src/redis.c 2020-02-03 13:36:13.585620539 +1100 @@ -145,11 +145,11 @@ setup_redis_module(struct proxy *proxy) proxy->slots = redisSlotsConnect(proxy->config, flags, proxylog, on_redis_connected, proxy, proxy->events, proxy); - if (archive_discovery) + if (archive_discovery && series_queries) pmDiscoverSetSlots(&redis_discover.module, proxy->slots); } - if (archive_discovery) { + if (archive_discovery && series_queries) { pmDiscoverSetEventLoop(&redis_discover.module, proxy->events); pmDiscoverSetConfiguration(&redis_discover.module, proxy->config); pmDiscoverSetMetricRegistry(&redis_discover.module, metric_registry);