update to upstream 3.0.0

This commit is contained in:
Andreas Gerstmayr 2020-11-26 12:42:03 +01:00
parent 691da46533
commit ad5bd6ee95
10 changed files with 279 additions and 167 deletions

7
.gitignore vendored
View File

@ -1,4 +1,5 @@
/grafana-pcp-*.tar.gz
/grafana-pcp-deps-*.tar.xz
/grafana-pcp-*.src.rpm
/results_grafana-pcp/
/grafana-pcp-*.tar.xz
/grafana-pcp-*.tar.xz.manifest
/grafana-pcp-*/
*.rpm

View File

@ -0,0 +1,13 @@
diff --git a/pkg/datasources/redis/datasource_test.go b/pkg/datasources/redis/datasource_test.go
index d677825..1ff9fd7 100644
--- a/pkg/datasources/redis/datasource_test.go
+++ b/pkg/datasources/redis/datasource_test.go
@@ -107,7 +107,7 @@ func TestDatasource(t *testing.T) {
So(response.Responses["A"].Frames, ShouldHaveLength, 1)
So(response.Responses["A"].Frames[0].Fields, ShouldHaveLength, 3)
So(response.Responses["A"].Frames[0].Fields[0].Len(), ShouldEqual, 1) // it's a counter metric with two values
- So(response.Responses["A"].Frames[0].Fields[0].At(0).(time.Time).UnixNano(), ShouldEqual, 1599320692309872128)
+ So(response.Responses["A"].Frames[0].Fields[0].At(0).(time.Time).UnixNano(), ShouldEqual, int64(1599320692309872128))
So(response.Responses["A"].Frames[0].Fields[1].Name, ShouldEqual, "disk.dev.read[nvme0n1]")
So(*response.Responses["A"].Frames[0].Fields[1].At(0).(*float64), ShouldEqual, 200)
So(response.Responses["A"].Frames[0].Fields[2].Name, ShouldEqual, "disk.dev.read[sda]")

45
Makefile Normal file
View File

@ -0,0 +1,45 @@
all: grafana-pcp-$(VER).tar.gz \
grafana-pcp-vendor-$(VER).tar.xz \
grafana-pcp-webpack-$(VER).tar.gz
grafana-pcp-$(VER).tar.gz grafana-pcp-$(VER)/:
wget https://github.com/performancecopilot/grafana-pcp/archive/v$(VER)/grafana-pcp-$(VER).tar.gz
rm -rf grafana-pcp-$(VER)
tar xfz grafana-pcp-$(VER).tar.gz
cd grafana-pcp-$(VER) && shopt -s nullglob && \
for patch in ../*.patch; do patch -p1 < $$patch; done
grafana-pcp-vendor-$(VER).tar.xz: grafana-pcp-$(VER)/
# Go
cd grafana-pcp-$(VER) && go mod vendor -v
awk '$$2~/^v/ && $$4 != "indirect" {print "Provides: bundled(golang(" $$1 ")) = " substr($$2, 2)}' grafana-pcp-$(VER)/go.mod | \
sed -E 's/=(.*)-(.*)-(.*)/=\1-\2.\3/g' > $@.manifest
# Node.js
cd grafana-pcp-$(VER) && yarn install --pure-lockfile
# Remove files with licensing issues
find grafana-pcp-$(VER) -type d -name 'node-notifier' -prune -exec rm -r {} \;
find grafana-pcp-$(VER) -name '*.exe' -delete
# Remove not required packages
rm -r grafana-pcp-$(VER)/node_modules/puppeteer
./list_bundled_nodejs_packages.py grafana-pcp-$(VER)/ >> $@.manifest
# Jsonnet
cd grafana-pcp-$(VER) && jb --jsonnetpkg-home=vendor_jsonnet install
# Create tarball
XZ_OPT=-9 tar cfJ $@ \
grafana-pcp-$(VER)/vendor \
$$(find grafana-pcp-$(VER) -type d -name "node_modules" -prune) \
grafana-pcp-$(VER)/vendor_jsonnet
grafana-pcp-webpack-$(VER).tar.gz: grafana-pcp-$(VER)/
cd grafana-pcp-$(VER) && \
yarn install --pure-lockfile && \
make dist-dashboards dist-frontend && \
chmod -R g-w,o-w dist
tar cfz $@ grafana-pcp-$(VER)/dist
clean:
rm -rf *.tar.gz *.tar.xz *.manifest *.rpm grafana-pcp-*/

View File

@ -1,20 +1,22 @@
# grafana-pcp
The grafana-pcp package
## Build instructions
```
VER=2.0.2
spectool -g grafana-pcp.spec
./create_dependency_bundle.sh grafana-pcp-$VER.tar.gz grafana-pcp-deps-$VER.tar.xz
./check_npm_dependencies.py grafana-pcp.spec grafana-pcp-$VER.tar.gz grafana-pcp-deps-$VER.tar.xz
fedpkg new-sources grafana-pcp-$VER.tar.gz grafana-pcp-deps-$VER.tar.xz
## Upgrade instructions
(replace X.Y.Z with the new grafana-pcp version)
fedpkg local
fedpkg lint
fedpkg mockbuild
fedpkg scratch-build --srpm
* update `Version`, `Release` and `%changelog` in the specfile
* create bundles and manifest: `VER=X.Y.Z make clean all`
* update specfile with contents of the `.manifest` file
* run local build: `rpkg local`
* run rpm linter: `rpkg lint -r grafana-pcp.rpmlintrc`
* run a scratch build: `fedpkg scratch-build --srpm`
* upload new source tarballs: `fedpkg new-sources *.tar.gz`
fedpkg build
fedpkg update
```
## Backporting
* create the patch
* declare and apply (`%prep`) the patch in the specfile
* if the patch affects Go or Node.js dependencies, or the webpack
* create new tarballs and rename them to `grafana-pcp-...-X.Y.Z-R.tar.gz`
* update the specfile with new tarball path and contents of the `.manifest` file
Note: the Makefile automatically applies all patches before creating the tarballs

View File

@ -1,42 +0,0 @@
#!/usr/bin/env python3
import sys
import tarfile
import json
import os.path
def read_spec_provides(spec_path):
with open(spec_path) as f:
for line in f:
if line.startswith('Provides:'):
yield line.strip()
def read_node_deps(source_archive_path):
root_dir = os.path.basename(source_archive_path)[:-len('.tar.gz')]
with tarfile.open(source_archive_path) as tar:
f = tar.extractfile(f'{root_dir}/package.json')
package_json = json.load(f)
return list(package_json['devDependencies'].keys()) + list(package_json['dependencies'].keys())
def read_node_deps_versions(dep_bundle_path, dependencies):
with tarfile.open(dep_bundle_path) as tar:
for dependency in dependencies:
f = tar.extractfile(f'node_modules/{dependency}/package.json')
package_json = json.load(f)
yield f'Provides: bundled(nodejs-{package_json["name"]}) = {package_json["version"]}'
if __name__ == '__main__':
if len(sys.argv) != 4:
print('usage: {} specfile source-archive dependency-bundle'.format(sys.argv[0]))
sys.exit(1)
provides = list(read_spec_provides(sys.argv[1]))
dependencies = sorted(read_node_deps(sys.argv[2]))
actual_provides = list(read_node_deps_versions(sys.argv[3], dependencies))
if provides == actual_provides:
print("Dependencies are up-to-date with the specfile.")
sys.exit(0)
else:
print("Dependencies don't match, please update the specfile:")
print ('\n'.join(actual_provides))
sys.exit(1)

View File

@ -1,37 +0,0 @@
#!/bin/bash -eu
SRC=$(readlink -f "${1:?Usage: $0 source destination}")
DEST=$(readlink -f "${2:?Usage: $0 source destination}")
if [ -f "$DEST" ]; then
echo "File $DEST exists already."
exit 0
fi
if [ "$#" -gt 2 ]; then
PATCHES=$(readlink -f "${@:3}")
else
PATCHES=""
fi
pushd "$(mktemp -d)"
echo Extracting sources...
tar xfz "$SRC"
cd grafana-pcp-*
echo Applying patches...
for patch in $PATCHES
do
patch -p1 < $patch
done
echo Installing dependencies...
yarn install
echo Removing files with licensing issues...
rm -rf node_modules/node-notifier
echo Compressing...
XZ_OPT=-9 tar cJf "$DEST" node_modules
popd

View File

@ -3,5 +3,4 @@ addFilter("W: spelling-error %description -l en_US pmseries -> .*")
addFilter("W: spelling-error %description -l en_US bpftrace -> .*")
addFilter("W: spelling-error %description -l en_US pmdabpftrace -> .*")
addFilter("W: strange-permission create_dependency_bundle.sh 755")
addFilter("W: invalid-url Source1: grafana-pcp-deps-.*")
addFilter("W: strange-permission list_bundled_nodejs_packages.py 755")

View File

@ -1,24 +1,54 @@
%global grafanapcp_arches %{lua: go_arches = {}
for arch in rpm.expand("%{go_arches}"):gmatch("%S+") do
go_arches[arch] = 1
end
for arch in rpm.expand("%{nodejs_arches}"):gmatch("%S+") do
if go_arches[arch] then
print(arch .. " ")
end
end}
# Specify if the frontend and dashboards will be compiled as part of the build or
# are attached as a webpack tarball (in case of an unsuitable nodejs or jsonnet version on the build system)
%define compile_frontend 0
Name: grafana-pcp
Version: 2.0.2
Release: 2%{?dist}
Version: 3.0.0
Release: 1%{?dist}
Summary: Performance Co-Pilot Grafana Plugin
%global github https://github.com/performancecopilot/grafana-pcp
%global install_dir %{_sharedstatedir}/grafana/plugins/grafana-pcp
BuildArch: noarch
ExclusiveArch: %{nodejs_arches}
License: ASL 2.0
URL: %{github}
URL: https://github.com/performancecopilot/grafana-pcp
Source0: %{github}/archive/v%{version}/%{name}-%{version}.tar.gz
Source1: grafana-pcp-deps-%{version}.tar.xz
Source2: create_dependency_bundle.sh
Source0: https://github.com/performancecopilot/grafana-pcp/archive/v%{version}/%{name}-%{version}.tar.gz
Source1: grafana-pcp-vendor-%{version}.tar.xz
%if %{compile_frontend} == 0
# Source2 contains the precompiled frontend and dashboards
Source2: grafana-pcp-webpack-%{version}.tar.gz
%endif
Source3: Makefile
Source4: list_bundled_nodejs_packages.py
BuildRequires: nodejs
Requires: grafana >= 6.6.0
Suggests: pcp >= 5.0.0
Patch1: 001-fix-test-on-32bit.patch
# Intersection of go_arches and nodejs_arches
ExclusiveArch: %{grafanapcp_arches}
BuildRequires: systemd-rpm-macros, golang, go-srpm-macros
%if 0%{?fedora}
BuildRequires: go-rpm-macros
%endif
%if %{compile_frontend}
BuildRequires: nodejs >= 1:12, nodejs < 1:13, yarnpkg, golang-github-google-jsonnet
%endif
# omit golang debugsource, see BZ 995136 and related
%global dwz_low_mem_die_limit 0
%global _debugsource_template %{nil}
%global install_dir %{_sharedstatedir}/grafana/plugins/performancecopilot-pcp-app
Requires: grafana >= 7.3.3
Suggests: pcp >= 5.2.2
Suggests: redis >= 5.0.0
Suggests: bpftrace >= 0.9.2
@ -29,48 +59,61 @@ Obsoletes: pcp-webapp-grafana <= 4.3.4
Obsoletes: pcp-webapp-graphite <= 4.3.4
Obsoletes: pcp-webapp-vector <= 4.3.4
# Bundled npm packages
Provides: bundled(nodejs-@babel/cli) = 7.8.4
Provides: bundled(nodejs-@babel/core) = 7.8.4
Provides: bundled(nodejs-@babel/preset-env) = 7.8.4
Provides: bundled(nodejs-@babel/preset-react) = 7.8.3
Provides: bundled(nodejs-@babel/preset-typescript) = 7.8.3
Provides: bundled(nodejs-@grafana/data) = 6.6.0
Provides: bundled(nodejs-@grafana/ui) = 6.6.0
Provides: bundled(nodejs-@types/benchmark) = 1.0.31
Provides: bundled(nodejs-@types/d3) = 5.7.2
Provides: bundled(nodejs-@types/grafana) = 4.6.3
Provides: bundled(nodejs-@types/jest) = 24.9.1
Provides: bundled(nodejs-@types/lodash) = 4.14.149
Provides: bundled(nodejs-babel-jest) = 24.9.0
Provides: bundled(nodejs-babel-loader) = 8.0.6
Provides: bundled(nodejs-babel-plugin-angularjs-annotate) = 0.10.0
Provides: bundled(nodejs-benchmark) = 2.1.4
Provides: bundled(nodejs-clean-webpack-plugin) = 0.1.19
Provides: bundled(nodejs-copy-webpack-plugin) = 5.1.1
Provides: bundled(nodejs-core-js) = 3.6.4
Provides: bundled(nodejs-css-loader) = 1.0.1
Provides: bundled(nodejs-d3-flame-graph) = 2.1.9
Provides: bundled(nodejs-d3-selection) = 1.4.1
Provides: bundled(nodejs-expr-eval) = 1.2.3
Provides: bundled(nodejs-jest) = 24.9.0
Provides: bundled(nodejs-jest-date-mock) = 1.0.8
Provides: bundled(nodejs-jsdom) = 9.12.0
Provides: bundled(nodejs-lodash) = 4.17.15
Provides: bundled(nodejs-memoize-one) = 5.1.1
Provides: bundled(nodejs-mocha) = 6.2.2
Provides: bundled(nodejs-prunk) = 1.3.1
Provides: bundled(nodejs-q) = 1.5.1
Provides: bundled(nodejs-regenerator-runtime) = 0.12.1
Provides: bundled(nodejs-request) = 2.88.0
Provides: bundled(nodejs-style-loader) = 0.22.1
Provides: bundled(nodejs-ts-jest) = 24.3.0
Provides: bundled(nodejs-ts-loader) = 4.5.0
Provides: bundled(nodejs-tslint) = 5.20.1
Provides: bundled(nodejs-tslint-config-airbnb) = 5.11.2
Provides: bundled(nodejs-typescript) = 3.7.5
Provides: bundled(nodejs-webpack) = 4.41.5
Provides: bundled(nodejs-webpack-cli) = 3.3.10
# vendored golang and node.js build dependencies
# this is for security purposes, if nodejs-foo ever needs an update,
# affected packages can be easily identified.
# Note: generated by the Makefile (see README.md)
Provides: bundled(golang(github.com/grafana/grafana-plugin-sdk-go)) = 0.79.0
Provides: bundled(golang(github.com/smartystreets/goconvey)) = 1.6.4
Provides: bundled(npm(@babel/plugin-transform-modules-commonjs)) = 7.12.1
Provides: bundled(npm(@grafana/data)) = 7.3.3
Provides: bundled(npm(@grafana/runtime)) = 7.3.3
Provides: bundled(npm(@grafana/toolkit)) = 7.3.3
Provides: bundled(npm(@grafana/ui)) = 7.3.3
Provides: bundled(npm(@types/blueimp-md5)) = 2.7.0
Provides: bundled(npm(@types/d3-selection)) = 1.4.3
Provides: bundled(npm(@types/enzyme)) = 3.10.5
Provides: bundled(npm(@types/enzyme-adapter-react-16)) = 1.0.6
Provides: bundled(npm(@types/expect-puppeteer)) = 3.3.1
Provides: bundled(npm(@types/jest)) = 24.0.13
Provides: bundled(npm(@types/jest-environment-puppeteer)) = 4.4.0
Provides: bundled(npm(@types/lodash)) = 4.14.165
Provides: bundled(npm(@types/memoize-one)) = 5.1.2
Provides: bundled(npm(@types/react-autosuggest)) = 9.3.14
Provides: bundled(npm(@types/react-redux)) = 7.1.11
Provides: bundled(npm(@types/redux)) = 3.6.0
Provides: bundled(npm(@types/redux-persist)) = 4.3.1
Provides: bundled(npm(@types/redux-persist-transform-filter)) = 0.0.4
Provides: bundled(npm(babel-plugin-remove-object-properties)) = 1.0.2
Provides: bundled(npm(blueimp-md5)) = 2.18.0
Provides: bundled(npm(core-js)) = 1.2.7
Provides: bundled(npm(d3-flame-graph)) = 3.1.1
Provides: bundled(npm(d3-selection)) = 1.4.1
Provides: bundled(npm(emotion)) = 10.0.27
Provides: bundled(npm(enzyme)) = 3.11.0
Provides: bundled(npm(enzyme-adapter-react-16)) = 1.15.5
Provides: bundled(npm(eslint-plugin-prettier)) = 3.1.4
Provides: bundled(npm(jest)) = 25.5.4
Provides: bundled(npm(jest-date-mock)) = 1.0.8
Provides: bundled(npm(jest-puppeteer)) = 4.4.0
Provides: bundled(npm(lodash)) = 4.17.19
Provides: bundled(npm(loglevel)) = 1.7.0
Provides: bundled(npm(loglevel-plugin-prefix)) = 0.8.4
Provides: bundled(npm(memoize-one)) = 4.1.0
Provides: bundled(npm(monaco-editor)) = 0.20.0
Provides: bundled(npm(monaco-editor-webpack-plugin)) = 2.0.0
Provides: bundled(npm(prettier)) = 1.19.1
Provides: bundled(npm(prettier-plugin-organize-imports)) = 1.1.1
Provides: bundled(npm(puppeteer)) = 5.5.0
Provides: bundled(npm(react-autosuggest)) = 10.0.3
Provides: bundled(npm(react-monaco-editor)) = 0.36.0
Provides: bundled(npm(react-redux)) = 7.2.2
Provides: bundled(npm(react-use)) = 15.3.4
Provides: bundled(npm(redux)) = 3.7.2
Provides: bundled(npm(redux-persist)) = 4.10.2
Provides: bundled(npm(redux-thunk)) = 2.3.0
Provides: bundled(npm(ts-jest)) = 26.3.0
Provides: bundled(npm(utility-types)) = 3.10.0
%description
@ -79,30 +122,73 @@ scalable time series from pmseries(1) and Redis, live PCP metrics and
bpftrace scripts from pmdabpftrace(1), as well as several dashboards.
%prep
%setup -q
%setup -q -a 1
%setup -q -T -D -b 0
%setup -q -T -D -b 1
%if %{compile_frontend} == 0
%setup -q -T -D -b 2
%endif
%patch1 -p1
# Set up Go build subdir and links
mkdir -p %{_builddir}/src/github.com/performancecopilot
ln -s %{_builddir}/%{name}-%{version} \
%{_builddir}/src/github.com/performancecopilot/grafana-pcp
%build
rm -rf dist
./node_modules/webpack/bin/webpack.js --config webpack.config.prod.js
# Build frontend datasources
%if %{compile_frontend}
make dist-dashboards dist-frontend
# webpack/copy-webpack-plugin sometimes outputs files with mode = 666 due to reasons unknown (race condition/umask issue afaics)
chmod -Rf a+rX,u+w,g-w,o-w dist
chmod -R g-w,o-w dist
%endif
# Build backend datasource
cd %{_builddir}/src/github.com/performancecopilot/grafana-pcp
export GOPATH=%{_builddir}
%gobuild -o dist/datasources/redis/pcp_redis_datasource_$(go env GOOS)_$(go env GOARCH) ./pkg
%check
./node_modules/jest/bin/jest.js --silent
%install
install -d -m 755 %{buildroot}/%{install_dir}
cp -a dist/* %{buildroot}/%{install_dir}
%postun
# uninstall of old package
%systemd_postun_with_restart grafana-server.service
%posttrans
# install or upgrade of new package
if [ -x /usr/bin/systemctl ]; then
/usr/bin/systemctl try-restart grafana-server.service || :
fi
%check
# Test frontend datasources
%if %{compile_frontend}
yarn test
%endif
# Test backend datasource
cd %{_builddir}/src/github.com/performancecopilot/grafana-pcp
export GOPATH=%{_builddir}
%gotest ./pkg/...
%files
%{install_dir}
%license LICENSE NOTICE
%doc README.md
%changelog
* Thu Nov 26 2020 Andreas Gerstmayr <agerstmayr@redhat.com> 3.0.0-1
- update to 3.0.0 tagged upstream community sources, see CHANGELOG
- bundle golang dependencies and (optionally) node.js dependencies
* Tue Jul 28 2020 Fedora Release Engineering <releng@fedoraproject.org> - 2.0.2-2
- Rebuilt for https://fedoraproject.org/wiki/Fedora_33_Mass_Rebuild

44
list_bundled_nodejs_packages.py Executable file
View File

@ -0,0 +1,44 @@
#!/usr/bin/env python3
import sys
import json
import re
from packaging import version
def read_declared_pkgs(package_json_path):
with open(package_json_path) as f:
package_json = json.load(f)
return list(package_json['dependencies'].keys()) + list(package_json['devDependencies'].keys())
def read_installed_pkgs(yarn_lock_path):
with open(yarn_lock_path) as f:
lockfile = f.read()
return re.findall(r'^"?' # can start with a "
r'(.+?)@.+(?:,.*)?:\n' # characters up to @
r' version "(.+)"', # and the version
lockfile, re.MULTILINE)
def list_provides(declared_pkgs, installed_pkgs):
for declared_pkg in declared_pkgs:
# there can be multiple versions installed of one package (transitive dependencies)
# but rpm doesn't support Provides: with a single package and multiple versions
# so let's declare the oldest version here
versions = [version.parse(pkg_version)
for pkg_name, pkg_version in installed_pkgs if pkg_name == declared_pkg]
oldest_version = sorted(versions)[0]
yield f"Provides: bundled(npm({declared_pkg})) = {oldest_version}"
if __name__ == "__main__":
if len(sys.argv) != 2:
print(f"usage: {sys.argv[0]} package-X.Y.Z/", file=sys.stdout)
sys.exit(1)
package_dir = sys.argv[1]
declared_pkgs = read_declared_pkgs(f"{package_dir}/package.json")
installed_pkgs = read_installed_pkgs(f"{package_dir}/yarn.lock")
provides = list_provides(declared_pkgs, installed_pkgs)
for provide in sorted(provides):
print(provide)

View File

@ -1,2 +1,3 @@
SHA512 (grafana-pcp-2.0.2.tar.gz) = 5a20bd3b6e5ce2d89ec131d4423ab46fd16b4b93949d80185e0b48b31067c6bcc6b89bbec7086912447c643bd7bed2d4625477adf994b4218d37eb08c731f304
SHA512 (grafana-pcp-deps-2.0.2.tar.xz) = eb52c58c7f3e8ed2dedf254342e126b00752552a690d0d1e2cef577846d59d5cb217b6504ca6bf0dbd4d32b8806c14392dde02bc6a7eff037ca322cd50ac701e
SHA512 (grafana-pcp-3.0.0.tar.gz) = 759dea2b958ebf3b9b1ce70675ee2cb44c71ea9c8a5548e036f18d3bc1a58438d3e40a331fab4c5f8414e2a797311fa5d4c3cf20b36d6d18e24db7f7b00388ba
SHA512 (grafana-pcp-vendor-3.0.0.tar.xz) = 1773775aa694c5d2136984c7406fd1df9d662dd64d042de2656aee460a84b463ef1dcb71ff1008e9c91478a8b7bdaabb6debaaea5e65770266b28ff85816039a
SHA512 (grafana-pcp-webpack-3.0.0.tar.gz) = d8423ee69532fac14f00f9bcdb4f7261fe54b1e6b397b76ae084502e4dd4b20abfc2420256e96032f1f85feda557c46fc191deeeaa88fc1fc0b97de0c9ba0c87