diff --git a/.gitignore b/.gitignore index 96bc697..a2498d1 100644 --- a/.gitignore +++ b/.gitignore @@ -1,4 +1,5 @@ /grafana-pcp-*.tar.gz -/grafana-pcp-deps-*.tar.xz -/grafana-pcp-*.src.rpm -/results_grafana-pcp/ +/grafana-pcp-*.tar.xz +/grafana-pcp-*.tar.xz.manifest +/grafana-pcp-*/ +*.rpm diff --git a/Makefile b/Makefile new file mode 100644 index 0000000..6fffbaa --- /dev/null +++ b/Makefile @@ -0,0 +1,45 @@ +all: grafana-pcp-$(VER).tar.gz \ + grafana-pcp-vendor-$(VER).tar.xz \ + grafana-pcp-webpack-$(VER).tar.gz + +grafana-pcp-$(VER).tar.gz grafana-pcp-$(VER)/: + wget https://github.com/performancecopilot/grafana-pcp/archive/v$(VER)/grafana-pcp-$(VER).tar.gz + rm -rf grafana-pcp-$(VER) + tar xfz grafana-pcp-$(VER).tar.gz + cd grafana-pcp-$(VER) && shopt -s nullglob && \ + for patch in ../*.patch; do patch -p1 < $$patch; done + +grafana-pcp-vendor-$(VER).tar.xz: grafana-pcp-$(VER)/ + # Go + cd grafana-pcp-$(VER) && go mod vendor -v + awk '$$2~/^v/ && $$4 != "indirect" {print "Provides: bundled(golang(" $$1 ")) = " substr($$2, 2)}' grafana-pcp-$(VER)/go.mod | \ + sed -E 's/=(.*)-(.*)-(.*)/=\1-\2.\3/g' > $@.manifest + + # Node.js + cd grafana-pcp-$(VER) && yarn install --pure-lockfile + # Remove files with licensing issues + find grafana-pcp-$(VER) -type d -name 'node-notifier' -prune -exec rm -r {} \; + find grafana-pcp-$(VER) -name '*.exe' -delete + # Remove not required packages + rm -r grafana-pcp-$(VER)/node_modules/puppeteer + ./list_bundled_nodejs_packages.py grafana-pcp-$(VER)/ >> $@.manifest + + # Jsonnet + cd grafana-pcp-$(VER) && jb --jsonnetpkg-home=vendor_jsonnet install + + # Create tarball + XZ_OPT=-9 tar cfJ $@ \ + grafana-pcp-$(VER)/vendor \ + $$(find grafana-pcp-$(VER) -type d -name "node_modules" -prune) \ + grafana-pcp-$(VER)/vendor_jsonnet + +grafana-pcp-webpack-$(VER).tar.gz: grafana-pcp-$(VER)/ + cd grafana-pcp-$(VER) && \ + yarn install --pure-lockfile && \ + make dist-dashboards dist-frontend && \ + chmod -R g-w,o-w dist + + tar cfz $@ grafana-pcp-$(VER)/dist + +clean: + rm -rf *.tar.gz *.tar.xz *.manifest *.rpm grafana-pcp-*/ diff --git a/README.md b/README.md index 377c1e9..3c78cea 100644 --- a/README.md +++ b/README.md @@ -1,20 +1,22 @@ # grafana-pcp - The grafana-pcp package -## Build instructions -``` -VER=2.0.2 -spectool -g grafana-pcp.spec -./create_dependency_bundle.sh grafana-pcp-$VER.tar.gz grafana-pcp-deps-$VER.tar.xz -./check_npm_dependencies.py grafana-pcp.spec grafana-pcp-$VER.tar.gz grafana-pcp-deps-$VER.tar.xz -fedpkg new-sources grafana-pcp-$VER.tar.gz grafana-pcp-deps-$VER.tar.xz +## Upgrade instructions +(replace X.Y.Z with the new grafana-pcp version) -fedpkg local -fedpkg lint -fedpkg mockbuild -fedpkg scratch-build --srpm +* update `Version`, `Release` and `%changelog` in the specfile +* create bundles and manifest: `VER=X.Y.Z make clean all` +* update specfile with contents of the `.manifest` file +* run local build: `rpkg local` +* run rpm linter: `rpkg lint -r grafana-pcp.rpmlintrc` +* run a scratch build: `fedpkg scratch-build --srpm` +* upload new source tarballs: `fedpkg new-sources *.tar.gz *.tar.xz` -fedpkg build -fedpkg update -``` +## Backporting +* create the patch +* declare and apply (`%prep`) the patch in the specfile +* if the patch affects Go or Node.js dependencies, or the webpack + * create new tarballs and rename them to `grafana-pcp-...-X.Y.Z-R.tar.gz` + * update the specfile with new tarball path and contents of the `.manifest` file + +Note: the Makefile automatically applies all patches before creating the tarballs diff --git a/check_npm_dependencies.py b/check_npm_dependencies.py deleted file mode 100755 index b10a762..0000000 --- a/check_npm_dependencies.py +++ /dev/null @@ -1,42 +0,0 @@ -#!/usr/bin/env python3 -import sys -import tarfile -import json -import os.path - -def read_spec_provides(spec_path): - with open(spec_path) as f: - for line in f: - if line.startswith('Provides:'): - yield line.strip() - -def read_node_deps(source_archive_path): - root_dir = os.path.basename(source_archive_path)[:-len('.tar.gz')] - with tarfile.open(source_archive_path) as tar: - f = tar.extractfile(f'{root_dir}/package.json') - package_json = json.load(f) - return list(package_json['devDependencies'].keys()) + list(package_json['dependencies'].keys()) - -def read_node_deps_versions(dep_bundle_path, dependencies): - with tarfile.open(dep_bundle_path) as tar: - for dependency in dependencies: - f = tar.extractfile(f'node_modules/{dependency}/package.json') - package_json = json.load(f) - yield f'Provides: bundled(nodejs-{package_json["name"]}) = {package_json["version"]}' - -if __name__ == '__main__': - if len(sys.argv) != 4: - print('usage: {} specfile source-archive dependency-bundle'.format(sys.argv[0])) - sys.exit(1) - - provides = list(read_spec_provides(sys.argv[1])) - dependencies = sorted(read_node_deps(sys.argv[2])) - actual_provides = list(read_node_deps_versions(sys.argv[3], dependencies)) - - if provides == actual_provides: - print("Dependencies are up-to-date with the specfile.") - sys.exit(0) - else: - print("Dependencies don't match, please update the specfile:") - print ('\n'.join(actual_provides)) - sys.exit(1) diff --git a/create_dependency_bundle.sh b/create_dependency_bundle.sh deleted file mode 100755 index fbcd787..0000000 --- a/create_dependency_bundle.sh +++ /dev/null @@ -1,37 +0,0 @@ -#!/bin/bash -eu - -SRC=$(readlink -f "${1:?Usage: $0 source destination}") -DEST=$(readlink -f "${2:?Usage: $0 source destination}") - -if [ -f "$DEST" ]; then - echo "File $DEST exists already." - exit 0 -fi -if [ "$#" -gt 2 ]; then - PATCHES=$(readlink -f "${@:3}") -else - PATCHES="" -fi - -pushd "$(mktemp -d)" - -echo Extracting sources... -tar xfz "$SRC" -cd grafana-pcp-* - -echo Applying patches... -for patch in $PATCHES -do - patch -p1 < $patch -done - -echo Installing dependencies... -yarn install - -echo Removing files with licensing issues... -rm -rf node_modules/node-notifier - -echo Compressing... -XZ_OPT=-9 tar cJf "$DEST" node_modules - -popd diff --git a/grafana-pcp.rpmlintrc b/grafana-pcp.rpmlintrc index 2aa6b59..d158ea8 100644 --- a/grafana-pcp.rpmlintrc +++ b/grafana-pcp.rpmlintrc @@ -3,5 +3,4 @@ addFilter("W: spelling-error %description -l en_US pmseries -> .*") addFilter("W: spelling-error %description -l en_US bpftrace -> .*") addFilter("W: spelling-error %description -l en_US pmdabpftrace -> .*") -addFilter("W: strange-permission create_dependency_bundle.sh 755") -addFilter("W: invalid-url Source1: grafana-pcp-deps-.*") +addFilter("W: strange-permission list_bundled_nodejs_packages.py 755") diff --git a/grafana-pcp.spec b/grafana-pcp.spec index 80adef4..67a89a6 100644 --- a/grafana-pcp.spec +++ b/grafana-pcp.spec @@ -1,24 +1,61 @@ +%global grafanapcp_arches %{lua: go_arches = {} + for arch in rpm.expand("%{go_arches}"):gmatch("%S+") do + go_arches[arch] = 1 + end + for arch in rpm.expand("%{nodejs_arches}"):gmatch("%S+") do + if go_arches[arch] then + print(arch .. " ") + end +end} + +# gobuild and gotest macros are defined in go-rpm-macros, which is not available on RHEL +# definitions lifted from Fedora 34 podman.spec +%if ! 0%{?gobuild:1} +%define gobuild(o:) GO111MODULE=off go build -buildmode pie -compiler gc -tags="rpm_crashtraceback ${BUILDTAGS:-}" -ldflags "${LDFLAGS:-} -B 0x$(head -c20 /dev/urandom|od -An -tx1|tr -d ' \\n') -extldflags '-Wl,-z,relro -Wl,-z,now -specs=/usr/lib/rpm/redhat/redhat-hardened-ld '" -a -v -x %{?**}; +%endif +%if ! 0%{?gotest:1} +%define gotest() GO111MODULE=off go test -buildmode pie -compiler gc -ldflags "${LDFLAGS:-} -extldflags '-Wl,-z,relro -Wl,-z,now -specs=/usr/lib/rpm/redhat/redhat-hardened-ld '" %{?**}; +%endif + +# Specify if the frontend and dashboards will be compiled as part of the build or are attached +# as a webpack tarball (in case of an unsuitable nodejs or jsonnet version on the build system) +%define compile_frontend 0 + Name: grafana-pcp -Version: 2.0.2 -Release: 2%{?dist} +Version: 3.0.1 +Release: 1%{?dist} Summary: Performance Co-Pilot Grafana Plugin - -%global github https://github.com/performancecopilot/grafana-pcp -%global install_dir %{_sharedstatedir}/grafana/plugins/grafana-pcp - -BuildArch: noarch -ExclusiveArch: %{nodejs_arches} - License: ASL 2.0 -URL: %{github} +URL: https://github.com/performancecopilot/grafana-pcp -Source0: %{github}/archive/v%{version}/%{name}-%{version}.tar.gz -Source1: grafana-pcp-deps-%{version}.tar.xz -Source2: create_dependency_bundle.sh +Source0: https://github.com/performancecopilot/grafana-pcp/archive/v%{version}/%{name}-%{version}.tar.gz +Source1: grafana-pcp-vendor-%{version}.tar.xz +%if %{compile_frontend} == 0 +# Source2 contains the precompiled frontend and dashboards +Source2: grafana-pcp-webpack-%{version}.tar.gz +%endif +Source3: Makefile +Source4: list_bundled_nodejs_packages.py -BuildRequires: nodejs -Requires: grafana >= 6.6.0 -Suggests: pcp >= 5.0.0 +# Intersection of go_arches and nodejs_arches +ExclusiveArch: %{grafanapcp_arches} + +BuildRequires: systemd-rpm-macros, golang, go-srpm-macros +%if 0%{?fedora} >= 31 +BuildRequires: go-rpm-macros +%endif +%if %{compile_frontend} +BuildRequires: nodejs >= 1:12, nodejs < 1:13, yarnpkg, golang-github-google-jsonnet +%endif + +# omit golang debugsource, see BZ 995136 and related +%global dwz_low_mem_die_limit 0 +%global _debugsource_template %{nil} + +%global install_dir %{_sharedstatedir}/grafana/plugins/performancecopilot-pcp-app + +Requires: grafana >= 7.3.6 +Suggests: pcp >= 5.2.2 Suggests: redis >= 5.0.0 Suggests: bpftrace >= 0.9.2 @@ -29,48 +66,61 @@ Obsoletes: pcp-webapp-grafana <= 4.3.4 Obsoletes: pcp-webapp-graphite <= 4.3.4 Obsoletes: pcp-webapp-vector <= 4.3.4 -# Bundled npm packages -Provides: bundled(nodejs-@babel/cli) = 7.8.4 -Provides: bundled(nodejs-@babel/core) = 7.8.4 -Provides: bundled(nodejs-@babel/preset-env) = 7.8.4 -Provides: bundled(nodejs-@babel/preset-react) = 7.8.3 -Provides: bundled(nodejs-@babel/preset-typescript) = 7.8.3 -Provides: bundled(nodejs-@grafana/data) = 6.6.0 -Provides: bundled(nodejs-@grafana/ui) = 6.6.0 -Provides: bundled(nodejs-@types/benchmark) = 1.0.31 -Provides: bundled(nodejs-@types/d3) = 5.7.2 -Provides: bundled(nodejs-@types/grafana) = 4.6.3 -Provides: bundled(nodejs-@types/jest) = 24.9.1 -Provides: bundled(nodejs-@types/lodash) = 4.14.149 -Provides: bundled(nodejs-babel-jest) = 24.9.0 -Provides: bundled(nodejs-babel-loader) = 8.0.6 -Provides: bundled(nodejs-babel-plugin-angularjs-annotate) = 0.10.0 -Provides: bundled(nodejs-benchmark) = 2.1.4 -Provides: bundled(nodejs-clean-webpack-plugin) = 0.1.19 -Provides: bundled(nodejs-copy-webpack-plugin) = 5.1.1 -Provides: bundled(nodejs-core-js) = 3.6.4 -Provides: bundled(nodejs-css-loader) = 1.0.1 -Provides: bundled(nodejs-d3-flame-graph) = 2.1.9 -Provides: bundled(nodejs-d3-selection) = 1.4.1 -Provides: bundled(nodejs-expr-eval) = 1.2.3 -Provides: bundled(nodejs-jest) = 24.9.0 -Provides: bundled(nodejs-jest-date-mock) = 1.0.8 -Provides: bundled(nodejs-jsdom) = 9.12.0 -Provides: bundled(nodejs-lodash) = 4.17.15 -Provides: bundled(nodejs-memoize-one) = 5.1.1 -Provides: bundled(nodejs-mocha) = 6.2.2 -Provides: bundled(nodejs-prunk) = 1.3.1 -Provides: bundled(nodejs-q) = 1.5.1 -Provides: bundled(nodejs-regenerator-runtime) = 0.12.1 -Provides: bundled(nodejs-request) = 2.88.0 -Provides: bundled(nodejs-style-loader) = 0.22.1 -Provides: bundled(nodejs-ts-jest) = 24.3.0 -Provides: bundled(nodejs-ts-loader) = 4.5.0 -Provides: bundled(nodejs-tslint) = 5.20.1 -Provides: bundled(nodejs-tslint-config-airbnb) = 5.11.2 -Provides: bundled(nodejs-typescript) = 3.7.5 -Provides: bundled(nodejs-webpack) = 4.41.5 -Provides: bundled(nodejs-webpack-cli) = 3.3.10 +# vendored golang and node.js build dependencies +# this is for security purposes, if nodejs-foo ever needs an update, +# affected packages can be easily identified. +# Note: generated by the Makefile (see README.md) +Provides: bundled(golang(github.com/grafana/grafana-plugin-sdk-go)) = 0.79.0 +Provides: bundled(golang(github.com/smartystreets/goconvey)) = 1.6.4 +Provides: bundled(npm(@babel/plugin-transform-modules-commonjs)) = 7.12.1 +Provides: bundled(npm(@grafana/data)) = 7.3.6 +Provides: bundled(npm(@grafana/runtime)) = 7.3.6 +Provides: bundled(npm(@grafana/toolkit)) = 7.3.6 +Provides: bundled(npm(@grafana/ui)) = 7.3.6 +Provides: bundled(npm(@types/blueimp-md5)) = 2.7.0 +Provides: bundled(npm(@types/d3-selection)) = 1.4.3 +Provides: bundled(npm(@types/enzyme)) = 3.10.5 +Provides: bundled(npm(@types/enzyme-adapter-react-16)) = 1.0.6 +Provides: bundled(npm(@types/expect-puppeteer)) = 3.3.1 +Provides: bundled(npm(@types/jest)) = 24.0.13 +Provides: bundled(npm(@types/jest-environment-puppeteer)) = 4.4.1 +Provides: bundled(npm(@types/lodash)) = 4.14.165 +Provides: bundled(npm(@types/memoize-one)) = 5.1.2 +Provides: bundled(npm(@types/react-autosuggest)) = 9.3.14 +Provides: bundled(npm(@types/react-redux)) = 7.1.12 +Provides: bundled(npm(@types/redux)) = 3.6.0 +Provides: bundled(npm(@types/redux-persist)) = 4.3.1 +Provides: bundled(npm(@types/redux-persist-transform-filter)) = 0.0.4 +Provides: bundled(npm(babel-plugin-remove-object-properties)) = 1.0.2 +Provides: bundled(npm(blueimp-md5)) = 2.18.0 +Provides: bundled(npm(core-js)) = 1.2.7 +Provides: bundled(npm(d3-flame-graph)) = 3.1.1 +Provides: bundled(npm(d3-selection)) = 1.4.1 +Provides: bundled(npm(emotion)) = 10.0.27 +Provides: bundled(npm(enzyme)) = 3.11.0 +Provides: bundled(npm(enzyme-adapter-react-16)) = 1.15.5 +Provides: bundled(npm(eslint-plugin-prettier)) = 3.1.4 +Provides: bundled(npm(jest)) = 25.5.4 +Provides: bundled(npm(jest-date-mock)) = 1.0.8 +Provides: bundled(npm(jest-puppeteer)) = 4.4.0 +Provides: bundled(npm(lodash)) = 4.17.19 +Provides: bundled(npm(loglevel)) = 1.7.1 +Provides: bundled(npm(loglevel-plugin-prefix)) = 0.8.4 +Provides: bundled(npm(memoize-one)) = 4.1.0 +Provides: bundled(npm(monaco-editor)) = 0.20.0 +Provides: bundled(npm(monaco-editor-webpack-plugin)) = 1.9.0 +Provides: bundled(npm(prettier)) = 1.19.1 +Provides: bundled(npm(prettier-plugin-organize-imports)) = 1.1.1 +Provides: bundled(npm(puppeteer)) = 5.5.0 +Provides: bundled(npm(react-autosuggest)) = 10.0.4 +Provides: bundled(npm(react-monaco-editor)) = 0.36.0 +Provides: bundled(npm(react-redux)) = 7.2.2 +Provides: bundled(npm(react-use)) = 15.3.4 +Provides: bundled(npm(redux)) = 3.7.2 +Provides: bundled(npm(redux-persist)) = 4.10.2 +Provides: bundled(npm(redux-thunk)) = 2.3.0 +Provides: bundled(npm(ts-jest)) = 26.3.0 +Provides: bundled(npm(utility-types)) = 3.10.0 %description @@ -79,30 +129,74 @@ scalable time series from pmseries(1) and Redis, live PCP metrics and bpftrace scripts from pmdabpftrace(1), as well as several dashboards. %prep -%setup -q -%setup -q -a 1 +%setup -q -T -D -b 0 +%setup -q -T -D -b 1 +%if %{compile_frontend} == 0 +%setup -q -T -D -b 2 +%endif + +# Set up Go build subdir and links +mkdir -p %{_builddir}/src/github.com/performancecopilot +ln -s %{_builddir}/%{name}-%{version} \ + %{_builddir}/src/github.com/performancecopilot/grafana-pcp + %build -rm -rf dist -./node_modules/webpack/bin/webpack.js --config webpack.config.prod.js - +# Build frontend datasources +%if %{compile_frontend} +make dist-dashboards dist-frontend # webpack/copy-webpack-plugin sometimes outputs files with mode = 666 due to reasons unknown (race condition/umask issue afaics) -chmod -Rf a+rX,u+w,g-w,o-w dist +chmod -R g-w,o-w dist +%endif + +# Build backend datasource +cd %{_builddir}/src/github.com/performancecopilot/grafana-pcp +export GOPATH=%{_builddir} +%gobuild -o dist/datasources/redis/pcp_redis_datasource_$(go env GOOS)_$(go env GOARCH) ./pkg -%check -./node_modules/jest/bin/jest.js --silent %install install -d -m 755 %{buildroot}/%{install_dir} cp -a dist/* %{buildroot}/%{install_dir} +%postun +# uninstall of old package +%systemd_postun_with_restart grafana-server.service + +%posttrans +# install or upgrade of new package +if [ -x /usr/bin/systemctl ]; then + /usr/bin/systemctl try-restart grafana-server.service || : +fi + + +%check +# Test frontend datasources +%if %{compile_frontend} +yarn test +%endif + +# Test backend datasource +cd %{_builddir}/src/github.com/performancecopilot/grafana-pcp +export GOPATH=%{_builddir} +%gotest ./pkg/... + + %files %{install_dir} %license LICENSE NOTICE %doc README.md + %changelog +* Wed Dec 23 2020 Andreas Gerstmayr 3.0.1-1 +- update to 3.0.1 tagged upstream community sources, see CHANGELOG + +* Thu Nov 26 2020 Andreas Gerstmayr 3.0.0-1 +- update to 3.0.0 tagged upstream community sources, see CHANGELOG +- bundle golang dependencies and (optionally) node.js dependencies + * Tue Jul 28 2020 Fedora Release Engineering - 2.0.2-2 - Rebuilt for https://fedoraproject.org/wiki/Fedora_33_Mass_Rebuild diff --git a/list_bundled_nodejs_packages.py b/list_bundled_nodejs_packages.py new file mode 100755 index 0000000..a7c5e22 --- /dev/null +++ b/list_bundled_nodejs_packages.py @@ -0,0 +1,44 @@ +#!/usr/bin/env python3 +import sys +import json +import re +from packaging import version + + +def read_declared_pkgs(package_json_path): + with open(package_json_path) as f: + package_json = json.load(f) + return list(package_json['dependencies'].keys()) + list(package_json['devDependencies'].keys()) + + +def read_installed_pkgs(yarn_lock_path): + with open(yarn_lock_path) as f: + lockfile = f.read() + return re.findall(r'^"?' # can start with a " + r'(.+?)@.+(?:,.*)?:\n' # characters up to @ + r' version "(.+)"', # and the version + lockfile, re.MULTILINE) + + +def list_provides(declared_pkgs, installed_pkgs): + for declared_pkg in declared_pkgs: + # there can be multiple versions installed of one package (transitive dependencies) + # but rpm doesn't support Provides: with a single package and multiple versions + # so let's declare the oldest version here + versions = [version.parse(pkg_version) + for pkg_name, pkg_version in installed_pkgs if pkg_name == declared_pkg] + oldest_version = sorted(versions)[0] + yield f"Provides: bundled(npm({declared_pkg})) = {oldest_version}" + + +if __name__ == "__main__": + if len(sys.argv) != 2: + print(f"usage: {sys.argv[0]} package-X.Y.Z/", file=sys.stdout) + sys.exit(1) + + package_dir = sys.argv[1] + declared_pkgs = read_declared_pkgs(f"{package_dir}/package.json") + installed_pkgs = read_installed_pkgs(f"{package_dir}/yarn.lock") + provides = list_provides(declared_pkgs, installed_pkgs) + for provide in sorted(provides): + print(provide) diff --git a/sources b/sources index e0b5d8b..5b5f44f 100644 --- a/sources +++ b/sources @@ -1,2 +1,3 @@ -SHA512 (grafana-pcp-2.0.2.tar.gz) = 5a20bd3b6e5ce2d89ec131d4423ab46fd16b4b93949d80185e0b48b31067c6bcc6b89bbec7086912447c643bd7bed2d4625477adf994b4218d37eb08c731f304 -SHA512 (grafana-pcp-deps-2.0.2.tar.xz) = eb52c58c7f3e8ed2dedf254342e126b00752552a690d0d1e2cef577846d59d5cb217b6504ca6bf0dbd4d32b8806c14392dde02bc6a7eff037ca322cd50ac701e +SHA512 (grafana-pcp-3.0.1.tar.gz) = 0dbad3692358cfa2138836b2354aeb9737aa0974125b2fb022d6445ea270bd55f914f70825ab8c25259c6a14180c575dc37e7929f05d7f7ae5f83bab65bd4dca +SHA512 (grafana-pcp-webpack-3.0.1.tar.gz) = bc49972d07201877027aa429a80f4f8d938e8b59afb6d36f401c1ea44a117d77ba01ebb7f74028da5a21cf95b53908a80b12856f163d5a4704fc98047b8b0a65 +SHA512 (grafana-pcp-vendor-3.0.1.tar.xz) = 3e1c7bb256c413d66e6d18d4a418b5a07e5c825e07e83c1308c475278dcf70a6e9246f55bc46cb658c45dd049333a02481777ae6032e6dac70b7f051ab4a6220