update patch handling and instructions

Makefile

@@ -8,10 +8,17 @@ SOURCE_TAR := $(NAME)-$(VERSION).tar.gz
 VENDOR_TAR := $(RPM_NAME)-vendor-$(VERSION)-$(RELEASE).tar.xz
 WEBPACK_TAR := $(RPM_NAME)-webpack-$(VERSION)-$(RELEASE).tar.gz
 
-ALL_PATCHES     := $(sort $(wildcard *.patch))
+# patches which must be applied before creating the vendor tarball, for example:
-VENDOR_PATCHES  := $(sort $(wildcard *.vendor.patch))
+# - changes in dependency versions
-COND_PATCHES    := $(sort $(wildcard *.cond.patch))
+# - changes in Go module imports (which affect the vendored Go modules)
-REGULAR_PATCHES := $(filter-out $(VENDOR_PATCHES) $(COND_PATCHES),$(ALL_PATCHES))
+PATCHES_PRE_VENDOR := \
+	005-remove-unused-dependencies.patch \
+	008-remove-unused-frontend-crypto.patch
+
+# patches which must be applied before creating the webpack, for example:
+# - changes in Node.js sources or vendored dependencies
+PATCHES_PRE_WEBPACK :=
+
 
 all: $(SOURCE_TAR) $(VENDOR_TAR) $(WEBPACK_TAR)
 
@@ -19,11 +26,12 @@ $(SOURCE_TAR):
 	spectool -g $(RPM_NAME).spec
 
 $(VENDOR_TAR): $(SOURCE_TAR)
+	# start with a clean state
 	rm -rf $(SOURCE_DIR)
 	tar xf $(SOURCE_TAR)
 
 	# Patches to apply before vendoring
-	for patch in $(REGULAR_PATCHES); do echo applying $$patch ...; patch -d $(SOURCE_DIR) -p1 --fuzz=0 < $$patch; done
+	for patch in $(PATCHES_PRE_VENDOR); do echo applying $$patch ...; patch -d $(SOURCE_DIR) -p1 --fuzz=0 < $$patch; done
 
 	# Go
 	cd $(SOURCE_DIR) && go mod vendor -v
@@ -46,15 +54,20 @@ $(VENDOR_TAR): $(SOURCE_TAR)
 	rm -r $(SOURCE_DIR)/node_modules/visjs-network/examples
 	./list_bundled_nodejs_packages.py $(SOURCE_DIR) >> $@.manifest
 
-	# Patches to apply after vendoring
-	for patch in $(VENDOR_PATCHES); do echo applying $$patch ...; patch -d $(SOURCE_DIR) -p1 --fuzz=0 < $$patch; done
-
 	# Create tarball
 	XZ_OPT=-9 time -p tar cJf $@ \
 		$(SOURCE_DIR)/vendor \
 		$$(find $(SOURCE_DIR) -type d -name "node_modules" -prune)
 
 $(WEBPACK_TAR): $(VENDOR_TAR)
+	# start with a clean state
+	rm -rf $(SOURCE_DIR)
+	tar xf $(SOURCE_TAR)
+	tar xf $(VENDOR_TAR)
+
+	# Patches to apply before creating the webpack
+	for patch in $(PATCHES_PRE_WEBPACK); do echo applying $$patch ...; patch -d $(SOURCE_DIR) -p1 --fuzz=0 < $$patch; done
+
 	cd $(SOURCE_DIR) && \
 		../build_frontend.sh
 

README.md

@@ -14,19 +14,26 @@ The grafana package
 * upload new source tarballs: `fedpkg new-sources *.tar.gz *.tar.xz`
 * commit new `sources` file
 
-## Backporting
+## Patches
 * create the patch
 * declare and apply (`%prep`) the patch in the specfile
 * if the patch affects Go or Node.js dependencies, or the webpack
+  * add the patch to `PATCHES_PRE_VENDOR` or `PATCHES_PRE_WEBPACK` in the Makefile
   * create new tarballs
-  * update the specfile with new tarball path and contents of the `.manifest` file
+  * update the specfile with new tarball name and contents of the `.manifest` file
 
-Note: the Makefile automatically applies patches before creating the tarballs
+### General guidelines
+* aim to apply all patches in the specfile
+* avoid rebuilding the tarballs
 
-## Patches
+Patches fall in several categories:
-* `*.patch`: regular patches applied to the source, applied in the Makefile before vendoring and in the specfile (e.g. updating dependencies)
+  * modify dependency versions
-* `*.vendor.patch`: patches applied to the vendor tarball (e.g. patching vendored sources before generating a webpack)
+  * modify both sources and vendored dependencies (e.g. CVEs)
-* `*.cond.patch`: conditionally applied patches in the specfile
+  * modify the Node.js source (i.e. affect the webpack)
+  * some patches are conditional (e.g. FIPS)
+
+Patches cannot be applied twice.
+It is not possible to unconditionally apply all patches in the Makefile, and great care must be taken to include the required patches at the correct stage of the build.
 
 ## Verification
 * compare the list of files with the upstream RPM at https://grafana.com/grafana/download

grafana.spec

@@ -85,11 +85,11 @@ Patch8:           008-remove-unused-frontend-crypto.patch
 # The Makefile removes a few files with crypto implementations
 # from the vendor tarball, which are not used in Grafana.
 # This patch removes all references to the deleted files.
-Patch9:           009-patch-unused-backend-crypto.vendor.patch
+Patch9:           009-patch-unused-backend-crypto.patch
 
 # This patch modifies the x/crypto/pbkdf2 function to use OpenSSL
 # if FIPS mode is enabled.
-Patch10:          010-fips.cond.patch
+Patch10:          010-fips.patch
 
 # Intersection of go_arches and nodejs_arches
 ExclusiveArch:    %{grafana_arches}
@@ -488,6 +488,7 @@ rm -r plugins-bundled
 %patch5 -p1
 %patch6 -p1
 %patch8 -p1
+%patch9 -p1
 %if %{enable_fips_mode}
 %patch10 -p1
 %endif

recreate_tarballs.sh

@@ -0,0 +1,17 @@
+#!/bin/bash -eu
+#
+# create vendor and webpack tarballs inside a container for reproducibility
+#
+
+cat <<EOF | podman build -t grafana-build -f - .
+FROM fedora:34
+
+RUN dnf install -y rpmdevtools time python3-packaging make golang nodejs yarnpkg
+
+RUN useradd builder
+USER builder
+WORKDIR /home/builder
+
+COPY Makefile grafana.spec *.patch build_frontend.sh list_bundled_nodejs_packages.py .
+RUN make
+EOF

run_container_build.sh

@@ -1,23 +0,0 @@
-#!/bin/bash -eu
-
-[ $# -lt 1 ] && echo "Usage: $0 fedora-version" && exit 1
-FEDORA_VERSION="$1"
-
-if [ -d deps ]; then
-    INSTALL_UNPUBLISHED_DEPENDENCIES=$'COPY deps/ /deps\nRUN cd /deps && dnf -y install *.rpm'
-else
-    INSTALL_UNPUBLISHED_DEPENDENCIES=""
-fi
-
-cat <<EOF | podman build -f - .
-FROM fedora:${FEDORA_VERSION}
-RUN dnf install -y rpkg
-RUN mkdir /grafana /deps
-
-${INSTALL_UNPUBLISHED_DEPENDENCIES}
-
-COPY grafana.spec *.patch grafana-*.tar.gz distro-defaults.ini Makefile create_webpack_manifest.py /grafana
-WORKDIR /grafana
-RUN dnf -y builddep grafana.spec
-RUN rpkg local
-EOF