Move Python related BuildRoot Policy scripts from redhat-rpm-config to python-srpm-macros

This allows us to maintain our own BuildRoot Policy scripts in an easier way.

This change needs to be coordinated with the addition of the files to python-srpm-macros.

redhat-rpm-config requires python-srpm-macros, so no significant change is expected for the packagers.

This also moves the Python BRPs to the end of the list which should be fine.

brp-fix-pyc-reproducibility

@@ -1,18 +0,0 @@
-#!/bin/bash -e
-
-# If using normal root, avoid changing anything.
-if [ -z "$RPM_BUILD_ROOT" -o "$RPM_BUILD_ROOT" = "/" ]; then
-  exit 0
-fi
-
-# Defined as %py_reproducible_pyc_path macro and passed here as
-# the first command-line argument
-path_to_fix=$1
-
-# First, check that the parser is available:
-if [ ! -x /usr/bin/marshalparser ]; then
-  echo "ERROR: If %py_reproducible_pyc_path is defined, you have to also BuildRequire: /usr/bin/marshalparser !"
-  exit 1
-fi
-
-find "$path_to_fix" -type f -name "*.pyc" | xargs /usr/bin/marshalparser --fix --overwrite

brp-python-bytecompile

@@ -1,141 +0,0 @@
-#!/bin/bash
-errors_terminate=$2
-
-# Usage of %_python_bytecompile_extra is not allowed anymore
-# See: https://fedoraproject.org/wiki/Changes/No_more_automagic_Python_bytecompilation_phase_3
-# Therefore $1 ($default_python) is not needed and is invoked with "" by default.
-# $default_python stays in the arguments for backward compatibility and $extra for the following check:
-extra=$3
-if [ 0$extra -eq 1 ]; then
-    echo -e "%_python_bytecompile_extra is discontinued, use %py_byte_compile instead.\nSee: https://fedoraproject.org/wiki/Changes/No_more_automagic_Python_bytecompilation_phase_3" >/dev/stderr
-    exit 1
-fi
-
-# If using normal root, avoid changing anything.
-if [ -z "$RPM_BUILD_ROOT" -o "$RPM_BUILD_ROOT" = "/" ]; then
-	exit 0
-fi
-
-# Figure out how deep we need to descend.  We could pick an insanely high
-# number and hope it's enough, but somewhere, somebody's sure to run into it.
-depth=`(find "$RPM_BUILD_ROOT" -type f -name "*.py" -print0 ; echo /) | \
-       xargs -0 -n 1 dirname | sed 's,[^/],,g' | sort -u | tail -n 1 | wc -c`
-if [ -z "$depth" -o "$depth" -le "1" ]; then
-	exit 0
-fi
-
-# This function now implements Python byte-compilation in three different ways:
-# Python >= 3.4 and < 3.9 uses a new module compileall2 - https://github.com/fedora-python/compileall2
-# Python < 3.4 (inc. Python 2) uses compileall module from stdlib with some hacks
-# When we drop support for Python 2, we'd be able to use all compileall2 features like:
-# - -s and -p options to manipulate with a path baked into pyc files instead of $real_libdir
-# - -o 0 -o 1 to produce multiple files in one run - each with a different optimization level - instead of $options
-# - removed useless $depth - both compileall and compileall2 are limited by sys.getrecursionlimit()
-# These changes will make this script much simpler
-# In Python >= 3.9, compileall2 was merged back to standard library (compileall) so we can use it directly again.
-function python_bytecompile()
-{
-    local options=$1
-    local python_binary=$2
-    local exclude=$3
-    local python_libdir="$4"
-    local depth=$5 # Not used for Python >= 3.4
-    local real_libdir=$6 # Not used for Python >= 3.4
-
-	python_version=$($python_binary -c "import sys; sys.stdout.write('{0.major}{0.minor}'.format(sys.version_info))")
-
-	#
-	# Python 3.9 and higher
-	#
-	if [ "$python_version" -ge 39 ]; then
-
-		[ ! -z $exclude ] && exclude="-x '$exclude'"
-		# -q disables verbose output
-		# -f forces the process to overwrite existing compiled files
-		# -x excludes paths defined by regex
-		# -e excludes symbolic links pointing outside the build root
-		# -x and -e together implements the same functionality as the Filter class below
-		# -s strips $RPM_BUILD_ROOT from the path
-		# -p prepends the leading slash to the path to make it absolute
-		$python_binary -B $options -m compileall -q -f $exclude -s "$RPM_BUILD_ROOT" -p / -e "$RPM_BUILD_ROOT" "$python_libdir"
-
-	#
-	# Python 3.4 and higher
-	#
-	elif [ "$python_version" -ge 34 ]; then
-
-		[ ! -z $exclude ] && exclude="-x '$exclude'"
-		# /usr/lib/rpm/redhat/ contains compileall2 Python module
-		# -q disables verbose output
-		# -f forces the process to overwrite existing compiled files
-		# -x excludes paths defined by regex
-		# -e excludes symbolic links pointing outside the build root
-		# -x and -e together implements the same functionality as the Filter class below
-		# -s strips $RPM_BUILD_ROOT from the path
-		# -p prepends the leading slash to the path to make it absolute
-		PYTHONPATH=/usr/lib/rpm/redhat/ $python_binary -B $options -m compileall2 -q -f $exclude -s "$RPM_BUILD_ROOT" -p / -e "$RPM_BUILD_ROOT" "$python_libdir"
-	else
-#
-# Python 3.3 and lower (incl. Python 2)
-#
-
-cat << EOF | $python_binary $options
-import compileall, sys, os, re
-
-python_libdir = "$python_libdir"
-depth = $depth
-real_libdir = "$real_libdir"
-build_root = "$RPM_BUILD_ROOT"
-exclude = r"$exclude"
-
-class Filter:
-    def search(self, path):
-        ret = not os.path.realpath(path).startswith(build_root)
-        if exclude:
-            ret = ret or re.search(exclude, path)
-        return ret
-
-sys.exit(not compileall.compile_dir(python_libdir, depth, real_libdir, force=1, rx=Filter(), quiet=1))
-EOF
-
-fi
-}
-
-# .pyc/.pyo files embed a "magic" value, identifying the ABI version of Python
-# bytecode that they are for.
-#
-# The files below RPM_BUILD_ROOT could be targeting multiple versions of
-# python (e.g. a single build that emits several subpackages e.g. a
-# python26-foo subpackage, a python31-foo subpackage etc)
-#
-# Support this by assuming that below each /usr/lib/python$VERSION/, all
-# .pyc/.pyo files are to be compiled for /usr/bin/python$VERSION.
-#
-# For example, below /usr/lib/python2.6/, we're targeting /usr/bin/python2.6
-# and below /usr/lib/python3.1/, we're targeting /usr/bin/python3.1
-
-# Disable Python hash seed randomization
-# This should help with byte-compilation reproducibility: https://bugzilla.redhat.com/show_bug.cgi?id=1686078
-export PYTHONHASHSEED=0
-
-shopt -s nullglob
-find "$RPM_BUILD_ROOT" -type d -print0|grep -z -E "/(usr|app)/lib(64)?/python[0-9]\.[0-9]+$" | while read -d "" python_libdir;
-do
-	python_binary=$(basename "$python_libdir")
-	real_libdir=${python_libdir/$RPM_BUILD_ROOT/}
-	echo "Bytecompiling .py files below $python_libdir using $python_binary"
-
-	# Generate normal (.pyc) byte-compiled files.
-	python_bytecompile "" "$python_binary" "" "$python_libdir" "$depth" "$real_libdir"
-	if [ $? -ne 0 -a 0$errors_terminate -ne 0 ]; then
-		# One or more of the files had a syntax error
-		exit 1
-	fi
-
-	# Generate optimized (.pyo) byte-compiled files.
-	python_bytecompile "-O" "$python_binary" "" "$python_libdir" "$depth" "$real_libdir"
-	if [ $? -ne 0 -a 0$errors_terminate -ne 0 ]; then
-		# One or more of the files had a syntax error
-		exit 1
-	fi
-done

macros

@@ -244,8 +244,6 @@ print(result)
 %__brp_strip_lto /usr/lib/rpm/redhat/brp-strip-lto %{__strip}
 %__brp_strip_comment_note /usr/lib/rpm/brp-strip-comment-note %{__strip} %{__objdump}
 %__brp_strip_static_archive /usr/lib/rpm/brp-strip-static-archive %{__strip}
-%__brp_python_bytecompile /usr/lib/rpm/redhat/brp-python-bytecompile "" "%{?_python_bytecompile_errors_terminate_build}" "%{?_python_bytecompile_extra}"
-%__brp_fix_pyc_reproducibility /usr/lib/rpm/redhat/brp-fix-pyc-reproducibility
 %__brp_check_rpaths /usr/lib/rpm/check-rpaths
 # __brp_mangle_shebangs_exclude - shebangs to exclude
 # __brp_mangle_shebangs_exclude_file - file from which to get shebangs to exclude
@@ -255,6 +253,10 @@ print(result)
 
 %__brp_llvm_compile_lto_elf /usr/lib/rpm/redhat/brp-llvm-compile-lto-elf %{build_cflags} %{build_ldflags}
 
+# note: %%__os_install_post_python is defined in python-srpm-macros and contains several policies
+# redhat-rpm-config maintainers, don't remove it from %%__os_install_post unless coordinating the change with Python maintainers
+# packagers, don't undefine the entire macro, see the individual macros in /usr/lib/rpm/macros.d/macros.python-srpm
+
 %__os_install_post    \
     %[ "%{toolchain}" == "clang" ? "%{?__brp_llvm_compile_lto_elf}" : "%{nil}" ] \
     %{?__brp_ldconfig} \
@@ -265,11 +267,9 @@ print(result)
     } \
     %{?__brp_strip_lto} \
     %{?__brp_strip_static_archive} \
-    %{?py_auto_byte_compile:%{?__brp_python_bytecompile}} \
-    %{?py_reproducible_pyc_path:%{?__brp_fix_pyc_reproducibility} "%{py_reproducible_pyc_path}"} \
     %{?__brp_check_rpaths} \
-    %{?__brp_python_hardlink} \
     %{?__brp_mangle_shebangs} \
+    %{?__os_install_post_python} \
 %{nil}
 
 %__spec_install_post\
@@ -286,16 +286,6 @@ print(result)
 # Should missing buildids terminate a build?
 %_missing_build_ids_terminate_build    1
 
-#
-## Automatically compile python files
-%py_auto_byte_compile 1
-
-#
-## Should python bytecompilation errors terminate a build?
-%_python_bytecompile_errors_terminate_build 1
-## Should python bytecompilation compile outisde python specific directories?
-%_python_bytecompile_extra 0
-
 # Use SHA-256 for FILEDIGESTS instead of default MD5
 %_source_filedigest_algorithm 8
 %_binary_filedigest_algorithm 8

redhat-rpm-config.spec

@@ -6,7 +6,7 @@
 
 Summary: Red Hat specific rpm configuration files
 Name: redhat-rpm-config
-Version: 189
+Version: 190
 Release: 1%{?dist}
 # No version specified.
 License: GPL+
@@ -49,14 +49,6 @@ Source155: macros.fedora-misc
 # and an echo when the mangling happens
 Source201: brp-mangle-shebangs
 
-# this comes from rpm itself
-# however, now we can do Fedora changes within
-Source202: brp-python-bytecompile
-
-# for fixing pyc files reproducibility with marshalparser
-# https://github.com/fedora-python/marshalparser
-Source203: brp-fix-pyc-reproducibility
-
 # for converting llvm LTO bitcode objects into ELF
 Source204: brp-llvm-compile-lto-elf
 
@@ -111,6 +103,7 @@ Requires: ocaml-srpm-macros
 Requires: openblas-srpm-macros
 Requires: perl-srpm-macros
 # ↓ Provides compileall2 Python module
+# TODO: require >= 3.10-6 in the next iteration to ensure Python BRPs are always present
 Requires: python-srpm-macros >= 3-46
 Requires: qt5-srpm-macros
 Requires: rust-srpm-macros
@@ -221,6 +214,9 @@ install -p -m 644 -t %{buildroot}%{_rpmluadir}/fedora/srpm forge.lua
 %{_rpmconfigdir}/macros.d/macros.kmp
 
 %changelog
+* Wed Jun 30 2021 Miro Hrončok <mhroncok@redhat.com> - 190-1
+- Move Python related BuildRoot Policy scripts from redhat-rpm-config to python-srpm-macros
+
 * Mon Jun 28 2021 Ben Burton <bab@debian.org> - 189-1
 - Adapt macros and BRP scripts for %%topdir with spaces
 - Fixes rhbz#1947416