rpms/python-rpm-macros
6
0
Fork 0
Code Issues Pull Requests Releases Activity

Add a script to clamp source mtimes, invoke it from the bytecompilation BRP/macro

Browse Source 
https://fedoraproject.org/wiki/Changes/ReproducibleBuildsClampMtimes
This commit is contained in:
Miro Hrončok 2022-12-15 21:06:17 +01:00
parent e4baf5ab7e
commit 77912c744d
4 changed files with 189 additions and 1 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

15
brp-python-bytecompile
View File

@ -16,6 +16,16 @@ if [ -z "$RPM_BUILD_ROOT" -o "$RPM_BUILD_ROOT" = "/" ]; then
exit 0
fi
# This function clamps the source mtime, see https://fedoraproject.org/wiki/Changes/ReproducibleBuildsClampMtimes
function python_clamp_source_mtime()
{
local _=$1
local python_binary=$2
local _=$3
local python_libdir="$4"
PYTHONPATH=/usr/lib/rpm/redhat/ $python_binary -B -m clamp_source_mtime -q "$python_libdir"
}
# This function now implements Python byte-compilation in three different ways:
# Python >= 3.4 and < 3.9 uses a new module compileall2 - https://github.com/fedora-python/compileall2
# In Python >= 3.9, compileall2 was merged back to standard library (compileall) so we can use it directly again.
@ -123,6 +133,11 @@ do
echo "Bytecompiling .py files below $python_libdir using $python_binary"
# Generate normal (.pyc) byte-compiled files.
python_clamp_source_mtime "" "$python_binary" "" "$python_libdir"
if [ $? -ne 0 -a 0$errors_terminate -ne 0 ]; then
# One or more of the files had inaccessible mtime
exit 1
fi
python_bytecompile "" "$python_binary" "" "$python_libdir"
if [ $? -ne 0 -a 0$errors_terminate -ne 0 ]; then
# One or more of the files had a syntax error

161
clamp_source_mtime.py Normal file
View File

@ -0,0 +1,161 @@
"""Module/script to clamp the mtimes of all .py files to $SOURCE_DATE_EPOCH
When called as a script with arguments, this compiles the directories
given as arguments recursively.
If upstream is interested, this can be later integrated to the compileall module
as an additional option (e.g. --clamp-source-mtime).
License:
This has been derived from the Python's compileall module
and it follows Python licensing. For more info see: https://www.python.org/psf/license/
"""
import os
import sys
# Python 3.6 and higher
PY36 = sys.version_info[0:2] >= (3, 6)
__all__ = ["clamp_dir", "clamp_file"]
def _walk_dir(dir, maxlevels, quiet=0):
if PY36 and quiet < 2 and isinstance(dir, os.PathLike):
dir = os.fspath(dir)
else:
dir = str(dir)
if not quiet:
print('Listing {!r}...'.format(dir))
try:
names = os.listdir(dir)
except OSError:
if quiet < 2:
print("Can't list {!r}".format(dir))
names = []
names.sort()
for name in names:
if name == '__pycache__':
continue
fullname = os.path.join(dir, name)
if not os.path.isdir(fullname):
yield fullname
elif (maxlevels > 0 and name != os.curdir and name != os.pardir and
os.path.isdir(fullname) and not os.path.islink(fullname)):
yield from _walk_dir(fullname, maxlevels=maxlevels - 1,
quiet=quiet)
def clamp_dir(dir, source_date_epoch, quiet=0):
"""Clamp the mtime of all modules in the given directory tree.
Arguments:
dir: the directory to byte-compile
source_date_epoch: integer parsed from $SOURCE_DATE_EPOCH
quiet: full output with False or 0, errors only with 1,
no output with 2
"""
maxlevels = sys.getrecursionlimit()
files = _walk_dir(dir, quiet=quiet, maxlevels=maxlevels)
success = True
for file in files:
if not clamp_file(file, source_date_epoch, quiet=quiet):
success = False
return success
def clamp_file(fullname, source_date_epoch, quiet=0):
"""Clamp the mtime of one file.
Arguments:
fullname: the file to byte-compile
source_date_epoch: integer parsed from $SOURCE_DATE_EPOCH
quiet: full output with False or 0, errors only with 1,
no output with 2
"""
if PY36 and quiet < 2 and isinstance(fullname, os.PathLike):
fullname = os.fspath(fullname)
else:
fullname = str(fullname)
name = os.path.basename(fullname)
if os.path.isfile(fullname) and not os.path.islink(fullname):
if name[-3:] == '.py':
try:
mtime = int(os.stat(fullname).st_mtime)
atime = int(os.stat(fullname).st_atime)
except OSError as e:
if quiet >= 2:
return False
elif quiet:
print('*** Error checking mtime of {!r}...'.format(fullname))
else:
print('*** ', end='')
print(e.__class__.__name__ + ':', e)
return False
if mtime > source_date_epoch:
if not quiet:
print('Clamping mtime of {!r}'.format(fullname))
try:
os.utime(fullname, (atime, source_date_epoch))
except OSError as e:
if quiet >= 2:
return False
elif quiet:
print('*** Error clamping mtime of {!r}...'.format(fullname))
else:
print('*** ', end='')
print(e.__class__.__name__ + ':', e)
return False
return True
def main():
"""Script main program."""
import argparse
source_date_epoch = os.getenv('SOURCE_DATE_EPOCH')
if not source_date_epoch:
print("Not clamping source mtimes, $SOURCE_DATE_EPOCH not set")
return True # This is a success, no action needed
try:
source_date_epoch = int(source_date_epoch)
except ValueError:
print("$SOURCE_DATE_EPOCH must be an integer")
return False
parser = argparse.ArgumentParser(
description='Clamp .py source mtime to $SOURCE_DATE_EPOCH.')
parser.add_argument('-q', action='count', dest='quiet', default=0,
help='output only error messages; -qq will suppress '
'the error messages as well.')
parser.add_argument('clamp_dest', metavar='FILE|DIR', nargs='+',
help=('zero or more file and directory paths '
'to clamp'))
args = parser.parse_args()
clamp_dests = args.clamp_dest
success = True
try:
for dest in clamp_dests:
if os.path.isfile(dest):
if not clamp_file(dest, quiet=args.quiet,
source_date_epoch=source_date_epoch):
success = False
else:
if not clamp_dir(dest, quiet=args.quiet,
source_date_epoch=source_date_epoch):
success = False
return success
except KeyboardInterrupt:
if args.quiet < 2:
print("\n[interrupted]")
return False
return True
if __name__ == '__main__':
exit_status = int(not main())
sys.exit(exit_status)

8
macros.pybytecompile
View File

@ -17,6 +17,12 @@
# Python 3.11+ no longer needs this: https://github.com/python/cpython/pull/27926 (but we support older Pythons as well)
%py_byte_compile()\
clamp_source_mtime () {\
python_binary="%{__env_unset_source_date_epoch_if_not_clamp_mtime} %1"\
bytecode_compilation_path="%2"\
PYTHONPATH="%{_rpmconfigdir}/redhat" $python_binary -s -B -m clamp_source_mtime $bytecode_compilation_path \
}\
\
py2_byte_compile () {\
python_binary="%{__env_unset_source_date_epoch_if_not_clamp_mtime} PYTHONHASHSEED=0 %1"\
bytecode_compilation_path="%2"\
@ -45,6 +51,8 @@ py39_byte_compile () {\
\
# Path to intepreter should not contain any arguments \
[[ "%1" =~ " -" ]] && echo "ERROR py_byte_compile: Path to interpreter should not contain any arguments" >&2 && exit 1 \
# First, clamp source mtime https://fedoraproject.org/wiki/Changes/ReproducibleBuildsClampMtimes \
clamp_source_mtime "%1" "%2"; \
# Get version without a dot (36 instead of 3.6), bash doesn't compare floats well \
python_version=$(%1 -c "import sys; sys.stdout.write('{0.major}{0.minor}'.format(sys.version_info))") \
# compileall2 is an enhanced fork of stdlib compileall module for Python >= 3.4 \

6
python-rpm-macros.spec
View File

@ -18,6 +18,7 @@ Source301: https://github.com/fedora-python/compileall2/raw/v%{compileall2_
Source302: import_all_modules.py
%global pathfix_version 1.0.0
Source303: https://github.com/fedora-python/pathfix/raw/v%{pathfix_version}/pathfix.py
Source304: clamp_source_mtime.py
# BRP scripts
# This one is from redhat-rpm-config < 190
@ -35,7 +36,7 @@ Source403: brp-fix-pyc-reproducibility
# macros and lua: MIT
# import_all_modules.py: MIT
# compileall2.py: PSFv2
# compileall2.py, clamp_source_mtime.py: PSFv2
# pathfix.py: PSFv2
# brp scripts: GPLv2+
License: MIT and Python and GPLv2+
@ -120,6 +121,7 @@ install -p -m 644 -t %{buildroot}%{_rpmluadir}/fedora/srpm python.lua
mkdir -p %{buildroot}%{_rpmconfigdir}/redhat
install -m 644 compileall2.py %{buildroot}%{_rpmconfigdir}/redhat/
install -m 644 clamp_source_mtime.py %{buildroot}%{_rpmconfigdir}/redhat/
install -m 644 import_all_modules.py %{buildroot}%{_rpmconfigdir}/redhat/
install -m 644 pathfix.py %{buildroot}%{_rpmconfigdir}/redhat/
install -m 755 brp-* %{buildroot}%{_rpmconfigdir}/redhat/
@ -150,6 +152,7 @@ grep -E '^#[^%%]*%%[^%%]' %{buildroot}%{rpmmacrodir}/macros.* && exit 1 || true
%files -n python-srpm-macros
%{rpmmacrodir}/macros.python-srpm
%{_rpmconfigdir}/redhat/compileall2.py
%{_rpmconfigdir}/redhat/clamp_source_mtime.py
%{_rpmconfigdir}/redhat/brp-python-bytecompile
%{_rpmconfigdir}/redhat/brp-python-hardlink
%{_rpmconfigdir}/redhat/brp-fix-pyc-reproducibility
@ -163,6 +166,7 @@ grep -E '^#[^%%]*%%[^%%]' %{buildroot}%{rpmmacrodir}/macros.* && exit 1 || true
* Mon Dec 19 2022 Miro Hrončok <mhroncok@redhat.com> - 3.11-7
- Bytecompilation: Unset $SOURCE_DATE_EPOCH when %%clamp_mtime_to_source_date_epoch is not set
- Bytecompilation: Pass --invalidation-mode=timestamp to compileall (on Python 3.7+)
- Bytecompilation: Clamp source mtime: https://fedoraproject.org/wiki/Changes/ReproducibleBuildsClampMtimes
* Sun Nov 13 2022 Miro Hrončok <mhroncok@redhat.com> - 3.11-6
- Set PYTEST_XDIST_AUTO_NUM_WORKERS=%%{_smp_build_ncpus} from %%pytest