diff --git a/brp-python-bytecompile b/brp-python-bytecompile index 347e789..3092bdf 100644 --- a/brp-python-bytecompile +++ b/brp-python-bytecompile @@ -16,6 +16,16 @@ if [ -z "$RPM_BUILD_ROOT" -o "$RPM_BUILD_ROOT" = "/" ]; then exit 0 fi +# This function clamps the source mtime, see https://fedoraproject.org/wiki/Changes/ReproducibleBuildsClampMtimes +function python_clamp_source_mtime() +{ + local _=$1 + local python_binary=$2 + local _=$3 + local python_libdir="$4" + PYTHONPATH=/usr/lib/rpm/redhat/ $python_binary -B -m clamp_source_mtime -q "$python_libdir" +} + # This function now implements Python byte-compilation in three different ways: # Python >= 3.4 and < 3.9 uses a new module compileall2 - https://github.com/fedora-python/compileall2 # In Python >= 3.9, compileall2 was merged back to standard library (compileall) so we can use it directly again. @@ -123,6 +133,11 @@ do echo "Bytecompiling .py files below $python_libdir using $python_binary" # Generate normal (.pyc) byte-compiled files. + python_clamp_source_mtime "" "$python_binary" "" "$python_libdir" + if [ $? -ne 0 -a 0$errors_terminate -ne 0 ]; then + # One or more of the files had inaccessible mtime + exit 1 + fi python_bytecompile "" "$python_binary" "" "$python_libdir" if [ $? -ne 0 -a 0$errors_terminate -ne 0 ]; then # One or more of the files had a syntax error diff --git a/clamp_source_mtime.py b/clamp_source_mtime.py new file mode 100644 index 0000000..7349ea3 --- /dev/null +++ b/clamp_source_mtime.py @@ -0,0 +1,161 @@ +"""Module/script to clamp the mtimes of all .py files to $SOURCE_DATE_EPOCH + +When called as a script with arguments, this compiles the directories +given as arguments recursively. + +If upstream is interested, this can be later integrated to the compileall module +as an additional option (e.g. --clamp-source-mtime). + +License: +This has been derived from the Python's compileall module +and it follows Python licensing. For more info see: https://www.python.org/psf/license/ +""" +import os +import sys + +# Python 3.6 and higher +PY36 = sys.version_info[0:2] >= (3, 6) + +__all__ = ["clamp_dir", "clamp_file"] + + +def _walk_dir(dir, maxlevels, quiet=0): + if PY36 and quiet < 2 and isinstance(dir, os.PathLike): + dir = os.fspath(dir) + else: + dir = str(dir) + if not quiet: + print('Listing {!r}...'.format(dir)) + try: + names = os.listdir(dir) + except OSError: + if quiet < 2: + print("Can't list {!r}".format(dir)) + names = [] + names.sort() + for name in names: + if name == '__pycache__': + continue + fullname = os.path.join(dir, name) + if not os.path.isdir(fullname): + yield fullname + elif (maxlevels > 0 and name != os.curdir and name != os.pardir and + os.path.isdir(fullname) and not os.path.islink(fullname)): + yield from _walk_dir(fullname, maxlevels=maxlevels - 1, + quiet=quiet) + + +def clamp_dir(dir, source_date_epoch, quiet=0): + """Clamp the mtime of all modules in the given directory tree. + + Arguments: + + dir: the directory to byte-compile + source_date_epoch: integer parsed from $SOURCE_DATE_EPOCH + quiet: full output with False or 0, errors only with 1, + no output with 2 + """ + maxlevels = sys.getrecursionlimit() + files = _walk_dir(dir, quiet=quiet, maxlevels=maxlevels) + success = True + for file in files: + if not clamp_file(file, source_date_epoch, quiet=quiet): + success = False + return success + + +def clamp_file(fullname, source_date_epoch, quiet=0): + """Clamp the mtime of one file. + + Arguments: + + fullname: the file to byte-compile + source_date_epoch: integer parsed from $SOURCE_DATE_EPOCH + quiet: full output with False or 0, errors only with 1, + no output with 2 + """ + if PY36 and quiet < 2 and isinstance(fullname, os.PathLike): + fullname = os.fspath(fullname) + else: + fullname = str(fullname) + name = os.path.basename(fullname) + + if os.path.isfile(fullname) and not os.path.islink(fullname): + if name[-3:] == '.py': + try: + mtime = int(os.stat(fullname).st_mtime) + atime = int(os.stat(fullname).st_atime) + except OSError as e: + if quiet >= 2: + return False + elif quiet: + print('*** Error checking mtime of {!r}...'.format(fullname)) + else: + print('*** ', end='') + print(e.__class__.__name__ + ':', e) + return False + if mtime > source_date_epoch: + if not quiet: + print('Clamping mtime of {!r}'.format(fullname)) + try: + os.utime(fullname, (atime, source_date_epoch)) + except OSError as e: + if quiet >= 2: + return False + elif quiet: + print('*** Error clamping mtime of {!r}...'.format(fullname)) + else: + print('*** ', end='') + print(e.__class__.__name__ + ':', e) + return False + return True + + +def main(): + """Script main program.""" + import argparse + + source_date_epoch = os.getenv('SOURCE_DATE_EPOCH') + if not source_date_epoch: + print("Not clamping source mtimes, $SOURCE_DATE_EPOCH not set") + return True # This is a success, no action needed + try: + source_date_epoch = int(source_date_epoch) + except ValueError: + print("$SOURCE_DATE_EPOCH must be an integer") + return False + + parser = argparse.ArgumentParser( + description='Clamp .py source mtime to $SOURCE_DATE_EPOCH.') + parser.add_argument('-q', action='count', dest='quiet', default=0, + help='output only error messages; -qq will suppress ' + 'the error messages as well.') + parser.add_argument('clamp_dest', metavar='FILE|DIR', nargs='+', + help=('zero or more file and directory paths ' + 'to clamp')) + + args = parser.parse_args() + clamp_dests = args.clamp_dest + + success = True + try: + for dest in clamp_dests: + if os.path.isfile(dest): + if not clamp_file(dest, quiet=args.quiet, + source_date_epoch=source_date_epoch): + success = False + else: + if not clamp_dir(dest, quiet=args.quiet, + source_date_epoch=source_date_epoch): + success = False + return success + except KeyboardInterrupt: + if args.quiet < 2: + print("\n[interrupted]") + return False + return True + + +if __name__ == '__main__': + exit_status = int(not main()) + sys.exit(exit_status) diff --git a/macros.pybytecompile b/macros.pybytecompile index 9c9da85..8c9fbee 100644 --- a/macros.pybytecompile +++ b/macros.pybytecompile @@ -17,6 +17,12 @@ # Python 3.11+ no longer needs this: https://github.com/python/cpython/pull/27926 (but we support older Pythons as well) %py_byte_compile()\ +clamp_source_mtime () {\ + python_binary="%{__env_unset_source_date_epoch_if_not_clamp_mtime} %1"\ + bytecode_compilation_path="%2"\ + PYTHONPATH="%{_rpmconfigdir}/redhat" $python_binary -s -B -m clamp_source_mtime $bytecode_compilation_path \ +}\ +\ py2_byte_compile () {\ python_binary="%{__env_unset_source_date_epoch_if_not_clamp_mtime} PYTHONHASHSEED=0 %1"\ bytecode_compilation_path="%2"\ @@ -45,6 +51,8 @@ py39_byte_compile () {\ \ # Path to intepreter should not contain any arguments \ [[ "%1" =~ " -" ]] && echo "ERROR py_byte_compile: Path to interpreter should not contain any arguments" >&2 && exit 1 \ +# First, clamp source mtime https://fedoraproject.org/wiki/Changes/ReproducibleBuildsClampMtimes \ +clamp_source_mtime "%1" "%2"; \ # Get version without a dot (36 instead of 3.6), bash doesn't compare floats well \ python_version=$(%1 -c "import sys; sys.stdout.write('{0.major}{0.minor}'.format(sys.version_info))") \ # compileall2 is an enhanced fork of stdlib compileall module for Python >= 3.4 \ diff --git a/python-rpm-macros.spec b/python-rpm-macros.spec index 8ca30d6..d9daf49 100644 --- a/python-rpm-macros.spec +++ b/python-rpm-macros.spec @@ -18,6 +18,7 @@ Source301: https://github.com/fedora-python/compileall2/raw/v%{compileall2_ Source302: import_all_modules.py %global pathfix_version 1.0.0 Source303: https://github.com/fedora-python/pathfix/raw/v%{pathfix_version}/pathfix.py +Source304: clamp_source_mtime.py # BRP scripts # This one is from redhat-rpm-config < 190 @@ -35,7 +36,7 @@ Source403: brp-fix-pyc-reproducibility # macros and lua: MIT # import_all_modules.py: MIT -# compileall2.py: PSFv2 +# compileall2.py, clamp_source_mtime.py: PSFv2 # pathfix.py: PSFv2 # brp scripts: GPLv2+ License: MIT and Python and GPLv2+ @@ -120,6 +121,7 @@ install -p -m 644 -t %{buildroot}%{_rpmluadir}/fedora/srpm python.lua mkdir -p %{buildroot}%{_rpmconfigdir}/redhat install -m 644 compileall2.py %{buildroot}%{_rpmconfigdir}/redhat/ +install -m 644 clamp_source_mtime.py %{buildroot}%{_rpmconfigdir}/redhat/ install -m 644 import_all_modules.py %{buildroot}%{_rpmconfigdir}/redhat/ install -m 644 pathfix.py %{buildroot}%{_rpmconfigdir}/redhat/ install -m 755 brp-* %{buildroot}%{_rpmconfigdir}/redhat/ @@ -150,6 +152,7 @@ grep -E '^#[^%%]*%%[^%%]' %{buildroot}%{rpmmacrodir}/macros.* && exit 1 || true %files -n python-srpm-macros %{rpmmacrodir}/macros.python-srpm %{_rpmconfigdir}/redhat/compileall2.py +%{_rpmconfigdir}/redhat/clamp_source_mtime.py %{_rpmconfigdir}/redhat/brp-python-bytecompile %{_rpmconfigdir}/redhat/brp-python-hardlink %{_rpmconfigdir}/redhat/brp-fix-pyc-reproducibility @@ -163,6 +166,7 @@ grep -E '^#[^%%]*%%[^%%]' %{buildroot}%{rpmmacrodir}/macros.* && exit 1 || true * Mon Dec 19 2022 Miro Hrončok - 3.11-7 - Bytecompilation: Unset $SOURCE_DATE_EPOCH when %%clamp_mtime_to_source_date_epoch is not set - Bytecompilation: Pass --invalidation-mode=timestamp to compileall (on Python 3.7+) +- Bytecompilation: Clamp source mtime: https://fedoraproject.org/wiki/Changes/ReproducibleBuildsClampMtimes * Sun Nov 13 2022 Miro Hrončok - 3.11-6 - Set PYTEST_XDIST_AUTO_NUM_WORKERS=%%{_smp_build_ncpus} from %%pytest