rpms/python-rpm-macros
6
0
Fork 0
Code Issues Pull Requests Releases Activity

Update of bundled compileall2 module to 0.7.0

Browse Source 
Adds the optional --hardlink-dupes flag for compileall2 for pyc deduplication

This is explained in https://discuss.python.org/t/3014
                 and https://github.com/fedora-python/compileall2/issues/16

This option is not yet used anywhere. That allows us to backport this to all
Fedoras but only use --hardlink-dupes on rawhide first.
This commit is contained in:
Miro Hrončok 2020-02-10 23:47:52 +01:00
parent e9f07b72aa
commit 456f3ecffb
2 changed files with 44 additions and 10 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

46
compileall2.py
View File

@ -19,6 +19,7 @@ import sys
import importlib.util import importlib.util
import py_compile import py_compile
import struct import struct
import filecmp
from functools import partial from functools import partial
from pathlib import Path from pathlib import Path
@ -86,7 +87,7 @@ def _walk_dir(dir, maxlevels, quiet=0):
def compile_dir(dir, maxlevels=None, ddir=None, force=False, def compile_dir(dir, maxlevels=None, ddir=None, force=False,
rx=None, quiet=0, legacy=False, optimize=-1, workers=1, rx=None, quiet=0, legacy=False, optimize=-1, workers=1,
invalidation_mode=None, stripdir=None, invalidation_mode=None, stripdir=None,
prependdir=None, limit_sl_dest=None): prependdir=None, limit_sl_dest=None, hardlink_dupes=False):
"""Byte-compile all modules in the given directory tree. """Byte-compile all modules in the given directory tree.
Arguments (only dir is required): Arguments (only dir is required):
@ -109,6 +110,7 @@ def compile_dir(dir, maxlevels=None, ddir=None, force=False,
after stripdir after stripdir
limit_sl_dest: ignore symlinks if they are pointing outside of limit_sl_dest: ignore symlinks if they are pointing outside of
the defined path the defined path
hardlink_dupes: hardlink duplicated pyc files
""" """
ProcessPoolExecutor = None ProcessPoolExecutor = None
if workers is not None: if workers is not None:
@ -144,14 +146,15 @@ def compile_dir(dir, maxlevels=None, ddir=None, force=False,
if not compile_file(file, ddir, force, rx, quiet, if not compile_file(file, ddir, force, rx, quiet,
legacy, optimize, invalidation_mode, legacy, optimize, invalidation_mode,
stripdir=stripdir, prependdir=prependdir, stripdir=stripdir, prependdir=prependdir,
limit_sl_dest=limit_sl_dest): limit_sl_dest=limit_sl_dest,
hardlink_dupes=hardlink_dupes):
success = False success = False
return success return success
def compile_file(fullname, ddir=None, force=False, rx=None, quiet=0, def compile_file(fullname, ddir=None, force=False, rx=None, quiet=0,
legacy=False, optimize=-1, legacy=False, optimize=-1,
invalidation_mode=None, stripdir=None, prependdir=None, invalidation_mode=None, stripdir=None, prependdir=None,
limit_sl_dest=None): limit_sl_dest=None, hardlink_dupes=False):
"""Byte-compile one file. """Byte-compile one file.
Arguments (only fullname is required): Arguments (only fullname is required):
@ -172,6 +175,7 @@ def compile_file(fullname, ddir=None, force=False, rx=None, quiet=0,
after stripdir after stripdir
limit_sl_dest: ignore symlinks if they are pointing outside of limit_sl_dest: ignore symlinks if they are pointing outside of
the defined path. the defined path.
hardlink_dupes: hardlink duplicated pyc files
""" """
if ddir is not None and (stripdir is not None or prependdir is not None): if ddir is not None and (stripdir is not None or prependdir is not None):
@ -212,6 +216,10 @@ def compile_file(fullname, ddir=None, force=False, rx=None, quiet=0,
if isinstance(optimize, int): if isinstance(optimize, int):
optimize = [optimize] optimize = [optimize]
if hardlink_dupes:
raise ValueError(("Hardlinking of duplicated bytecode makes sense "
"only for more than one optimization level."))
if rx is not None: if rx is not None:
mo = rx.search(fullname) mo = rx.search(fullname)
if mo: if mo:
@ -256,7 +264,8 @@ def compile_file(fullname, ddir=None, force=False, rx=None, quiet=0,
if not quiet: if not quiet:
print('Compiling {!r}...'.format(fullname)) print('Compiling {!r}...'.format(fullname))
try: try:
for opt_level, cfile in opt_cfiles.items(): for index, opt_level in enumerate(sorted(optimize)):
cfile = opt_cfiles[opt_level]
if PY37: if PY37:
ok = py_compile.compile(fullname, cfile, dfile, True, ok = py_compile.compile(fullname, cfile, dfile, True,
optimize=opt_level, optimize=opt_level,
@ -264,6 +273,18 @@ def compile_file(fullname, ddir=None, force=False, rx=None, quiet=0,
else: else:
ok = py_compile.compile(fullname, cfile, dfile, True, ok = py_compile.compile(fullname, cfile, dfile, True,
optimize=opt_level) optimize=opt_level)
if index > 0 and hardlink_dupes:
previous_cfile = opt_cfiles[optimize[index - 1]]
if previous_cfile == cfile and optimize[0] not in (1, 2):
# Python 3.4 has only one .pyo file for -O and -OO so
# we hardlink it only if there is a .pyc file
# with the same content
previous_cfile = opt_cfiles[optimize[0]]
if previous_cfile != cfile and filecmp.cmp(cfile, previous_cfile, shallow=False):
os.unlink(cfile)
os.link(previous_cfile, cfile)
except py_compile.PyCompileError as err: except py_compile.PyCompileError as err:
success = False success = False
if quiet >= 2: if quiet >= 2:
@ -379,11 +400,14 @@ def main():
parser.add_argument('-j', '--workers', default=1, parser.add_argument('-j', '--workers', default=1,
type=int, help='Run compileall concurrently') type=int, help='Run compileall concurrently')
parser.add_argument('-o', action='append', type=int, dest='opt_levels', parser.add_argument('-o', action='append', type=int, dest='opt_levels',
help=('Optimization levels to run compilation with.' help=('Optimization levels to run compilation with. '
'Default is -1 which uses optimization level of' 'Default is -1 which uses optimization level of '
'Python interpreter itself (specified by -O).')) 'Python interpreter itself (specified by -O).'))
parser.add_argument('-e', metavar='DIR', dest='limit_sl_dest', parser.add_argument('-e', metavar='DIR', dest='limit_sl_dest',
help='Ignore symlinks pointing outsite of the DIR') help='Ignore symlinks pointing outsite of the DIR')
parser.add_argument('--hardlink-dupes', action='store_true',
dest='hardlink_dupes',
help='Hardlink duplicated pyc files')
if PY37: if PY37:
invalidation_modes = [mode.name.lower().replace('_', '-') invalidation_modes = [mode.name.lower().replace('_', '-')
@ -413,6 +437,10 @@ def main():
if args.opt_levels is None: if args.opt_levels is None:
args.opt_levels = [-1] args.opt_levels = [-1]
if len(args.opt_levels) == 1 and args.hardlink_dupes:
parser.error(("Hardlinking of duplicated bytecode makes sense "
"only for more than one optimization level."))
if args.ddir is not None and ( if args.ddir is not None and (
args.stripdir is not None or args.prependdir is not None args.stripdir is not None or args.prependdir is not None
): ):
@ -449,7 +477,8 @@ def main():
stripdir=args.stripdir, stripdir=args.stripdir,
prependdir=args.prependdir, prependdir=args.prependdir,
optimize=args.opt_levels, optimize=args.opt_levels,
limit_sl_dest=args.limit_sl_dest): limit_sl_dest=args.limit_sl_dest,
hardlink_dupes=args.hardlink_dupes):
success = False success = False
else: else:
if not compile_dir(dest, maxlevels, args.ddir, if not compile_dir(dest, maxlevels, args.ddir,
@ -459,7 +488,8 @@ def main():
stripdir=args.stripdir, stripdir=args.stripdir,
prependdir=args.prependdir, prependdir=args.prependdir,
optimize=args.opt_levels, optimize=args.opt_levels,
limit_sl_dest=args.limit_sl_dest): limit_sl_dest=args.limit_sl_dest,
hardlink_dupes=args.hardlink_dupes):
success = False success = False
return success return success
else: else:

8
python-rpm-macros.spec
View File

@ -1,6 +1,6 @@
Name: python-rpm-macros Name: python-rpm-macros
Version: 3 Version: 3
Release: 53%{?dist} Release: 54%{?dist}
Summary: The unversioned Python RPM macros Summary: The unversioned Python RPM macros
# macros: MIT, compileall2.py: PSFv2 # macros: MIT, compileall2.py: PSFv2
@ -10,7 +10,7 @@ Source1: macros.python-srpm
Source2: macros.python2 Source2: macros.python2
Source3: macros.python3 Source3: macros.python3
Source4: macros.pybytecompile Source4: macros.pybytecompile
Source5: https://github.com/fedora-python/compileall2/raw/v0.5.0/compileall2.py Source5: https://github.com/fedora-python/compileall2/raw/v0.7.0/compileall2.py
BuildArch: noarch BuildArch: noarch
# For %%python3_pkgversion used in %%python_provide and compileall2.py # For %%python3_pkgversion used in %%python_provide and compileall2.py
@ -78,6 +78,10 @@ install -m 644 %{SOURCE5} \
%changelog %changelog
* Mon Feb 10 2020 Miro Hrončok <mhroncok@redhat.com> - 3-54
- Update of bundled compileall2 module to 0.7.0
Adds the optional --hardlink-dupes flag for compileall2 for pyc deduplication
* Thu Feb 06 2020 Miro Hrončok <mhroncok@redhat.com> - 3-53 * Thu Feb 06 2020 Miro Hrončok <mhroncok@redhat.com> - 3-53
- Define %%py(2|3)?_shbang_opts_nodash to be used with pathfix.py -a - Define %%py(2|3)?_shbang_opts_nodash to be used with pathfix.py -a