Add the import_all_modules_py3.11.py file for the python3.11-rpm-macros subpackage

Resolves: rhbz#2207631

- The script is copied veratim from the current Fedora version.
- The script is namespaced with the `_py3_11` version, because it's only used for Python 3.11 at this point, and when we're adding the Python 3.12 stack, we don't want to name clash with it. The script might be different at that point, and also we can't depend on Python 3.11 macros from Python 3.12 macros.
- The script is excluded from bytecompilation, because it's run but never imported, and we don't want to pollute the `/usr/lib/rpm/redhat/` directory.

check-pyc-timestamps.py

@@ -16,6 +16,7 @@ LEVELS = (None, 1, 2)
 # list of globs of test and other files that we expect not to have bytecode
 not_compiled = [
     '/usr/bin/*',
+    '/usr/lib/rpm/redhat/*',
     '*/test/bad_coding.py',
     '*/test/bad_coding2.py',
     '*/test/badsyntax_*.py',

import_all_modules_py3_11.py

@@ -0,0 +1,171 @@
+'''Script to perform import of each module given to %%py_check_import
+'''
+import argparse
+import importlib
+import fnmatch
+import os
+import re
+import site
+import sys
+
+from contextlib import contextmanager
+from pathlib import Path
+
+
+def read_modules_files(file_paths):
+    '''Read module names from the files (modules must be newline separated).
+
+    Return the module names list or, if no files were provided, an empty list.
+    '''
+
+    if not file_paths:
+        return []
+
+    modules = []
+    for file in file_paths:
+        file_contents = file.read_text()
+        modules.extend(file_contents.split())
+    return modules
+
+
+def read_modules_from_cli(argv):
+    '''Read module names from command-line arguments (space or comma separated).
+
+    Return the module names list.
+    '''
+
+    if not argv:
+        return []
+
+    # %%py3_check_import allows to separate module list with comma or whitespace,
+    # we need to unify the output to a list of particular elements
+    modules_as_str = ' '.join(argv)
+    modules = re.split(r'[\s,]+', modules_as_str)
+    # Because of shell expansion in some less typical cases it may happen
+    # that a trailing space will occur at the end of the list.
+    # Remove the empty items from the list before passing it further
+    modules = [m for m in modules if m]
+    return modules
+
+
+def filter_top_level_modules_only(modules):
+    '''Filter out entries with nested modules (containing dot) ie. 'foo.bar'.
+
+    Return the list of top-level modules.
+    '''
+
+    return [module for module in modules if '.' not in module]
+
+
+def any_match(text, globs):
+    '''Return True if any of given globs fnmatchcase's the given text.'''
+
+    return any(fnmatch.fnmatchcase(text, g) for g in globs)
+
+
+def exclude_unwanted_module_globs(globs, modules):
+    '''Filter out entries which match the either of the globs given as argv.
+
+    Return the list of filtered modules.
+    '''
+
+    return [m for m in modules if not any_match(m, globs)]
+
+
+def read_modules_from_all_args(args):
+    '''Return a joined list of modules from all given command-line arguments.
+    '''
+
+    modules = read_modules_files(args.filename)
+    modules.extend(read_modules_from_cli(args.modules))
+    if args.exclude:
+        modules = exclude_unwanted_module_globs(args.exclude, modules)
+
+    if args.top_level:
+        modules = filter_top_level_modules_only(modules)
+
+    # Error when someone accidentally managed to filter out everything
+    if len(modules) == 0:
+        raise ValueError('No modules to check were left')
+
+    return modules
+
+
+def import_modules(modules):
+    '''Procedure to perform import check for each module name from the given list of modules.
+    '''
+
+    for module in modules:
+        print('Check import:', module, file=sys.stderr)
+        importlib.import_module(module)
+
+
+def argparser():
+    parser = argparse.ArgumentParser(
+        description='Generate list of all importable modules for import check.'
+    )
+    parser.add_argument(
+        'modules', nargs='*',
+        help=('Add modules to check the import (space or comma separated).'),
+    )
+    parser.add_argument(
+        '-f', '--filename', action='append', type=Path,
+        help='Add importable module names list from file.',
+    )
+    parser.add_argument(
+        '-t', '--top-level', action='store_true',
+        help='Check only top-level modules.',
+    )
+    parser.add_argument(
+        '-e', '--exclude', action='append',
+        help='Provide modules globs to be excluded from the check.',
+    )
+    return parser
+
+
+@contextmanager
+def remove_unwanteds_from_sys_path():
+    '''Remove cwd and this script's parent from sys.path for the import test.
+    Bring the original contents back after import is done (or failed)
+    '''
+
+    cwd_absolute = Path.cwd().absolute()
+    this_file_parent = Path(__file__).parent.absolute()
+    old_sys_path = list(sys.path)
+    for path in old_sys_path:
+        if Path(path).absolute() in (cwd_absolute, this_file_parent):
+            sys.path.remove(path)
+    try:
+        yield
+    finally:
+        sys.path = old_sys_path
+
+
+def addsitedirs_from_environ():
+    '''Load directories from the _PYTHONSITE environment variable (separated by :)
+    and load the ones already present in sys.path via site.addsitedir()
+    to handle .pth files in them.
+
+    This is needed to properly import old-style namespace packages with nspkg.pth files.
+    See https://bugzilla.redhat.com/2018551 for a more detailed rationale.'''
+    for path in os.getenv('_PYTHONSITE', '').split(':'):
+        if path in sys.path:
+            site.addsitedir(path)
+
+
+def main(argv=None):
+
+    cli_args = argparser().parse_args(argv)
+
+    if not cli_args.modules and not cli_args.filename:
+        raise ValueError('No modules to check were provided')
+
+    modules = read_modules_from_all_args(cli_args)
+
+    with remove_unwanteds_from_sys_path():
+        addsitedirs_from_environ()
+        import_modules(modules)
+
+
+if __name__ == '__main__':
+    main()

macros.python3.11

@@ -61,7 +61,7 @@
   if rpm.expand("%{?py3_shebang_flags}") ~= "" then
     command = command .. "-%{py3_shebang_flags}"
   end
-  command = command .. " %{_rpmconfigdir}/redhat/import_all_modules.py "
+  command = command .. " %{_rpmconfigdir}/redhat/import_all_modules_py3_11.py "
   -- handle multiline arguments correctly, see https://bugzilla.redhat.com/2018809
   local args=rpm.expand('%{?**}'):gsub("[%s\\\\]*%s+", " ")
   print(command .. args)

python3.11.spec

@@ -20,7 +20,7 @@ URL: https://www.python.org/
 #global prerel ...
 %global upstream_version %{general_version}%{?prerel}
 Version: %{general_version}%{?prerel:~%{prerel}}
-Release: 3%{?dist}
+Release: 4%{?dist}
 License: Python
 
 
@@ -252,7 +252,10 @@ Source0: %{url}ftp/python/%{general_version}/Python-%{upstream_version}.tar.xz
 Source1: %{url}ftp/python/%{general_version}/Python-%{upstream_version}.tar.xz.asc
 # The release manager for Python 3.11 is pablogsal
 Source2: https://keybase.io/pablogsal/pgp_keys.asc
+
+# Sources for the python3.11-rpm-macros
 Source3: macros.python3.11
+Source4: import_all_modules_py3_11.py
 
 # A simple script to check timestamps of bytecode files
 # Run in check section with Python that is currently being built
@@ -1108,6 +1111,10 @@ mkdir -p %{buildroot}%{rpmmacrodir}/
 install -m 644 %{SOURCE3} \
     %{buildroot}/%{rpmmacrodir}/
 
+# Add a script that is being used by python3.11-rpm-macros
+mkdir -p %{buildroot}%{_rpmconfigdir}/redhat
+install -m 644 %{SOURCE4} %{buildroot}%{_rpmconfigdir}/redhat/
+
 # All ghost files controlled by alternatives need to exist for the files
 # section check to succeed
 # - Don't list /usr/bin/python as a ghost file so `yum install /usr/bin/python`
@@ -1306,6 +1313,7 @@ fi
 
 %files -n %{pkgname}-rpm-macros
 %{rpmmacrodir}/macros.python%{pybasever}
+%{_rpmconfigdir}/redhat/import_all_modules_py3_11.py
 
 %files -n %{pkgname}
 %doc README.rst
@@ -1809,6 +1817,10 @@ fi
 # ======================================================
 
 %changelog
+* Thu Aug 10 2023 Tomas Orsava <torsava@redhat.com> - 3.11.4-4
+- Add the import_all_modules_py3_11.py file for the python3.11-rpm-macros subpackage
+Resolves: rhbz#2207631
+
 * Wed Aug 09 2023 Petr Viktorin <pviktori@redhat.com> - 3.11.4-3
 - Fix symlink handling in the fix for CVE-2023-24329
 Resolves: rhbz#263261