import python-rpm-macros-3.9-48.el9

SOURCES/import_all_modules.py

@@ -0,0 +1,171 @@
+'''Script to perform import of each module given to %%py_check_import
+'''
+import argparse
+import importlib
+import fnmatch
+import os
+import re
+import site
+import sys
+
+from contextlib import contextmanager
+from pathlib import Path
+
+
+def read_modules_files(file_paths):
+    '''Read module names from the files (modules must be newline separated).
+
+    Return the module names list or, if no files were provided, an empty list.
+    '''
+
+    if not file_paths:
+        return []
+
+    modules = []
+    for file in file_paths:
+        file_contents = file.read_text()
+        modules.extend(file_contents.split())
+    return modules
+
+
+def read_modules_from_cli(argv):
+    '''Read module names from command-line arguments (space or comma separated).
+
+    Return the module names list.
+    '''
+
+    if not argv:
+        return []
+
+    # %%py3_check_import allows to separate module list with comma or whitespace,
+    # we need to unify the output to a list of particular elements
+    modules_as_str = ' '.join(argv)
+    modules = re.split(r'[\s,]+', modules_as_str)
+    # Because of shell expansion in some less typical cases it may happen
+    # that a trailing space will occur at the end of the list.
+    # Remove the empty items from the list before passing it further
+    modules = [m for m in modules if m]
+    return modules
+
+
+def filter_top_level_modules_only(modules):
+    '''Filter out entries with nested modules (containing dot) ie. 'foo.bar'.
+
+    Return the list of top-level modules.
+    '''
+
+    return [module for module in modules if '.' not in module]
+
+
+def any_match(text, globs):
+    '''Return True if any of given globs fnmatchcase's the given text.'''
+
+    return any(fnmatch.fnmatchcase(text, g) for g in globs)
+
+
+def exclude_unwanted_module_globs(globs, modules):
+    '''Filter out entries which match the either of the globs given as argv.
+
+    Return the list of filtered modules.
+    '''
+
+    return [m for m in modules if not any_match(m, globs)]
+
+
+def read_modules_from_all_args(args):
+    '''Return a joined list of modules from all given command-line arguments.
+    '''
+
+    modules = read_modules_files(args.filename)
+    modules.extend(read_modules_from_cli(args.modules))
+    if args.exclude:
+        modules = exclude_unwanted_module_globs(args.exclude, modules)
+
+    if args.top_level:
+        modules = filter_top_level_modules_only(modules)
+
+    # Error when someone accidentally managed to filter out everything
+    if len(modules) == 0:
+        raise ValueError('No modules to check were left')
+
+    return modules
+
+
+def import_modules(modules):
+    '''Procedure to perform import check for each module name from the given list of modules.
+    '''
+
+    for module in modules:
+        print('Check import:', module, file=sys.stderr)
+        importlib.import_module(module)
+
+
+def argparser():
+    parser = argparse.ArgumentParser(
+        description='Generate list of all importable modules for import check.'
+    )
+    parser.add_argument(
+        'modules', nargs='*',
+        help=('Add modules to check the import (space or comma separated).'),
+    )
+    parser.add_argument(
+        '-f', '--filename', action='append', type=Path,
+        help='Add importable module names list from file.',
+    )
+    parser.add_argument(
+        '-t', '--top-level', action='store_true',
+        help='Check only top-level modules.',
+    )
+    parser.add_argument(
+        '-e', '--exclude', action='append',
+        help='Provide modules globs to be excluded from the check.',
+    )
+    return parser
+
+
+@contextmanager
+def remove_unwanteds_from_sys_path():
+    '''Remove cwd and this script's parent from sys.path for the import test.
+    Bring the original contents back after import is done (or failed)
+    '''
+
+    cwd_absolute = Path.cwd().absolute()
+    this_file_parent = Path(__file__).parent.absolute()
+    old_sys_path = list(sys.path)
+    for path in old_sys_path:
+        if Path(path).absolute() in (cwd_absolute, this_file_parent):
+            sys.path.remove(path)
+    try:
+        yield
+    finally:
+        sys.path = old_sys_path
+
+
+def addsitedirs_from_environ():
+    '''Load directories from the _PYTHONSITE environment variable (separated by :)
+    and load the ones already present in sys.path via site.addsitedir()
+    to handle .pth files in them.
+
+    This is needed to properly import old-style namespace packages with nspkg.pth files.
+    See https://bugzilla.redhat.com/2018551 for a more detailed rationale.'''
+    for path in os.getenv('_PYTHONSITE', '').split(':'):
+        if path in sys.path:
+            site.addsitedir(path)
+
+
+def main(argv=None):
+
+    cli_args = argparser().parse_args(argv)
+
+    if not cli_args.modules and not cli_args.filename:
+        raise ValueError('No modules to check were provided')
+
+    modules = read_modules_from_all_args(cli_args)
+
+    with remove_unwanteds_from_sys_path():
+        addsitedirs_from_environ()
+        import_modules(modules)
+
+
+if __name__ == '__main__':
+    main()

SOURCES/macros.python

@@ -1,12 +1,14 @@
 # unversioned macros: used with user defined __python, no longer part of rpm >= 4.15
 # __python is defined to error by default in the srpm macros
-%python_sitelib %(%{__python} -Esc "import sysconfig; print(sysconfig.get_path('purelib'))")
+# nb: $RPM_BUILD_ROOT is not set when the macros are expanded (at spec parse time)
-%python_sitearch %(%{__python} -Esc "import sysconfig; print(sysconfig.get_path('platlib'))")
+#     so we set it manually (to empty string), making our Python prefer the correct install scheme location
-%python_version %(%{__python} -Esc "import sys; sys.stdout.write('{0.major}.{0.minor}'.format(sys.version_info))")
+%python_sitelib %(RPM_BUILD_ROOT= %{__python} -Esc "import sysconfig; print(sysconfig.get_path('purelib'))")
-%python_version_nodots %(%{__python} -Esc "import sys; sys.stdout.write('{0.major}{0.minor}'.format(sys.version_info))")
+%python_sitearch %(RPM_BUILD_ROOT= %{__python} -Esc "import sysconfig; print(sysconfig.get_path('platlib'))")
-%python_platform %(%{__python} -Esc "import sysconfig; print(sysconfig.get_platform())")
+%python_version %(RPM_BUILD_ROOT= %{__python} -Esc "import sys; sys.stdout.write('{0.major}.{0.minor}'.format(sys.version_info))")
-%python_platform_triplet %(%{__python} -Esc "import sysconfig; print(sysconfig.get_config_var('MULTIARCH'))")
+%python_version_nodots %(RPM_BUILD_ROOT= %{__python} -Esc "import sys; sys.stdout.write('{0.major}{0.minor}'.format(sys.version_info))")
-%python_ext_suffix %(%{__python} -Esc "import sysconfig; print(sysconfig.get_config_var('EXT_SUFFIX'))")
+%python_platform %(RPM_BUILD_ROOT= %{__python} -Esc "import sysconfig; print(sysconfig.get_platform())")
+%python_platform_triplet %(RPM_BUILD_ROOT= %{__python} -Esc "import sysconfig; print(sysconfig.get_config_var('MULTIARCH'))")
+%python_ext_suffix %(RPM_BUILD_ROOT= %{__python} -Esc "import sysconfig; print(sysconfig.get_config_var('EXT_SUFFIX'))")
 
 %py_setup setup.py
 %py_shbang_opts -s
@@ -67,16 +69,28 @@
 }
 
 # With $PATH and $PYTHONPATH set to the %%buildroot,
-# try to import the given Python module(s).
+# try to import the Python module(s) given as command-line args or read from file (-f).
+# Respect the custom values of %%py_shebang_flags or set nothing if it's undefined.
+# Filter and check import on only top-level modules using -t flag.
+# Exclude unwanted modules by passing their globs to -e option.
 # Useful as a smoke test in %%check when running tests is not feasible.
-# Use spaces or commas as separators.
+# Use spaces or commas as separators if providing list directly.
-%py_check_import() %{expand:\\\
+# Use newlines as separators if providing list in a file.
-  (cd %{_topdir} &&\\\
+%py_check_import(e:tf:) %{expand:\\\
   PATH="%{buildroot}%{_bindir}:$PATH"\\\
   PYTHONPATH="${PYTHONPATH:-%{buildroot}%{python_sitearch}:%{buildroot}%{python_sitelib}}"\\\
+  _PYTHONSITE="%{buildroot}%{python_sitearch}:%{buildroot}%{python_sitelib}"\\\
   PYTHONDONTWRITEBYTECODE=1\\\
-  %{__python} -c "import %{lua:local m=rpm.expand('%{?*}'):gsub('[%s,]+', ', ');print(m)}"
+  %{lua:
-  )
+  local command = "%{__python} "
+  if rpm.expand("%{?py_shebang_flags}") ~= "" then
+    command = command .. "-%{py_shebang_flags}"
+  end
+  command = command .. " %{_rpmconfigdir}/redhat/import_all_modules.py "
+  -- handle multiline arguments correctly, see https://bugzilla.redhat.com/2018809
+  local args=rpm.expand('%{?**}'):gsub("[%s\\\\]*%s+", " ")
+  print(command .. args)
+  }
 }
 
 %python_provide() %{lua:

SOURCES/macros.python-srpm

@@ -1,22 +1,3 @@
-# Define the Python interpreter paths in the SRPM macros so that
-# - they can be used in Build/Requires
-# - they can be used in non-Python packages where requiring pythonX-devel would
-#   be an overkill
-
-# use the underscored macros to redefine the behavior of %%python3_version etc.
-%__python2 /usr/bin/python2
-%__python3 /usr/bin/python3
-
-# use the non-underscored macros to refer to Python in spec, etc.
-%python2 %__python2
-%python3 %__python3
-
-# See https://fedoraproject.org/wiki/Changes/PythonMacroError
-%__python %{error:attempt to use unversioned python, define %%__python to %{__python2} or %{__python3} explicitly}
-
-# Users can use %%python only if they redefined %%__python (e.g. to %%__python3)
-%python %__python
-
 # There are multiple Python 3 versions packaged, but only one can be the "main" version
 # That means that it owns the "python3" namespace:
 #     - python3 package name
@@ -49,6 +30,25 @@
 # Alternatively, it can be overridden in spec (e.g. to "3.8") when building for alternate Python stacks.
 %python3_pkgversion 3
 
+# Define the Python interpreter paths in the SRPM macros so that
+# - they can be used in Build/Requires
+# - they can be used in non-Python packages where requiring pythonX-devel would
+#   be an overkill
+
+# use the underscored macros to redefine the behavior of %%python3_version etc.
+%__python2 /usr/bin/python2
+%__python3 /usr/bin/python%{python3_pkgversion}
+
+# use the non-underscored macros to refer to Python in spec, etc.
+%python2 %__python2
+%python3 %__python3
+
+# See https://fedoraproject.org/wiki/Changes/PythonMacroError
+%__python %{error:attempt to use unversioned python, define %%__python to %{__python2} or %{__python3} explicitly}
+
+# Users can use %%python only if they redefined %%__python (e.g. to %%__python3)
+%python %__python
+
 # Define where Python wheels will be stored and the prefix of -wheel packages
 # - In Fedora we want wheel subpackages named e.g. `python-pip-wheel` that
 #   install packages into `/usr/share/python-wheels`. Both names are not

SOURCES/macros.python3

@@ -1,10 +1,12 @@
-%python3_sitelib %(%{__python3} -Ic "import sysconfig; print(sysconfig.get_path('purelib'))")
+# nb: $RPM_BUILD_ROOT is not set when the macros are expanded (at spec parse time)
-%python3_sitearch %(%{__python3} -Ic "import sysconfig; print(sysconfig.get_path('platlib'))")
+#     so we set it manually (to empty string), making our Python prefer the correct install scheme location
-%python3_version %(%{__python3} -Ic "import sys; sys.stdout.write('{0.major}.{0.minor}'.format(sys.version_info))")
+%python3_sitelib %(RPM_BUILD_ROOT= %{__python3} -Ic "import sysconfig; print(sysconfig.get_path('purelib'))")
-%python3_version_nodots %(%{__python3} -Ic "import sys; sys.stdout.write('{0.major}{0.minor}'.format(sys.version_info))")
+%python3_sitearch %(RPM_BUILD_ROOT= %{__python3} -Ic "import sysconfig; print(sysconfig.get_path('platlib'))")
-%python3_platform %(%{__python3} -Ic "import sysconfig; print(sysconfig.get_platform())")
+%python3_version %(RPM_BUILD_ROOT= %{__python3} -Ic "import sys; sys.stdout.write('{0.major}.{0.minor}'.format(sys.version_info))")
-%python3_platform_triplet %(%{__python3} -Ic "import sysconfig; print(sysconfig.get_config_var('MULTIARCH'))")
+%python3_version_nodots %(RPM_BUILD_ROOT= %{__python3} -Ic "import sys; sys.stdout.write('{0.major}{0.minor}'.format(sys.version_info))")
-%python3_ext_suffix %(%{__python3} -Ic "import sysconfig; print(sysconfig.get_config_var('EXT_SUFFIX'))")
+%python3_platform %(RPM_BUILD_ROOT= %{__python3} -Ic "import sysconfig; print(sysconfig.get_platform())")
+%python3_platform_triplet %(RPM_BUILD_ROOT= %{__python3} -Ic "import sysconfig; print(sysconfig.get_config_var('MULTIARCH'))")
+%python3_ext_suffix %(RPM_BUILD_ROOT= %{__python3} -Ic "import sysconfig; print(sysconfig.get_config_var('EXT_SUFFIX'))")
 %py3dir %{_builddir}/python3-%{name}-%{version}-%{release}
 
 %py3_shbang_opts -s
@@ -65,16 +67,28 @@
 }
 
 # With $PATH and $PYTHONPATH set to the %%buildroot,
-# try to import the given Python 3 module(s).
+# try to import the Python 3 module(s) given as command-line args or read from file (-f).
+# Respect the custom values of %%py3_shebang_flags or set nothing if it's undefined.
+# Filter and check import on only top-level modules using -t flag.
+# Exclude unwanted modules by passing their globs to -e option.
 # Useful as a smoke test in %%check when running tests is not feasible.
-# Use spaces or commas as separators.
+# Use spaces or commas as separators if providing list directly.
-%py3_check_import() %{expand:\\\
+# Use newlines as separators if providing list in a file.
-  (cd %{_topdir} &&\\\
+%py3_check_import(e:tf:) %{expand:\\\
   PATH="%{buildroot}%{_bindir}:$PATH"\\\
   PYTHONPATH="${PYTHONPATH:-%{buildroot}%{python3_sitearch}:%{buildroot}%{python3_sitelib}}"\\\
+  _PYTHONSITE="%{buildroot}%{python3_sitearch}:%{buildroot}%{python3_sitelib}"\\\
   PYTHONDONTWRITEBYTECODE=1\\\
-  %{__python3} -c "import %{lua:local m=rpm.expand('%{?*}'):gsub('[%s,]+', ', ');print(m)}"
+  %{lua:
-  )
+  local command = "%{__python3} "
+  if rpm.expand("%{?py3_shebang_flags}") ~= "" then
+    command = command .. "-%{py3_shebang_flags}"
+  end
+  command = command .. " %{_rpmconfigdir}/redhat/import_all_modules.py "
+  -- handle multiline arguments correctly, see https://bugzilla.redhat.com/2018809
+  local args=rpm.expand('%{?**}'):gsub("[%s\\\\]*%s+", " ")
+  print(command .. args)
+  }
 }
 
 # This only supports Python 3.5+ and will never work with Python 2.

SPECS/python-rpm-macros.spec

@@ -1,10 +1,12 @@
 Name:           python-rpm-macros
 Version:        3.9
-Release:        43%{?dist}
+Release:        48%{?dist}
 Summary:        The common Python RPM macros
 URL:            https://src.fedoraproject.org/rpms/python-rpm-macros/
 
-# macros and lua: MIT, compileall2.py: PSFv2
+# macros and lua: MIT
+# import_all_modules.py: MIT
+# compileall2.py: PSFv2
 License:        MIT and Python
 
 # Macros:
@@ -19,6 +21,7 @@ Source201:      python.lua
 # Python code
 %global compileall2_version 0.7.1
 Source301:      https://github.com/fedora-python/compileall2/raw/v%{compileall2_version}/compileall2.py
+Source302:      import_all_modules.py
 
 BuildArch:      noarch
 
@@ -54,7 +57,7 @@ Summary:        RPM macros for building Python 3 packages
 # For %%__python3 and %%python3
 Requires:       python-srpm-macros = %{version}-%{release}
 
-# For %%py_setup
+# For %%py_setup and import_all_modules.py
 Requires:       python-rpm-macros = %{version}-%{release}
 
 %description -n python3-rpm-macros
@@ -75,6 +78,7 @@ install -p -m 644 -t %{buildroot}%{_rpmluadir}/fedora/srpm python.lua
 
 mkdir -p %{buildroot}%{_rpmconfigdir}/redhat
 install -m 644 compileall2.py %{buildroot}%{_rpmconfigdir}/redhat/
+install -m 644 import_all_modules.py %{buildroot}%{_rpmconfigdir}/redhat/
 
 
 %check
@@ -85,6 +89,7 @@ install -m 644 compileall2.py %{buildroot}%{_rpmconfigdir}/redhat/
 %files
 %{rpmmacrodir}/macros.python
 %{rpmmacrodir}/macros.pybytecompile
+%{_rpmconfigdir}/redhat/import_all_modules.py
 
 %files -n python-srpm-macros
 %{rpmmacrodir}/macros.python-srpm
@@ -96,6 +101,28 @@ install -m 644 compileall2.py %{buildroot}%{_rpmconfigdir}/redhat/
 
 
 %changelog
+* Tue Dec 21 2021 Karolina Surma <ksurma@redhat.com> - 3.9-48
+- Fix CI test configuration, so that pytest can import the package code
+
+* Wed Dec 08 2021 Miro Hrončok <mhroncok@redhat.com> - 3.9-47
+- Set %%__python3 value according to %%python3_pkgversion
+  I.e. when %%python3_pkgversion is 3.12, %%__python3 is /usr/bin/python3.12
+
+* Mon Nov 01 2021 Karolina Surma <ksurma@redhat.com> - 3.9-46
+- Fix multiline arguments processing for %%py_check_import
+- Fix %%py_shebang_flags handling within %%py_check_import
+- Process .pth files in buildroot's sitedirs in %%py_check_import
+- Move import_all_modules.py from python-srpm-macros to python-rpm-macros
+
+* Mon Oct 25 2021 Karolina Surma <ksurma@redhat.com> - 3.9-45
+- Introduce -f (read from file) option to %%py{3}_check_import
+- Introduce -t (filter top-level modules) option to %%py{3}_check_import
+- Introduce -e (exclude module globs) option to %%py{3}_check_import
+
+* Thu Sep 09 2021 Miro Hrončok <mhroncok@redhat.com> - 3.9-44
+- Set $RPM_BUILD_ROOT in %%{python3_...} macros
+  to allow selecting alternate sysconfig install scheme based on that variable
+
 * Wed Aug 11 2021 Tomas Orsava <torsava@redhat.com> - 3.9-43
 - Define a new macros %%python_wheel_dir and %%python_wheel_pkg_prefix
 - Related: rhbz#1982668