From 0f5c66a86e405c3f983a909a62c99bbf9305a267 Mon Sep 17 00:00:00 2001 From: eabdullin Date: Thu, 21 Sep 2023 19:56:30 +0000 Subject: [PATCH] import CS pyproject-rpm-macros-1.9.0-1.el9 --- .pyproject-rpm-macros.metadata | 0 SOURCES/README.md | 128 ++++------ SOURCES/macros.aaa-pyproject-srpm | 2 +- SOURCES/macros.pyproject | 34 ++- SOURCES/pyproject_buildrequires.py | 149 +++++++----- .../pyproject_buildrequires_testcases.yaml | 227 ++++++++++++++++-- SOURCES/pyproject_save_files.py | 14 +- SOURCES/pyproject_wheel.py | 57 ++++- SOURCES/test_pyproject_buildrequires.py | 17 +- SPECS/pyproject-rpm-macros.spec | 61 ++++- 10 files changed, 495 insertions(+), 194 deletions(-) delete mode 100644 .pyproject-rpm-macros.metadata diff --git a/.pyproject-rpm-macros.metadata b/.pyproject-rpm-macros.metadata deleted file mode 100644 index e69de29..0000000 diff --git a/SOURCES/README.md b/SOURCES/README.md index 970580b..256f3f5 100644 --- a/SOURCES/README.md +++ b/SOURCES/README.md @@ -163,6 +163,43 @@ The `%pyproject_buildrequires` macro also accepts the `-r` flag for backward com it means "include runtime dependencies" which has been the default since version 0-53. +Passing config settings to build backends +----------------------------------------- + +The `%pyproject_buildrequires` and `%pyproject_wheel` macros accept a `-C` flag +to pass [configuration settings][config_settings] to the build backend. +Options take the form of `-C KEY`, `-C KEY=VALUE`, or `-C--option-with-dashes`. +Pass `-C` multiple times to specify multiple options. +This option is equivalent to pip's `--config-settings` flag. +These are passed on to PEP 517 hooks' `config_settings` argument as a Python +dictionary. + +The `%pyproject_buildrequires` macro passes these options to the +`get_requires_for_build_wheel` and `prepare_metadata_for_build_wheel` hooks. +Passing `-C` to `%pyproject_buildrequires` is incompatible with `-N` which does +not call these hooks at all. + +The `%pyproject_wheel` macro passes these options to the `build_wheel` hook. + +Consult the project's upstream documentation and/or the corresponding build +backend's documentation for more information. +Note that some projects don't use config settings at all +and other projects may only accept config settings for one of the two steps. + +Note that the current implementation of the macros uses `pip` to build wheels. +On some systems (notably on RHEL 9 with Python 3.9), +`pip` is too old to understand `--config-settings`. +Using the `-C` option for `%pyproject_wheel` (or `%pyproject_buildrequires -w`) +is not supported there and will result to an error like: + + Usage: + /usr/bin/python3 -m pip wheel [options] ... + ... + no such option: --config-settings + +[config_settings]: https://peps.python.org/pep-0517/#config-settings + + Running tox based tests ----------------------- @@ -336,91 +373,6 @@ These arguments are still required: Multiple subpackages are generated when multiple names are provided. -PROVISIONAL: Importing just-built (extension) modules in %build ---------------------------------------------------------------- - -Sometimes, it is desired to be able to import the *just-built* extension modules -in the `%build` section, e.g. to build the documentation with Sphinx. - - %build - %pyproject_wheel - ... build the docs here ... - -With pure Python packages, it might be possible to set `PYTHONPATH=${PWD}` or `PYTHONPATH=${PWD}/src`. -However, it is a bit more complicated with extension modules. - -The location of just-built modules might differ depending on Python version, architecture, pip version, etc. -Hence, the macro `%{pyproject_build_lib}` exists to be used like this: - - %build - %pyproject_wheel - PYTHONPATH=%{pyproject_build_lib} ... build the docs here ... - -This macro is currently **provisional** and the behavior might change. -Please subscribe to Fedora's [python-devel list] if you use the macro. - -The `%{pyproject_build_lib}` macro expands to an Shell `$(...)` expression and does not work when put into single quotes (`'`). - -Depending on the pip version, the expanded value will differ: - -[python-devel list]: https://lists.fedoraproject.org/archives/list/python-devel@lists.fedoraproject.org/ - -### New pip 21.3+ with in-tree-build and setuptools 62.1+ (Fedora 37+) - -Always use the macro from the same directory where you called `%pyproject_wheel` from. -The value will expand to something like: - -* `/builddir/build/BUILD/%{name}-%{version}/build/lib.linux-x86_64-cpython-311` for wheels with extension modules -* `/builddir/build/BUILD/%{name}-%{version}/build/lib` for pure Python wheels - -If multiple wheels were built from the same directory, -some pure Python and some with extension modules, -the expanded value will be combined with `:`: - -* `/builddir/build/BUILD/%{name}-%{version}/build/lib.linux-x86_64-cypthon-311:/builddir/build/BUILD/%{name}-%{version}/build/lib` - -If multiple wheels were built from different directories, -the value will differ depending on the current directory. - - -### New pip 21.3+ with in-tree-build and older setuptools (Fedora 36) - -Always use the macro from the same directory where you called `%pyproject_wheel` from. -The value will expand to something like: - -* `/builddir/build/BUILD/%{name}-%{version}/build/lib.linux-x86_64-3.10` for wheels with extension modules -* `/builddir/build/BUILD/%{name}-%{version}/build/lib` for pure Python wheels - -If multiple wheels were built from the same directory, -some pure Python and some with extension modules, -the expanded value will be combined with `:`: - -* `/builddir/build/BUILD/%{name}-%{version}/build/lib.linux-x86_64-3.10:/builddir/build/BUILD/%{name}-%{version}/build/lib` - -If multiple wheels were built from different directories, -the value will differ depending on the current directory. - - -### Older pip with out-of-tree-build (Fedora 35 and EL 9) - -The value will expand to something like: - -* `/builddir/build/BUILD/%{name}-%{version}/.pyproject-builddir/pip-req-build-xxxxxxxx/build/lib.linux-x86_64-3.10` for wheels with extension modules -* `/builddir/build/BUILD/%{name}-%{version}/.pyproject-builddir/pip-req-build-xxxxxxxx/build/lib` for pure Python wheels - -Note that the exact value is **not stable** between builds -(the `xxxxxxxx` part is randomly generated, -neither you should consider the `.pyproject-builddir` directory to remain stable). - -If multiple wheels are built, -the expanded value will always be combined with `:` regardless of the current directory, e.g.: - -* `/builddir/build/BUILD/%{name}-%{version}/.pyproject-builddir/pip-req-build-xxxxxxxx/build/lib.linux-x86_64-3.10:/builddir/build/BUILD/%{name}-%{version}/.pyproject-builddir/pip-req-build-yyyyyyyy/build/lib.linux-x86_64-3.10:/builddir/build/BUILD/%{name}-%{version}/.pyproject-builddir/pip-req-build-zzzzzzzz/build/lib` - -**Note:** If you manage to build some wheels with in-tree-build and some with out-of-tree-build option, -the expanded value will contain all relevant directories. - - Limitations ----------- @@ -473,6 +425,12 @@ so be prepared for problems. [pip's documentation]: https://pip.pypa.io/en/stable/cli/pip_install/#vcs-support +Deprecated +---------- + +The `%{pyproject_build_lib}` macro is deprecated, don't use it. + + Testing the macros ------------------ diff --git a/SOURCES/macros.aaa-pyproject-srpm b/SOURCES/macros.aaa-pyproject-srpm index 9dab590..d845e96 100644 --- a/SOURCES/macros.aaa-pyproject-srpm +++ b/SOURCES/macros.aaa-pyproject-srpm @@ -4,4 +4,4 @@ # this macro will cause the package with the real macro to be installed. # When macros.pyproject is installed, it overrides this macro. # Note: This needs to maintain the same set of options as the real macro. -%pyproject_buildrequires(rRxtNwe:) echo 'pyproject-rpm-macros' && exit 0 +%pyproject_buildrequires(rRxtNwe:C:) echo 'pyproject-rpm-macros' && exit 0 diff --git a/SOURCES/macros.pyproject b/SOURCES/macros.pyproject index 8cc59ae..dfb3e2d 100644 --- a/SOURCES/macros.pyproject +++ b/SOURCES/macros.pyproject @@ -19,21 +19,29 @@ %_pyproject_modules %{_builddir}/%{_pyproject_files_prefix}-pyproject-modules %_pyproject_ghost_distinfo %{_builddir}/%{_pyproject_files_prefix}-pyproject-ghost-distinfo %_pyproject_record %{_builddir}/%{_pyproject_files_prefix}-pyproject-record +%_pyproject_buildrequires %{_builddir}/%{_pyproject_files_prefix}-pyproject-buildrequires # Avoid leaking %%{_pyproject_builddir} to pytest collection # https://bugzilla.redhat.com/show_bug.cgi?id=1935212 # The value is read and used by the %%pytest and %%tox macros: %_set_pytest_addopts %global __pytest_addopts --ignore=%{_pyproject_builddir} -%pyproject_wheel() %{expand:\\\ +%pyproject_wheel(C:) %{expand:\\\ %_set_pytest_addopts mkdir -p "%{_pyproject_builddir}" CFLAGS="${CFLAGS:-${RPM_OPT_FLAGS}}" LDFLAGS="${LDFLAGS:-${RPM_LD_FLAGS}}" TMPDIR="%{_pyproject_builddir}" \\\ -%{__python3} -Bs %{_rpmconfigdir}/redhat/pyproject_wheel.py %{_pyproject_wheeldir} +%{__python3} -Bs %{_rpmconfigdir}/redhat/pyproject_wheel.py %{?**} %{_pyproject_wheeldir} } -%pyproject_build_lib %{expand:\\\ +%pyproject_build_lib %{!?__pyproject_build_lib_warned:%{warn:The %%{pyproject_build_lib} macro is deprecated. +It only works with setuptools and is not build-backend-agnostic. +The macro is not scheduled for removal, but there is a possibility of incompatibilities with future versions of setuptools. +As a replacement for the macro for the setuptools backend on Fedora 37+, you can use $PWD/build/lib for pure Python packages, +or $PWD/build/lib.%%{python3_platform}-cpython-%%{python3_version_nodots} for packages with extension modules. +Other build backends and older distributions may need different paths. +See https://lists.fedoraproject.org/archives/list/python-devel@lists.fedoraproject.org/thread/HMLOPAU3RZLXD4BOJHTIPKI3I4U6U7OE/ for details. +}%global __pyproject_build_lib_warned 1}%{expand:\\\ $( pyproject_build_lib=() if [ -d build/lib.%{python3_platform}-cpython-%{python3_version_nodots} ]; then @@ -96,7 +104,14 @@ fi %pyproject_extras_subpkg(n:i:f:F) %{expand:%{?python_extras_subpkg:%{python_extras_subpkg%{?!-i:%{?!-f:%{?!-F: -f %{_pyproject_ghost_distinfo}}}} %**}}} +# Escaping an actual percentage sign in path by 8 signs has been verified in RPM 4.16 and 4.17. +# See this thread http://lists.rpm.org/pipermail/rpm-list/2021-June/002048.html +# Since RPM 4.19, 2 signs are needed instead. +# On the CI, we build tests/escape_percentages.spec to verify the assumptions. +# We should check RPM version here instead of Fedora/RHEL, but it's hard; +# see https://github.com/rpm-software-management/rpm/issues/2523 %pyproject_save_files() %{expand:\\\ +%{expr:0%{?fedora} >= 39 || 0%{?rhel} >= 10 ? "RPM_PERCENTAGES_COUNT=2" : "RPM_PERCENTAGES_COUNT=8" } \\ %{__python3} %{_rpmconfigdir}/redhat/pyproject_save_files.py \\ --output-files "%{pyproject_files}" \\ --output-modules "%{_pyproject_modules}" \\ @@ -125,7 +140,7 @@ fi # Note: Keep the options in sync with this macro from macros.aaa-pyproject-srpm -%pyproject_buildrequires(rRxtNwe:) %{expand:\\\ +%pyproject_buildrequires(rRxtNwe:C:) %{expand:\\\ %_set_pytest_addopts # The _auto_set_build_flags feature does not do this in %%generate_buildrequires section, # but we want to get an environment consistent with %%build: @@ -143,6 +158,7 @@ fi %{-e:%{error:The -N and -e options are mutually exclusive}} %{-t:%{error:The -N and -t options are mutually exclusive}} %{-w:%{error:The -N and -w options are mutually exclusive}} +%{-C:%{error:The -N and -C options are mutually exclusive}} } %{-e:%{expand:%global toxenv %(%{__python3} -s %{_rpmconfigdir}/redhat/pyproject_construct_toxenv.py %{?**})}} echo 'pyproject-rpm-macros' # first stdout line matches the implementation in macros.aaa-pyproject-srpm @@ -151,9 +167,9 @@ echo 'python%{python3_pkgversion}dist(pip) >= 19' echo 'python%{python3_pkgversion}dist(packaging)' %{!-N:if [ -f pyproject.toml ]; then %["%{python3_pkgversion}" == "3" - ? "echo '(python%{python3_pkgversion}dist(toml) if python%{python3_pkgversion}-devel < 3.11)'" + ? "echo '(python%{python3_pkgversion}dist(tomli) if python%{python3_pkgversion}-devel < 3.11)'" : "%[v"%{python3_pkgversion}" < v"3.11" - ? "echo 'python%{python3_pkgversion}dist(toml)'" + ? "echo 'python%{python3_pkgversion}dist(tomli)'" : "true # will use tomllib, echo nothing" ]" ] @@ -169,9 +185,13 @@ fi} rm -rfv *.dist-info/ >&2 if [ -f %{__python3} ]; then mkdir -p "%{_pyproject_builddir}" + echo -n > %{_pyproject_buildrequires} CFLAGS="${CFLAGS:-${RPM_OPT_FLAGS}}" LDFLAGS="${LDFLAGS:-${RPM_LD_FLAGS}}" TMPDIR="%{_pyproject_builddir}" \\\ - RPM_TOXENV="%{toxenv}" HOSTNAME="rpmbuild" %{__python3} -Bs %{_rpmconfigdir}/redhat/pyproject_buildrequires.py %{?!_python_no_extras_requires:--generate-extras} --python3_pkgversion %{python3_pkgversion} --wheeldir %{_pyproject_wheeldir} %{?**} + RPM_TOXENV="%{toxenv}" HOSTNAME="rpmbuild" %{__python3} -Bs %{_rpmconfigdir}/redhat/pyproject_buildrequires.py %{?!_python_no_extras_requires:--generate-extras} --python3_pkgversion %{python3_pkgversion} --wheeldir %{_pyproject_wheeldir} --output %{_pyproject_buildrequires} %{?**} >&2 + cat %{_pyproject_buildrequires} fi +# Incomplete .dist-info dir might confuse importlib.metadata +rm -rfv *.dist-info/ >&2 } diff --git a/SOURCES/pyproject_buildrequires.py b/SOURCES/pyproject_buildrequires.py index 323ab2a..844fd38 100644 --- a/SOURCES/pyproject_buildrequires.py +++ b/SOURCES/pyproject_buildrequires.py @@ -4,9 +4,7 @@ import os import sys import importlib.metadata import argparse -import tempfile import traceback -import contextlib import json import subprocess import re @@ -16,6 +14,7 @@ import pathlib import zipfile from pyproject_requirements_txt import convert_requirements_txt +from pyproject_wheel import parse_config_settings_args # Some valid Python version specifiers are not supported. @@ -46,39 +45,6 @@ except ImportError as e: from pyproject_convert import convert -@contextlib.contextmanager -def hook_call(): - """Context manager that records all stdout content (on FD level) - and prints it to stderr at the end, with a 'HOOK STDOUT: ' prefix.""" - tmpfile = io.TextIOWrapper( - tempfile.TemporaryFile(buffering=0), - encoding='utf-8', - errors='replace', - write_through=True, - ) - - stdout_fd = 1 - stdout_fd_dup = os.dup(stdout_fd) - stdout_orig = sys.stdout - - # begin capture - sys.stdout = tmpfile - os.dup2(tmpfile.fileno(), stdout_fd) - - try: - yield - finally: - # end capture - sys.stdout = stdout_orig - os.dup2(stdout_fd_dup, stdout_fd) - - tmpfile.seek(0) # rewind - for line in tmpfile: - print_err('HOOK STDOUT:', line, end='') - - tmpfile.close() - - def guess_reason_for_invalid_requirement(requirement_str): if ':' in requirement_str: message = ( @@ -100,10 +66,11 @@ def guess_reason_for_invalid_requirement(requirement_str): class Requirements: - """Requirement printer""" + """Requirement gatherer. The macro will eventually print out output_lines.""" def __init__(self, get_installed_version, extras=None, - generate_extras=False, python3_pkgversion='3'): + generate_extras=False, python3_pkgversion='3', config_settings=None): self.get_installed_version = get_installed_version + self.output_lines = [] self.extras = set() if extras: @@ -111,9 +78,11 @@ class Requirements: self.add_extras(*extra.split(',')) self.missing_requirements = False + self.ignored_alien_requirements = [] self.generate_extras = generate_extras self.python3_pkgversion = python3_pkgversion + self.config_settings = config_settings def add_extras(self, *extras): self.extras |= set(e.strip() for e in extras) @@ -130,7 +99,7 @@ class Requirements: return True return False - def add(self, requirement_str, *, source=None): + def add(self, requirement_str, *, package_name=None, source=None): """Output a Python-style requirement string as RPM dep""" print_err(f'Handling {requirement_str} from {source}') @@ -152,6 +121,21 @@ class Requirements: if (requirement.marker is not None and not self.evaluate_all_environments(requirement)): print_err(f'Ignoring alien requirement:', requirement_str) + self.ignored_alien_requirements.append(requirement_str) + return + + # Handle self-referencing requirements + if package_name and canonicalize_name(package_name) == name: + # Self-referential extras need to be handled specially + if requirement.extras: + if not (requirement.extras <= self.extras): # only handle it if needed + # let all further requirements know we want those extras + self.add_extras(*requirement.extras) + # re-add all of the alien requirements ignored in the past + # they might no longer be alien now + self.readd_ignored_alien_requirements(package_name=package_name) + else: + print_err(f'Ignoring self-referential requirement without extras:', requirement_str) return # We need to always accept pre-releases as satisfying the requirement @@ -192,12 +176,12 @@ class Requirements: together.append(convert(python3dist(name, python3_pkgversion=self.python3_pkgversion), specifier.operator, specifier.version)) if len(together) == 0: - print(python3dist(name, - python3_pkgversion=self.python3_pkgversion)) + dep = python3dist(name, python3_pkgversion=self.python3_pkgversion) + self.output_lines.append(dep) elif len(together) == 1: - print(together[0]) + self.output_lines.append(together[0]) else: - print(f"({' with '.join(together)})") + self.output_lines.append(f"({' with '.join(together)})") def check(self, *, source=None): """End current pass if any unsatisfied dependencies were output""" @@ -210,23 +194,25 @@ class Requirements: for req_str in requirement_strs: self.add(req_str, **kwargs) + def readd_ignored_alien_requirements(self, **kwargs): + """add() previously ignored alien requirements again.""" + requirements, self.ignored_alien_requirements = self.ignored_alien_requirements, [] + kwargs.setdefault('source', 'Previously ignored alien requirements') + self.extend(requirements, **kwargs) + def toml_load(opened_binary_file): try: # tomllib is in the standard library since 3.11.0b1 - import tomllib as toml_module - load_from = opened_binary_file + import tomllib except ImportError: try: - # note: we could use tomli here, - # but for backwards compatibility with RHEL 9, we use toml instead - import toml as toml_module - load_from = io.TextIOWrapper(opened_binary_file, encoding='utf-8') + import tomli as tomllib except ImportError as e: print_err('Import error:', e) # already echoed by the %pyproject_buildrequires macro sys.exit(0) - return toml_module.load(load_from) + return tomllib.load(opened_binary_file) def get_backend(requirements): @@ -285,17 +271,30 @@ def get_backend(requirements): def generate_build_requirements(backend, requirements): get_requires = getattr(backend, 'get_requires_for_build_wheel', None) if get_requires: - with hook_call(): - new_reqs = get_requires() + new_reqs = get_requires(config_settings=requirements.config_settings) requirements.extend(new_reqs, source='get_requires_for_build_wheel') requirements.check(source='get_requires_for_build_wheel') -def requires_from_metadata_file(metadata_file): - message = email.parser.Parser().parse(metadata_file, headersonly=True) +def parse_metadata_file(metadata_file): + return email.parser.Parser().parse(metadata_file, headersonly=True) + + +def requires_from_parsed_metadata_file(message): return {k: message.get_all(k, ()) for k in ('Requires', 'Requires-Dist')} +def package_name_from_parsed_metadata_file(message): + return message.get('name') + + +def package_name_and_requires_from_metadata_file(metadata_file): + message = parse_metadata_file(metadata_file) + package_name = package_name_from_parsed_metadata_file(message) + requires = requires_from_parsed_metadata_file(message) + return package_name, requires + + def generate_run_requirements_hook(backend, requirements): hook_name = 'prepare_metadata_for_build_wheel' prepare_metadata = getattr(backend, hook_name, None) @@ -306,11 +305,13 @@ def generate_run_requirements_hook(backend, requirements): 'Use the provisional -w flag to build the wheel and parse the metadata from it, ' 'or use the -R flag not to generate runtime dependencies.' ) - with hook_call(): - dir_basename = prepare_metadata('.') + dir_basename = prepare_metadata('.', config_settings=requirements.config_settings) with open(dir_basename + '/METADATA') as metadata_file: - for key, requires in requires_from_metadata_file(metadata_file).items(): - requirements.extend(requires, source=f'hook generated metadata: {key}') + name, requires = package_name_and_requires_from_metadata_file(metadata_file) + for key, req in requires.items(): + requirements.extend(req, + package_name=name, + source=f'hook generated metadata: {key} ({name})') def find_built_wheel(wheeldir): @@ -328,7 +329,11 @@ def generate_run_requirements_wheel(backend, requirements, wheeldir): wheel = find_built_wheel(wheeldir) if not wheel: import pyproject_wheel - returncode = pyproject_wheel.build_wheel(wheeldir=wheeldir, stdout=sys.stderr) + returncode = pyproject_wheel.build_wheel( + wheeldir=wheeldir, + stdout=sys.stderr, + config_settings=requirements.config_settings, + ) if returncode != 0: raise RuntimeError('Failed to build the wheel for %pyproject_buildrequires -w.') wheel = find_built_wheel(wheeldir) @@ -340,8 +345,11 @@ def generate_run_requirements_wheel(backend, requirements, wheeldir): for name in wheelfile.namelist(): if name.count('/') == 1 and name.endswith('.dist-info/METADATA'): with io.TextIOWrapper(wheelfile.open(name), encoding='utf-8') as metadata_file: - for key, requires in requires_from_metadata_file(metadata_file).items(): - requirements.extend(requires, source=f'built wheel metadata: {key}') + name, requires = package_name_and_requires_from_metadata_file(metadata_file) + for key, req in requires.items(): + requirements.extend(req, + package_name=name, + source=f'built wheel metadata: {key} ({name})') break else: raise RuntimeError('Could not find *.dist-info/METADATA in built wheel.') @@ -412,16 +420,20 @@ def python3dist(name, op=None, version=None, python3_pkgversion="3"): def generate_requires( *, include_runtime=False, build_wheel=False, wheeldir=None, toxenv=None, extras=None, get_installed_version=importlib.metadata.version, # for dep injection - generate_extras=False, python3_pkgversion="3", requirement_files=None, use_build_system=True + generate_extras=False, python3_pkgversion="3", requirement_files=None, use_build_system=True, + output, config_settings=None, ): """Generate the BuildRequires for the project in the current directory + The generated BuildRequires are written to the provided output. + This is the main Python entry point. """ requirements = Requirements( get_installed_version, extras=extras or [], generate_extras=generate_extras, - python3_pkgversion=python3_pkgversion + python3_pkgversion=python3_pkgversion, + config_settings=config_settings, ) try: @@ -444,6 +456,8 @@ def generate_requires( generate_run_requirements(backend, requirements, build_wheel=build_wheel, wheeldir=wheeldir) except EndPass: return + finally: + output.write_text(os.linesep.join(requirements.output_lines) + os.linesep) def main(argv): @@ -469,6 +483,9 @@ def main(argv): '-p', '--python3_pkgversion', metavar='PYTHON3_PKGVERSION', default="3", help=argparse.SUPPRESS, ) + parser.add_argument( + '--output', type=pathlib.Path, required=True, help=argparse.SUPPRESS, + ) parser.add_argument( '--wheeldir', metavar='PATH', default=None, help=argparse.SUPPRESS, @@ -506,6 +523,12 @@ def main(argv): metavar='REQUIREMENTS.TXT', help=('Add buildrequires from file'), ) + parser.add_argument( + '-C', + dest='config_settings', + action='append', + help='Configuration settings to pass to the PEP 517 backend', + ) args = parser.parse_args(argv) @@ -539,6 +562,8 @@ def main(argv): python3_pkgversion=args.python3_pkgversion, requirement_files=args.requirement_files, use_build_system=args.use_build_system, + output=args.output, + config_settings=parse_config_settings_args(args.config_settings), ) except Exception: # Log the traceback explicitly (it's useful debug info) diff --git a/SOURCES/pyproject_buildrequires_testcases.yaml b/SOURCES/pyproject_buildrequires_testcases.yaml index 1a2e6bb..aba6002 100644 --- a/SOURCES/pyproject_buildrequires_testcases.yaml +++ b/SOURCES/pyproject_buildrequires_testcases.yaml @@ -17,7 +17,7 @@ Insufficient version of setuptools: installed: setuptools: 5 wheel: 1 - toml: 1 + tomli: 1 pyproject.toml: | # empty setup.py: | @@ -42,7 +42,7 @@ Default build system, empty setup.py: installed: setuptools: 50 wheel: 1 - toml: 1 + tomli: 1 include_runtime: false pyproject.toml: | # empty @@ -58,7 +58,7 @@ pyproject.toml with build-backend and setup.py: installed: setuptools: 50 wheel: 1 - toml: 1 + tomli: 1 setup.py: | # empty pyproject.toml: | @@ -81,7 +81,7 @@ Erroring setup.py: Bad character in version: installed: - toml: 1 + tomli: 1 pyproject.toml: | [build-system] requires = ["pkg == 0.$.^.*"] @@ -89,7 +89,7 @@ Bad character in version: Single value version with unsupported compatible operator: installed: - toml: 1 + tomli: 1 pyproject.toml: | [build-system] requires = ["pkg ~= 42", "foo"] @@ -98,7 +98,7 @@ Single value version with unsupported compatible operator: Asterisk in version with unsupported compatible operator: installed: - toml: 1 + tomli: 1 pyproject.toml: | [build-system] requires = ["pkg ~= 0.1.*", "foo"] @@ -107,7 +107,7 @@ Asterisk in version with unsupported compatible operator: Local path as requirement: installed: - toml: 1 + tomli: 1 pyproject.toml: | [build-system] requires = ["./pkg-1.2.3.tar.gz", "foo"] @@ -116,7 +116,7 @@ Local path as requirement: Pip's egg=pkgName requirement not in requirements file: installed: - toml: 1 + tomli: 1 pyproject.toml: | [build-system] requires = ["git+https://github.com/monty/spam.git@master#egg=spam", "foo"] @@ -125,7 +125,7 @@ Pip's egg=pkgName requirement not in requirements file: URL without egg fragment as requirement: installed: - toml: 1 + tomli: 1 pyproject.toml: | [build-system] requires = ["git+https://github.com/pkg-dev/pkg.git@96dbe5e3", "foo"] @@ -137,7 +137,7 @@ Build system dependencies in pyproject.toml with extras: installed: setuptools: 50 wheel: 1 - toml: 1 + tomli: 1 pyproject.toml: | [build-system] requires = [ @@ -186,7 +186,7 @@ Build system dependencies in pyproject.toml without extras: installed: setuptools: 50 wheel: 1 - toml: 1 + tomli: 1 pyproject.toml: | [build-system] requires = [ @@ -389,7 +389,7 @@ Run dependencies with extras and build wheel option: result: 0 stderr_contains: "Reading metadata from {wheeldir}/pytest-6.6.6-py3-none-any.whl" -Tox dependencies: +tox dependencies: installed: setuptools: 50 wheel: 1 @@ -433,7 +433,7 @@ Tox dependencies: python3dist(inst) result: 0 -Tox extras: +tox extras: installed: setuptools: 50 wheel: 1 @@ -497,7 +497,7 @@ Tox extras: python3dist(extra-dep[extra_dep]) result: 0 -Tox provision unsatisfied: +tox provision unsatisfied: installed: setuptools: 50 wheel: 1 @@ -542,7 +542,7 @@ Tox provision unsatisfied: python3dist(tox) >= 3.999 result: 0 -Tox provision satisfied: +tox provision satisfied: installed: setuptools: 50 wheel: 1 @@ -654,7 +654,7 @@ With pyproject.toml, requirements file and with -N option: installed: setuptools: 50 wheel: 1 - toml: 1 + tomli: 1 lxml: 3.9 ncclient: 1 cryptography: 2 @@ -690,7 +690,7 @@ With pyproject.toml, requirements file and without -N option: installed: setuptools: 50 wheel: 1 - toml: 1 + tomli: 1 lxml: 3.9 ncclient: 1 cryptography: 2 @@ -801,7 +801,7 @@ Pre-releases are accepted: installed: setuptools: 50 wheel: 1 - toml: 1 + tomli: 1 cffi: 1.15.0rc2 pyproject.toml: | [build-system] @@ -820,7 +820,7 @@ Pre-releases are accepted: result: 0 -Wrapped subprocess prints to stdout from setup.py: +Stdout from wrapped subprocess does not appear in output: installed: setuptools: 50 wheel: 1 @@ -834,5 +834,192 @@ Wrapped subprocess prints to stdout from setup.py: python3dist(setuptools) >= 40.8 python3dist(wheel) python3dist(wheel) - stderr_contains: "HOOK STDOUT: LEAK?" + result: 0 + +pyproject.toml with runtime dependencies: + skipif: not SETUPTOOLS_60 + installed: + setuptools: 50 + wheel: 1 + tomli: 1 + pyproject.toml: | + [build-system] + requires = ["setuptools"] + build-backend = "setuptools.build_meta" + [project] + name = "my_package" + version = "0.1" + dependencies = [ + "foo", + 'importlib-metadata; python_version<"3.8"', + ] + expected: | + python3dist(setuptools) + python3dist(wheel) + python3dist(foo) + result: 0 + +pyproject.toml with runtime dependencies and partially selected extras: + skipif: not SETUPTOOLS_60 + installed: + setuptools: 50 + wheel: 1 + tomli: 1 + extras: + - tests + pyproject.toml: | + [build-system] + requires = ["setuptools"] + build-backend = "setuptools.build_meta" + [project] + name = "my_package" + version = "0.1" + dependencies = [ + "foo", + 'importlib-metadata; python_version<"3.8"', + ] + [project.optional-dependencies] + tests = ["pytest>=5", "pytest-mock"] + docs = ["sphinx", "python-docs-theme"] + expected: | + python3dist(setuptools) + python3dist(wheel) + python3dist(foo) + python3dist(pytest) >= 5 + python3dist(pytest-mock) + result: 0 + +Self-referencing extras (sooner): + installed: + setuptools: 50 + wheel: 1 + tomli: 1 + extras: + - dev # this is deliberately sooner in the alphabet than the referenced ones + pyproject.toml: | + [build-system] + requires = ["setuptools"] + build-backend = "setuptools.build_meta" + setup.cfg: | + [metadata] + name = my_package + version = 0.1 + [options] + install_requires = + foo + importlib-metadata; python_version<"3.8" + [options.extras_require] + tests = pytest>=5; pytest-mock + docs = sphinx; python-docs-theme + dev = my_package[docs,tests] + expected: | + python3dist(setuptools) + python3dist(wheel) + python3dist(foo) + python3dist(sphinx) + python3dist(python-docs-theme) + python3dist(pytest) >= 5 + python3dist(pytest-mock) + result: 0 + +Self-referencing extras (later): + installed: + setuptools: 50 + wheel: 1 + tomli: 1 + extras: + - xdev # this is deliberately later in the alphabet than the referenced ones + pyproject.toml: | + [build-system] + requires = ["setuptools"] + build-backend = "setuptools.build_meta" + setup.cfg: | + [metadata] + name = my_package + version = 0.1 + [options] + install_requires = + foo + importlib-metadata; python_version<"3.8" + [options.extras_require] + tests = pytest>=5; pytest-mock + docs = sphinx; python-docs-theme + xdev = my_package[docs,tests] + expected: | + python3dist(setuptools) + python3dist(wheel) + python3dist(foo) + python3dist(sphinx) + python3dist(python-docs-theme) + python3dist(pytest) >= 5 + python3dist(pytest-mock) + result: 0 + +Self-referencing extras (maze): + installed: + setuptools: 50 + wheel: 1 + tomli: 1 + extras: + - start + pyproject.toml: | + [build-system] + requires = ["setuptools"] + build-backend = "setuptools.build_meta" + setup.cfg: | + [metadata] + name = my_package + version = 0.1 + [options.extras_require] + start = my_package[left,right]; startdep + left = my_package[right,forward]; leftdep + right = my_package[left,forward]; rightdep + forward = my_package[backward]; forwarddep + backward = my_package[left,right]; backwarddep + never = my_package[forward]; neverdep + expected: | + python3dist(setuptools) + python3dist(wheel) + python3dist(backwarddep) + python3dist(forwarddep) + python3dist(leftdep) + python3dist(rightdep) + python3dist(startdep) + result: 0 + +config_settings_control: + include_runtime: false + config_settings: + pyproject.toml: | + [build-system] + build-backend = "test_backend" + backend-path = ["."] + test_backend.py: | + def get_requires_for_build_wheel(config_settings=None): + if not (config_settings is None or isinstance(config_settings, dict)): + raise TypeError + if config_settings and "test-config-setting" in config_settings: + return ["test-config-setting"] + return ["test-no-config-setting"] + expected: | + python3dist(test-no-config-setting) + result: 0 + +config_settings: + include_runtime: false + config_settings: + test-config-setting: "" + pyproject.toml: | + [build-system] + build-backend = "test_backend" + backend-path = ["."] + test_backend.py: | + def get_requires_for_build_wheel(config_settings=None): + if not (config_settings is None or isinstance(config_settings, dict)): + raise TypeError + if config_settings and "test-config-setting" in config_settings: + return ["test-config-setting"] + return ["test-no-config-setting"] + expected: | + python3dist(test-config-setting) result: 0 diff --git a/SOURCES/pyproject_save_files.py b/SOURCES/pyproject_save_files.py index 00d706d..551a876 100644 --- a/SOURCES/pyproject_save_files.py +++ b/SOURCES/pyproject_save_files.py @@ -12,6 +12,9 @@ from importlib.metadata import Distribution # From RPM's build/files.c strtokWithQuotes delim argument RPM_FILES_DELIMETERS = ' \n\t' +# See the comment in the macro that wraps this script +RPM_PERCENTAGES_COUNT = int(os.getenv('RPM_PERCENTAGES_COUNT', '2')) + # RPM hardcodes the lists of manpage extensions and directories, # so we have to maintain separate ones :( # There is an issue for RPM to provide the lists as macros: @@ -441,13 +444,13 @@ def escape_rpm_path(path): '"/usr/lib/python3.9/site-packages/setuptools/script (dev).tmpl"' >>> escape_rpm_path('/usr/share/data/100%valid.path') - '/usr/share/data/100%%%%%%%%valid.path' + '/usr/share/data/100%%valid.path' >>> escape_rpm_path('/usr/share/data/100 % valid.path') - '"/usr/share/data/100 %%%%%%%% valid.path"' + '"/usr/share/data/100 %% valid.path"' >>> escape_rpm_path('/usr/share/data/1000 %% valid.path') - '"/usr/share/data/1000 %%%%%%%%%%%%%%%% valid.path"' + '"/usr/share/data/1000 %%%% valid.path"' >>> escape_rpm_path('/usr/share/data/spaces and "quotes"') Traceback (most recent call last): @@ -461,10 +464,7 @@ def escape_rpm_path(path): """ orig_path = path = str(path) if "%" in path: - # Escaping by 8 %s has been verified in RPM 4.16 and 4.17, but probably not stable - # See this thread http://lists.rpm.org/pipermail/rpm-list/2021-June/002048.html - # On the CI, we build tests/escape_percentages.spec to verify this assumption - path = path.replace("%", "%" * 8) + path = path.replace("%", "%" * RPM_PERCENTAGES_COUNT) if any(symbol in path for symbol in RPM_FILES_DELIMETERS): if '"' in path: # As far as we know, RPM cannot list such file individually diff --git a/SOURCES/pyproject_wheel.py b/SOURCES/pyproject_wheel.py index 1936d9c..6d62176 100644 --- a/SOURCES/pyproject_wheel.py +++ b/SOURCES/pyproject_wheel.py @@ -1,8 +1,46 @@ +import argparse import sys import subprocess -def build_wheel(*, wheeldir, stdout=None): +def parse_config_settings_args(config_settings): + """ + Given a list of config `KEY=VALUE` formatted config settings, + return a dictionary that can be passed to PEP 517 hook functions. + """ + if not config_settings: + return config_settings + new_config_settings = {} + for arg in config_settings: + key, _, value = arg.partition('=') + if key in new_config_settings: + if not isinstance(new_config_settings[key], list): + # convert the existing value to a list + new_config_settings[key] = [new_config_settings[key]] + new_config_settings[key].append(value) + else: + new_config_settings[key] = value + return new_config_settings + + +def get_config_settings_args(config_settings): + """ + Given a dictionary of PEP 517 backend config_settings, + yield --config-settings args that can be passed to pip's CLI + """ + if not config_settings: + return + for key, values in config_settings.items(): + if not isinstance(values, list): + values = [values] + for value in values: + if value == '': + yield f'--config-settings={key}' + else: + yield f'--config-settings={key}={value}' + + +def build_wheel(*, wheeldir, stdout=None, config_settings=None): command = ( sys.executable, '-m', 'pip', @@ -15,11 +53,26 @@ def build_wheel(*, wheeldir, stdout=None): '--no-clean', '--progress-bar', 'off', '--verbose', + *get_config_settings_args(config_settings), '.', ) cp = subprocess.run(command, stdout=stdout) return cp.returncode +def parse_args(argv=None): + parser = argparse.ArgumentParser(prog='%pyproject_wheel') + parser.add_argument('wheeldir', help=argparse.SUPPRESS) + parser.add_argument( + '-C', + dest='config_settings', + action='append', + help='Configuration settings to pass to the PEP 517 backend', + ) + args = parser.parse_args(argv) + args.config_settings = parse_config_settings_args(args.config_settings) + return args + + if __name__ == '__main__': - sys.exit(build_wheel(wheeldir=sys.argv[1])) + sys.exit(build_wheel(**vars(parse_args()))) diff --git a/SOURCES/test_pyproject_buildrequires.py b/SOURCES/test_pyproject_buildrequires.py index 74f3ae8..0fa07db 100644 --- a/SOURCES/test_pyproject_buildrequires.py +++ b/SOURCES/test_pyproject_buildrequires.py @@ -1,11 +1,15 @@ from pathlib import Path import importlib.metadata +import packaging.version import pytest +import setuptools import yaml from pyproject_buildrequires import generate_requires +SETUPTOOLS_VERSION = packaging.version.parse(setuptools.__version__) +SETUPTOOLS_60 = SETUPTOOLS_VERSION >= packaging.version.parse('60') testcases = {} with Path(__file__).parent.joinpath('pyproject_buildrequires_testcases.yaml').open() as f: @@ -21,12 +25,16 @@ def test_data(case_name, capfd, tmp_path, monkeypatch): monkeypatch.chdir(cwd) wheeldir = cwd.joinpath('wheeldir') wheeldir.mkdir() + output = tmp_path.joinpath('output.txt') if case.get('xfail'): pytest.xfail(case.get('xfail')) + if case.get('skipif') and eval(case.get('skipif')): + pytest.skip(case.get('skipif')) + for filename in case: - file_types = ('.toml', '.py', '.in', '.ini', '.txt') + file_types = ('.toml', '.py', '.in', '.ini', '.txt', '.cfg') if filename.endswith(file_types): cwd.joinpath(filename).write_text(case[filename]) @@ -54,6 +62,8 @@ def test_data(case_name, capfd, tmp_path, monkeypatch): generate_extras=case.get('generate_extras', False), requirement_files=requirement_files, use_build_system=use_build_system, + output=output, + config_settings=case.get('config_settings'), ) except SystemExit as e: assert e.code == case['result'] @@ -69,14 +79,15 @@ def test_data(case_name, capfd, tmp_path, monkeypatch): assert 'expected' in case or 'stderr_contains' in case out, err = capfd.readouterr() + dependencies = output.read_text() if 'expected' in case: expected = case['expected'] if isinstance(expected, list): # at least one of them needs to match - assert any(out == e for e in expected) + assert dependencies in expected else: - assert out == expected + assert dependencies == expected # stderr_contains may be a string or list of strings stderr_contains = case.get('stderr_contains') diff --git a/SPECS/pyproject-rpm-macros.spec b/SPECS/pyproject-rpm-macros.spec index ed22fe8..6108f90 100644 --- a/SPECS/pyproject-rpm-macros.spec +++ b/SPECS/pyproject-rpm-macros.spec @@ -2,9 +2,10 @@ Name: pyproject-rpm-macros Summary: RPM macros for PEP 517 Python packages License: MIT -# Disable tests on RHEL9 as to not pull in the test dependencies -# Specify --with tests to run the tests e.g. on EPEL -%bcond_with tests +%bcond tests 1 +# pytest-xdist and tox are not desired in RHEL +%bcond pytest_xdist %{undefined rhel} +%bcond tox_tests %{undefined rhel} # The idea is to follow the spirit of semver # Given version X.Y.Z: @@ -12,7 +13,7 @@ License: MIT # Increment Y and reset Z when new macros or features are added # Increment Z when this is a bugfix or a cosmetic change # Dropping support for EOL Fedoras is *not* considered a breaking change -Version: 1.6.2 +Version: 1.9.0 Release: 1%{?dist} # Macro files @@ -49,14 +50,26 @@ BuildArch: noarch %if %{with tests} BuildRequires: python3dist(pytest) +%if %{with pytest_xdist} BuildRequires: python3dist(pytest-xdist) +%endif BuildRequires: python3dist(pyyaml) BuildRequires: python3dist(packaging) BuildRequires: python3dist(pip) BuildRequires: python3dist(setuptools) +%if %{with tox_tests} BuildRequires: python3dist(tox-current-env) >= 0.0.6 +%endif BuildRequires: python3dist(wheel) -BuildRequires: (python3dist(toml) if python3-devel < 3.11) +BuildRequires: (python3dist(tomli) if python3 < 3.11) + +# RHEL 9: We also run pytest with Python 3.11 +BuildRequires: python3.11dist(pytest) +BuildRequires: python3.11dist(pyyaml) +BuildRequires: python3.11dist(packaging) +BuildRequires: python3.11dist(pip) +BuildRequires: python3.11dist(setuptools) +BuildRequires: python3.11dist(wheel) %endif # We build on top of those: @@ -120,10 +133,21 @@ install -pm 644 pyproject_construct_toxenv.py %{buildroot}%{_rpmconfigdir}/redha install -pm 644 pyproject_requirements_txt.py %{buildroot}%{_rpmconfigdir}/redhat/ install -pm 644 pyproject_wheel.py %{buildroot}%{_rpmconfigdir}/redhat/ -%if %{with tests} %check +# assert the two signatures of %%pyproject_buildrequires match exactly +signature1="$(grep '^%%pyproject_buildrequires' macros.pyproject | cut -d' ' -f1)" +signature2="$(grep '^%%pyproject_buildrequires' macros.aaa-pyproject-srpm | cut -d' ' -f1)" +test "$signature1" == "$signature2" +# but also assert we are not comparing empty strings +test "$signature1" != "" + +%if %{with tests} export HOSTNAME="rpmbuild" # to speedup tox in network-less mock, see rhbz#1856356 -%pytest -vv --doctest-modules -n auto +%pytest -vv --doctest-modules %{?with_pytest_xdist:-n auto} %{!?with_tox_tests:-k "not tox"} + +# RHEL 9 only: +%global __pytest %{__pytest}-3.11 +%pytest -vv --doctest-modules -k "not tox" # brp-compress is provided as an argument to get the right directory macro expansion %{python3} compare_mandata.py -f %{_rpmconfigdir}/brp-compress @@ -149,6 +173,29 @@ export HOSTNAME="rpmbuild" # to speedup tox in network-less mock, see rhbz#1856 %changelog +* Wed May 31 2023 Maxwell G - 1.9.0-1 +- Allow passing config_settings to the build backend. + +* Wed May 31 2023 Miro Hrončok - 1.8.1-1 +- On Python older than 3.11, use tomli instead of deprecated toml +- Fix literal %% handling in %%{pyproject_files} on RPM 4.19 + +* Tue May 23 2023 Miro Hrončok - 1.8.0-2 +- Rebuilt for ELN dependency changes + +* Thu Apr 27 2023 Miro Hrončok - 1.8.0-1 +- %%pyproject_buildrequires: Add support for self-referential extras requirements +- Deprecate the provisional %%{pyproject_build_lib} macro + See https://lists.fedoraproject.org/archives/list/python-devel@lists.fedoraproject.org/thread/HMLOPAU3RZLXD4BOJHTIPKI3I4U6U7OE/ + +* Fri Mar 31 2023 Miro Hrončok - 1.7.0-1 +- %%pyproject_buildrequires: Redirect stdout to stderr via Shell +- Dependencies are recorded to a text file that is catted at the end + +* Mon Feb 13 2023 Lumír Balhar - 1.6.3-1 +- Remove .dist-info directory at the end of %%pyproject_buildrequires +- An incomplete .dist-info directory in $PWD can confuse tests in %%check + * Wed Feb 08 2023 Lumír Balhar - 1.6.2-1 - Improve detection of lang files