diff --git a/README.md b/README.md index f7f5d7b..4967930 100644 --- a/README.md +++ b/README.md @@ -79,8 +79,21 @@ using the `-R` flag: %generate_buildrequires %pyproject_buildrequires -R -Alternatively, the runtime dependencies can be obtained by building the wheel and reading the metadata from the built wheel. -This can be enabled by using the `-w` flag. +Alternatively, if the project specifies its dependencies in the pyproject.toml +`[project]` table (as defined in [PEP 621](https://www.python.org/dev/peps/pep-0621/)), +the runtime dependencies can be obtained by reading that metadata. + +This can be enabled by using the `-p` flag. +This flag supports reading both the runtime dependencies, and the selected extras +(see the `-x` flag described below). + +Please note that not all build backends which use pyproject.toml support the +`[project]` table scheme. +For example, poetry-core (at least in 1.9.0) defines package metadata in the +custom `[tool.poetry]` table which is not supported by the `%pyproject_buildrequires` macro. + +Finally, the runtime dependencies can be obtained by building the wheel and reading the metadata from the built wheel. +This can be enabled with the `-w` flag and cannot be combined with `-p`. Support for building wheels with `%pyproject_buildrequires -w` is **provisional** and the behavior might change. Please subscribe to Fedora's [python-devel list] if you use the option. @@ -158,7 +171,7 @@ Dependencies will be loaded from them: For packages not using build system you can use `-N` to entirely skip automatical generation of requirements and install requirements only from manually specified files. `-N` option implies `-R` and cannot be used in combination with other options mentioned above -(`-w`, `-e`, `-t`, `-x`). +(`-w`, `-e`, `-t`, `-x`, `-p`). The `%pyproject_buildrequires` macro also accepts the `-r` flag for backward compatibility; it means "include runtime dependencies" which has been the default since version 0-53. diff --git a/macros.aaa-pyproject-srpm b/macros.aaa-pyproject-srpm index 9bfe84e..06972fc 100644 --- a/macros.aaa-pyproject-srpm +++ b/macros.aaa-pyproject-srpm @@ -4,7 +4,7 @@ # this macro will cause the package with the real macro to be installed. # When macros.pyproject is installed, it overrides this macro. # Note: This needs to maintain the same set of options as the real macro. -%pyproject_buildrequires(rRxtNwe:C:) echo 'pyproject-rpm-macros' && exit 0 +%pyproject_buildrequires(rRxtNwpe:C:) echo 'pyproject-rpm-macros' && exit 0 # Declarative buildsystem, requires RPM 4.20+ to work diff --git a/macros.pyproject b/macros.pyproject index 23ab565..a682ed6 100644 --- a/macros.pyproject +++ b/macros.pyproject @@ -154,7 +154,7 @@ fi # Note: Keep the options in sync with this macro from macros.aaa-pyproject-srpm -%pyproject_buildrequires(rRxtNwe:C:) %{expand:\\\ +%pyproject_buildrequires(rRxtNwpe:C:) %{expand:\\\ %_set_pytest_addopts # The _auto_set_build_flags feature does not do this in %%generate_buildrequires section, # but we want to get an environment consistent with %%build: @@ -168,6 +168,7 @@ fi %{-e:%{error:The -R and -e options are mutually exclusive}} %{-t:%{error:The -R and -t options are mutually exclusive}} %{-w:%{error:The -R and -w options are mutually exclusive}} +%{-p:%{error:The -R and -p options are mutually exclusive}} } %{-N: %{-r:%{error:The -N and -r options are mutually exclusive}} @@ -175,8 +176,12 @@ fi %{-e:%{error:The -N and -e options are mutually exclusive}} %{-t:%{error:The -N and -t options are mutually exclusive}} %{-w:%{error:The -N and -w options are mutually exclusive}} +%{-p:%{error:The -N and -p options are mutually exclusive}} %{-C:%{error:The -N and -C options are mutually exclusive}} } +%{-w: +%{-p:%{error:The -w and -p options are mutually exclusive}} +} %{-e:%{expand:%global toxenv %(%{__python3} -s %{_rpmconfigdir}/redhat/pyproject_construct_toxenv.py %{?**})}} echo 'pyproject-rpm-macros' # first stdout line matches the implementation in macros.aaa-pyproject-srpm echo 'python%{python3_pkgversion}-devel' diff --git a/pyproject-rpm-macros.spec b/pyproject-rpm-macros.spec index bf119b0..ca54355 100644 --- a/pyproject-rpm-macros.spec +++ b/pyproject-rpm-macros.spec @@ -14,7 +14,7 @@ License: MIT # Increment Y and reset Z when new macros or features are added # Increment Z when this is a bugfix or a cosmetic change # Dropping support for EOL Fedoras is *not* considered a breaking change -Version: 1.14.0 +Version: 1.15.0 Release: 1%{?dist} # Macro files @@ -196,6 +196,10 @@ export HOSTNAME="rpmbuild" # to speedup tox in network-less mock, see rhbz#1856 %changelog +* Tue Sep 17 2024 Karolina Surma - 1.15.0-1 +- Add a possibility to read runtime requirements from pyproject.toml [project] table +- Fixes: rhbz#2261939 + * Tue Jul 23 2024 Miro Hrončok - 1.14.0-1 - Add a provisional RPM Declarative Buildsystem (RPM 4.20+) diff --git a/pyproject_buildrequires.py b/pyproject_buildrequires.py index 515da95..9913e98 100644 --- a/pyproject_buildrequires.py +++ b/pyproject_buildrequires.py @@ -10,6 +10,7 @@ import subprocess import re import tempfile import email.parser +import functools import pathlib import zipfile @@ -34,6 +35,7 @@ def print_err(*args, **kwargs): try: + from packaging.markers import Marker from packaging.requirements import Requirement, InvalidRequirement from packaging.utils import canonicalize_name except ImportError as e: @@ -99,7 +101,7 @@ class Requirements: return True return False - def add(self, requirement_str, *, package_name=None, source=None): + def add(self, requirement_str, *, package_name=None, source=None, extra=None): """Output a Python-style requirement string as RPM dep""" print_err(f'Handling {requirement_str} from {source}') @@ -118,6 +120,13 @@ class Requirements: ) name = canonicalize_name(requirement.name) + + if extra is not None: + extra_str = f'extra == "{extra}"' + if requirement.marker is not None: + extra_str = f'({requirement.marker}) and {extra_str}' + requirement.marker = Marker(extra_str) + if (requirement.marker is not None and not self.evaluate_all_environments(requirement)): print_err(f'Ignoring alien requirement:', requirement_str) @@ -215,7 +224,8 @@ def toml_load(opened_binary_file): return tomllib.load(opened_binary_file) -def get_backend(requirements): +@functools.cache +def load_pyproject(): try: f = open('pyproject.toml', 'rb') except FileNotFoundError: @@ -223,6 +233,11 @@ def get_backend(requirements): else: with f: pyproject_data = toml_load(f) + return pyproject_data + + +def get_backend(requirements): + pyproject_data = load_pyproject() buildsystem_data = pyproject_data.get('build-system', {}) requirements.extend( @@ -310,7 +325,9 @@ def generate_run_requirements_hook(backend, requirements): raise ValueError( 'The build backend cannot provide build metadata ' '(incl. runtime requirements) before build. ' - 'Use the provisional -w flag to build the wheel and parse the metadata from it, ' + 'If the dependencies are specified in the pyproject.toml [project] ' + 'table, you can use the -p flag to read them.' + 'Alternatively, use the provisional -w flag to build the wheel and parse the metadata from it, ' 'or use the -R flag not to generate runtime dependencies.' ) dir_basename = prepare_metadata('.', config_settings=requirements.config_settings) @@ -368,8 +385,35 @@ def generate_run_requirements_wheel(backend, requirements, wheeldir): raise RuntimeError('Could not find *.dist-info/METADATA in built wheel.') -def generate_run_requirements(backend, requirements, *, build_wheel, wheeldir): - if build_wheel: +def generate_run_requirements_pyproject(requirements): + pyproject_data = load_pyproject() + + if not (project_table := pyproject_data.get('project', {})): + raise ValueError('Could not find the [project] table in pyproject.toml.') + + dynamic_fields = project_table.get('dynamic', []) + if 'dependencies' in dynamic_fields or 'optional-dependencies' in dynamic_fields: + raise ValueError('Could not read the dependencies or optional-dependencies ' + 'from the [project] table in pyproject.toml, as the field is dynamic.') + + dependencies = project_table.get('dependencies', []) + name = project_table.get('name') + requirements.extend(dependencies, + package_name=name, + source=f'pyproject.toml generated metadata: [dependencies] ({name})') + + optional_dependencies = project_table.get('optional-dependencies', {}) + for extra, dependencies in optional_dependencies.items(): + requirements.extend(dependencies, + package_name=name, + source=f'pyproject.toml generated metadata: [optional-dependencies] {extra} ({name})', + extra=extra) + + +def generate_run_requirements(backend, requirements, *, build_wheel, read_pyproject_dependencies, wheeldir): + if read_pyproject_dependencies: + generate_run_requirements_pyproject(requirements) + elif build_wheel: generate_run_requirements_wheel(backend, requirements, wheeldir) else: generate_run_requirements_hook(backend, requirements) @@ -434,6 +478,7 @@ def generate_requires( *, include_runtime=False, build_wheel=False, wheeldir=None, toxenv=None, extras=None, get_installed_version=importlib.metadata.version, # for dep injection generate_extras=False, python3_pkgversion="3", requirement_files=None, use_build_system=True, + read_pyproject_dependencies=False, output, config_settings=None, ): """Generate the BuildRequires for the project in the current directory @@ -450,8 +495,8 @@ def generate_requires( ) try: - if (include_runtime or toxenv) and not use_build_system: - raise ValueError('-N option cannot be used in combination with -r, -e, -t, -x options') + if (include_runtime or toxenv or read_pyproject_dependencies) and not use_build_system: + raise ValueError('-N option cannot be used in combination with -r, -e, -t, -x, -p options') if requirement_files: for req_file in requirement_files: requirements.extend( @@ -466,7 +511,8 @@ def generate_requires( include_runtime = True generate_tox_requirements(toxenv, requirements) if include_runtime: - generate_run_requirements(backend, requirements, build_wheel=build_wheel, wheeldir=wheeldir) + generate_run_requirements(backend, requirements, build_wheel=build_wheel, + read_pyproject_dependencies=read_pyproject_dependencies, wheeldir=wheeldir) except EndPass: return finally: @@ -493,7 +539,7 @@ def main(argv): help=argparse.SUPPRESS, ) parser.add_argument( - '-p', '--python3_pkgversion', metavar='PYTHON3_PKGVERSION', + '--python3_pkgversion', metavar='PYTHON3_PKGVERSION', default="3", help=argparse.SUPPRESS, ) parser.add_argument( @@ -523,6 +569,11 @@ def main(argv): help=('Generate run-time requirements by building the wheel ' '(useful for build backends without the prepare_metadata_for_build_wheel hook)'), ) + parser.add_argument( + '-p', '--read-pyproject-dependencies', action='store_true', default=False, + help=('Generate dependencies from [project] table of pyproject.toml ' + 'instead of calling prepare_metadata_for_build_wheel hook)'), + ) parser.add_argument( '-R', '--no-runtime', action='store_false', dest='runtime', help="Don't generate run-time requirements (implied by -N)", @@ -575,6 +626,7 @@ def main(argv): python3_pkgversion=args.python3_pkgversion, requirement_files=args.requirement_files, use_build_system=args.use_build_system, + read_pyproject_dependencies=args.read_pyproject_dependencies, output=args.output, config_settings=parse_config_settings_args(args.config_settings), ) diff --git a/pyproject_buildrequires_testcases.yaml b/pyproject_buildrequires_testcases.yaml index 892e3c3..315e52e 100644 --- a/pyproject_buildrequires_testcases.yaml +++ b/pyproject_buildrequires_testcases.yaml @@ -906,7 +906,7 @@ pyproject.toml with runtime dependencies and partially selected extras: tomli: 1 extras: - tests - pyproject.toml: | + pyproject.toml: &pyproject_with_extras | [build-system] requires = ["setuptools"] build-backend = "setuptools.build_meta" @@ -1062,3 +1062,187 @@ config_settings: expected: | python3dist(test-config-setting) result: 0 + +pyproject.toml with runtime dependencies read from it: + skipif: not SETUPTOOLS_60 + read_pyproject_dependencies: true + installed: + setuptools: 50 + wheel: 1 + tomli: 1 + pyproject.toml: | + [build-system] + requires = ["setuptools"] + build-backend = "setuptools.build_meta" + [project] + name = "my_package" + version = "0.1" + dependencies = [ + "foo", + 'importlib-metadata; python_version<"3.8"', + ] + expected: | + python3dist(setuptools) + python3dist(wheel) + python3dist(foo) + result: 0 + +pyproject.toml with extras - only runtime dependencies read from it: + skipif: not SETUPTOOLS_60 + read_pyproject_dependencies: true + installed: + setuptools: 50 + wheel: 1 + tomli: 1 + pyproject.toml: *pyproject_with_extras + expected: | + python3dist(setuptools) + python3dist(wheel) + python3dist(foo) + result: 0 + +pyproject.toml with runtime dependencies and partially selected extras read from it: + skipif: not SETUPTOOLS_60 + read_pyproject_dependencies: true + installed: + setuptools: 50 + wheel: 1 + tomli: 1 + extras: + - tests + pyproject.toml: *pyproject_with_extras + expected: | + python3dist(setuptools) + python3dist(wheel) + python3dist(foo) + python3dist(pytest) >= 5 + python3dist(pytest-mock) + result: 0 + +pyproject.toml with runtime dependencies and all extras read from it: + skipif: not SETUPTOOLS_60 + read_pyproject_dependencies: true + installed: + setuptools: 50 + wheel: 1 + tomli: 1 + extras: + - tests + - docs + pyproject.toml: *pyproject_with_extras + expected: | + python3dist(setuptools) + python3dist(wheel) + python3dist(foo) + python3dist(pytest) >= 5 + python3dist(pytest-mock) + python3dist(sphinx) + python3dist(python-docs-theme) + result: 0 + +pyproject.toml without dependencies: + skipif: not SETUPTOOLS_60 + read_pyproject_dependencies: true + installed: + setuptools: 50 + wheel: 1 + tomli: 1 + pyproject.toml: | + [build-system] + requires = ["setuptools"] + build-backend = "setuptools.build_meta" + [project] + name = "my_package" + version = "0.1" + expected: | + python3dist(setuptools) + python3dist(wheel) + result: 0 + +pyproject.toml without project table: + skipif: not SETUPTOOLS_60 + read_pyproject_dependencies: true + installed: + setuptools: 50 + wheel: 1 + pyproject.toml: | + [build-system] + requires = ["setuptools"] + build-backend = "setuptools.build_meta" + except: ValueError + +no pyproject.toml: + read_pyproject_dependencies: true + installed: + setuptools: 50 + wheel: 1 + except: FileNotFoundError + +pyproject.toml with dynamic dependencies: + skipif: not SETUPTOOLS_60 + read_pyproject_dependencies: true + installed: + setuptools: 50 + wheel: 1 + tomli: 1 + pyproject.toml: | + [build-system] + requires = ["setuptools"] + build-backend = "setuptools.build_meta" + [project] + name = "my_package" + version = "0.1" + dynamic = ["dependencies"] + [tool.setuptools.dynamic] + dependencies = { file = ["deps.txt"] } + deps.txt: | + foo < 7.0 + sphinx + except: ValueError + +pyproject.toml with dynamic optional dependencies: + skipif: not SETUPTOOLS_60 + read_pyproject_dependencies: true + installed: + setuptools: 50 + wheel: 1 + tomli: 1 + extras: + - docs + pyproject.toml: | + [build-system] + requires = ["setuptools"] + build-backend = "setuptools.build_meta" + [project] + name = "my_package" + version = "0.1" + dynamic = ["optional-dependencies"] + [tool.setuptools.dynamic.optional-dependencies.docs] + file = ["deps.txt"] + deps.txt: | + sphinx~=7.0.1 + except: ValueError + +pyproject.toml with dynamic table and no deps: + skipif: not SETUPTOOLS_60 + read_pyproject_dependencies: true + installed: + setuptools: 50 + wheel: 1 + tomli: 1 + pyproject.toml: | + [build-system] + requires = ["setuptools"] + build-backend = "setuptools.build_meta" + [project] + name = "my_package" + version = "0.1" + dynamic = ["readme"] + [tool.setuptools.dynamic] + readme = { file = ["readme.txt"] } + readme.txt: | + nothing interesting here + expected: | + python3dist(setuptools) + python3dist(wheel) + result: 0 diff --git a/test_pyproject_buildrequires.py b/test_pyproject_buildrequires.py index 0fa07db..e5c3b9b 100644 --- a/test_pyproject_buildrequires.py +++ b/test_pyproject_buildrequires.py @@ -6,7 +6,7 @@ import pytest import setuptools import yaml -from pyproject_buildrequires import generate_requires +from pyproject_buildrequires import generate_requires, load_pyproject SETUPTOOLS_VERSION = packaging.version.parse(setuptools.__version__) SETUPTOOLS_60 = SETUPTOOLS_VERSION >= packaging.version.parse('60') @@ -16,6 +16,18 @@ with Path(__file__).parent.joinpath('pyproject_buildrequires_testcases.yaml').op testcases = yaml.safe_load(f) +@pytest.fixture(autouse=True) +def clear_pyproject_data(): + """ + Clear pyproject data before each test. + In reality we build one RPM package at a time, so we can keep the once-loaded + pyproject.toml contents. + When testing, the cached data would leak the once-loaded data to all the + following test cases. + """ + load_pyproject.cache_clear() + + @pytest.mark.parametrize('case_name', testcases) def test_data(case_name, capfd, tmp_path, monkeypatch): case = testcases[case_name] @@ -51,6 +63,7 @@ def test_data(case_name, capfd, tmp_path, monkeypatch): requirement_files = case.get('requirement_files', []) requirement_files = [open(f) for f in requirement_files] use_build_system = case.get('use_build_system', True) + read_pyproject_dependencies = case.get('read_pyproject_dependencies', False) try: generate_requires( get_installed_version=get_installed_version, @@ -62,6 +75,7 @@ def test_data(case_name, capfd, tmp_path, monkeypatch): generate_extras=case.get('generate_extras', False), requirement_files=requirement_files, use_build_system=use_build_system, + read_pyproject_dependencies=read_pyproject_dependencies, output=output, config_settings=case.get('config_settings'), ) diff --git a/tests/python-markdown-it-py.spec b/tests/python-markdown-it-py.spec new file mode 100644 index 0000000..5c34b21 --- /dev/null +++ b/tests/python-markdown-it-py.spec @@ -0,0 +1,50 @@ +Name: python-markdown-it-py +Version: 3.0.0 +Release: 0%{?dist} +Summary: Python port of markdown-it +License: MIT +URL: https://github.com/executablebooks/markdown-it-py +Source0: %{url}/archive/v%{version}/markdown-it-py-%{version}.tar.gz +BuildArch: noarch + +BuildRequires: python3-devel + +%description +This package tests generating of runtime requirements from pyproject.toml +Upstream has got many more extras than we package, +so it's a good example to test it's filtered correctly. + +%package -n python3-markdown-it-py +Summary: %{summary} + +%description -n python3-markdown-it-py +... + +%pyproject_extras_subpkg -n python3-markdown-it-py linkify + +%prep +%autosetup -p1 -n markdown-it-py-%{version} + +%generate_buildrequires +%pyproject_buildrequires -x testing,linkify -p + +%build +%pyproject_wheel + +%install +%pyproject_install +%pyproject_save_files markdown_it -L + +%check +# sphinx-copybutton is in [rtd] extra, should not appear +grep "python3dist(sphinx-copybutton)" %_pyproject_buildrequires && exit 1 || true +# "pytest-benchmark" is in [benchmarking] extra, should not appear +grep "python3dist(pytest-benchmark)" %_pyproject_buildrequires && exit 1 || true +# "pytest-regressions" is in [testing] extra, should appear +grep "python3dist(pytest-regressions)" %_pyproject_buildrequires +# "linkify-it-py" is in [linkify] extra, should appear +grep "python3dist(linkify-it-py)" %_pyproject_buildrequires + + +%files -n python3-markdown-it-py -f %{pyproject_files} +%{_bindir}/markdown-it diff --git a/tests/tests.yml b/tests/tests.yml index 1dccc6e..ae696b4 100644 --- a/tests/tests.yml +++ b/tests/tests.yml @@ -95,6 +95,9 @@ - userpath: dir: . run: ./mocktest.sh python-userpath + - markdown_it_py: + dir: . + run: ./mocktest.sh python-markdown-it-py - double_install: dir: . run: ./mocktest.sh double-install