From 307d2bef63eadd1ac3ecede2938f6c58e338e183 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Miro=20Hron=C4=8Dok?= Date: Mon, 4 Nov 2024 13:32:52 +0100 Subject: [PATCH] %pyproject_buildrequires: Add support for dependency groups (PEP 735), via the -g flag --- README.md | 13 ++- macros.aaa-pyproject-srpm | 2 +- macros.pyproject | 20 +++-- pyproject-rpm-macros.spec | 6 +- pyproject_buildrequires.py | 86 +++++++++++++++++++- pyproject_buildrequires_testcases.yaml | 107 +++++++++++++++++++++++++ test_pyproject_buildrequires.py | 1 + 7 files changed, 222 insertions(+), 13 deletions(-) diff --git a/README.md b/README.md index 7ef8c9e..0fb73bd 100644 --- a/README.md +++ b/README.md @@ -124,6 +124,14 @@ For example, if upstream suggests installing test dependencies with %generate_buildrequires %pyproject_buildrequires -x testing +For projects that specify test requirements using [PEP 735] dependency groups, +these can be added using the `-g` flag. +Multiple groups can be supplied by repeating the flag or as a comma separated list. +For example, if upstream uses a dependency group called `tests`, the test deps would be generated by: + + %generate_buildrequires + %pyproject_buildrequires -g tests + For projects that specify test requirements in their [tox] configuration, these can be added using the `-t` flag (default tox environment) or the `-e` flag followed by the tox environment. @@ -153,10 +161,12 @@ such plugins will be BuildRequired as well. Not all plugins are guaranteed to play well with [tox-current-env], in worst case, patch/sed the requirement out from the tox configuration. -Note that neither `-x` or `-t` can be used with `-R`, +Note that neither `-x` or `-t` can be used with `-R` or `-N`, because runtime dependencies are always required for testing. You can only use those options if the build backend supports the [prepare-metadata-for-build-wheel hook], or together with `-p` or `-w`. +However, using `-g` with `-R` or `-N` is supported because dependency groups don't need to be used for testing +and can be obtained by reading `pyproject.toml` only. [tox]: https://tox.readthedocs.io/ [tox-current-env]: https://github.com/fedora-python/tox-current-env/ @@ -520,6 +530,7 @@ so be prepared for problems. [PEP 517]: https://www.python.org/dev/peps/pep-0517/ [PEP 518]: https://www.python.org/dev/peps/pep-0518/ [PEP 639]: https://www.python.org/dev/peps/pep-0639/ +[PEP 735]: https://www.python.org/dev/peps/pep-0735/ [pip's documentation]: https://pip.pypa.io/en/stable/cli/pip_install/#vcs-support diff --git a/macros.aaa-pyproject-srpm b/macros.aaa-pyproject-srpm index 06972fc..1b06ac3 100644 --- a/macros.aaa-pyproject-srpm +++ b/macros.aaa-pyproject-srpm @@ -4,7 +4,7 @@ # this macro will cause the package with the real macro to be installed. # When macros.pyproject is installed, it overrides this macro. # Note: This needs to maintain the same set of options as the real macro. -%pyproject_buildrequires(rRxtNwpe:C:) echo 'pyproject-rpm-macros' && exit 0 +%pyproject_buildrequires(rRxtNwpe:g:C:) echo 'pyproject-rpm-macros' && exit 0 # Declarative buildsystem, requires RPM 4.20+ to work diff --git a/macros.pyproject b/macros.pyproject index a02c29a..b2de0e7 100644 --- a/macros.pyproject +++ b/macros.pyproject @@ -158,9 +158,16 @@ fi %default_toxenv py%{python3_version_nodots} %toxenv %{default_toxenv} +%_pyproject_tomlidep %["%{python3_pkgversion}" == "3"\ + ? "echo '(python%{python3_pkgversion}dist(tomli) if python%{python3_pkgversion}-devel < 3.11)'"\ + : "%[v"%{python3_pkgversion}" < v"3.11"\ + ? "echo 'python%{python3_pkgversion}dist(tomli)'"\ + : "true # will use tomllib, echo nothing"\ + ]"\ + ] # Note: Keep the options in sync with this macro from macros.aaa-pyproject-srpm -%pyproject_buildrequires(rRxtNwpe:C:) %{expand:\\\ +%pyproject_buildrequires(rRxtNwpe:g:C:) %{expand:\\\ %_set_pytest_addopts # The default flags expect the package note file to exist # see https://bugzilla.redhat.com/show_bug.cgi?id=2097535 @@ -181,6 +188,9 @@ fi %{-w:%{error:The -N and -w options are mutually exclusive}} %{-p:%{error:The -N and -p options are mutually exclusive}} %{-C:%{error:The -N and -C options are mutually exclusive}} +%{-g:if [ -f pyproject.toml ]; then + %_pyproject_tomlidep +fi} } %{-w: %{-p:%{error:The -w and -p options are mutually exclusive}} @@ -191,13 +201,7 @@ echo 'python%{python3_pkgversion}-devel' echo 'python%{python3_pkgversion}dist(packaging)' %{!-N:echo 'python%{python3_pkgversion}dist(pip) >= 19' if [ -f pyproject.toml ]; then - %["%{python3_pkgversion}" == "3" - ? "echo '(python%{python3_pkgversion}dist(tomli) if python%{python3_pkgversion}-devel < 3.11)'" - : "%[v"%{python3_pkgversion}" < v"3.11" - ? "echo 'python%{python3_pkgversion}dist(tomli)'" - : "true # will use tomllib, echo nothing" - ]" - ] + %_pyproject_tomlidep elif [ -f setup.py ]; then # Note: If the default requirements change, also change them in the script! echo 'python%{python3_pkgversion}dist(setuptools) >= 40.8' diff --git a/pyproject-rpm-macros.spec b/pyproject-rpm-macros.spec index 3d3ac78..0683637 100644 --- a/pyproject-rpm-macros.spec +++ b/pyproject-rpm-macros.spec @@ -14,7 +14,7 @@ License: MIT # Increment Y and reset Z when new macros or features are added # Increment Z when this is a bugfix or a cosmetic change # Dropping support for EOL Fedoras is *not* considered a breaking change -Version: 1.15.1 +Version: 1.16.0 Release: 1%{?dist} # Macro files @@ -173,6 +173,10 @@ export HOSTNAME="rpmbuild" # to speedup tox in network-less mock, see rhbz#1856 %changelog +* Mon Nov 04 2024 Miro Hrončok - 1.16.0-1 +- %%pyproject_buildrequires: Add support for dependency groups (PEP 735), via the -g flag +- Fixes: rhbz#2318849 + * Thu Oct 03 2024 Karolina Surma - 1.15.1-1 - Fix handling of self-referencing extras when reading pyproject.toml diff --git a/pyproject_buildrequires.py b/pyproject_buildrequires.py index bbe3651..4a371a7 100644 --- a/pyproject_buildrequires.py +++ b/pyproject_buildrequires.py @@ -468,6 +468,80 @@ def generate_tox_requirements(toxenv, requirements): source=f'tox --print-deps-only: {toxenv}') +def generate_dependency_groups(requested_groups, requirements): + """Adapted from https://peps.python.org/pep-0735/#reference-implementation (public domain)""" + from collections import defaultdict + + def _normalize_name(name: str) -> str: + return re.sub(r"[-_.]+", "-", name).lower() + + def _normalize_group_names(dependency_groups: dict) -> dict: + original_names = defaultdict(list) + normalized_groups = {} + + for group_name, value in dependency_groups.items(): + normed_group_name = _normalize_name(group_name) + original_names[normed_group_name].append(group_name) + normalized_groups[normed_group_name] = value + + errors = [] + for normed_name, names in original_names.items(): + if len(names) > 1: + errors.append(f"{normed_name} ({', '.join(names)})") + if errors: + raise ValueError(f"Duplicate dependency group names: {', '.join(errors)}") + + return normalized_groups + + def _resolve_dependency_group( + dependency_groups: dict, group: str, past_groups: tuple[str, ...] = () + ) -> list[str]: + if group in past_groups: + raise ValueError(f"Cyclic dependency group include: {group} -> {past_groups}") + + if group not in dependency_groups: + raise LookupError(f"Dependency group '{group}' not found") + + raw_group = dependency_groups[group] + if not isinstance(raw_group, list): + raise ValueError(f"Dependency group '{group}' is not a list") + + realized_group = [] + for item in raw_group: + if isinstance(item, str): + realized_group.append(item) + elif isinstance(item, dict): + if tuple(item.keys()) != ("include-group",): + raise ValueError(f"Invalid dependency group item: {item}") + + include_group = _normalize_name(next(iter(item.values()))) + realized_group.extend( + _resolve_dependency_group( + dependency_groups, include_group, past_groups + (group,) + ) + ) + else: + raise ValueError(f"Invalid dependency group item: {item}") + + return realized_group + + def resolve(dependency_groups: dict, group: str) -> list[str]: + if not isinstance(dependency_groups, dict): + raise TypeError("Dependency Groups table is not a dict") + return _resolve_dependency_group(dependency_groups, _normalize_name(group)) + + pyproject_data = load_pyproject() + dependency_groups_raw = pyproject_data.get("dependency-groups", {}) + dependency_groups = _normalize_group_names(dependency_groups_raw) + + for group_names in requested_groups: + for group_name in group_names.split(","): + requirements.extend( + resolve(dependency_groups, group_name), + source=f"Dependency group {group_name}", + ) + + def python3dist(name, op=None, version=None, python3_pkgversion="3"): prefix = f"python{python3_pkgversion}dist" @@ -480,7 +554,7 @@ def python3dist(name, op=None, version=None, python3_pkgversion="3"): def generate_requires( - *, include_runtime=False, build_wheel=False, wheeldir=None, toxenv=None, extras=None, + *, include_runtime=False, build_wheel=False, wheeldir=None, toxenv=None, extras=None, dependency_groups=None, get_installed_version=importlib.metadata.version, # for dep injection generate_extras=False, python3_pkgversion="3", requirement_files=None, use_build_system=True, read_pyproject_dependencies=False, @@ -514,7 +588,9 @@ def generate_requires( generate_build_requirements(backend, requirements) if toxenv: include_runtime = True - generate_tox_requirements(toxenv, requirements) + generate_tox_requirements(toxenv, requirements) # TODO extend dependency_groups + if dependency_groups: + generate_dependency_groups(dependency_groups, requirements) if include_runtime: generate_run_requirements(backend, requirements, build_wheel=build_wheel, read_pyproject_dependencies=read_pyproject_dependencies, wheeldir=wheeldir) @@ -559,6 +635,11 @@ def main(argv): help='comma separated list of "extras" for runtime requirements ' '(e.g. -x testing,feature-x) (implies --runtime, can be repeated)', ) + parser.add_argument( + '-g', '--dependency-groups', metavar='GROUPS', action='append', + help='comma separated list of dependency groups (PEP 735) for requirements ' + '(e.g. -g tests,docs) (can be repeated)', + ) parser.add_argument( '-t', '--tox', action='store_true', help=('generate test tequirements from tox environment ' @@ -627,6 +708,7 @@ def main(argv): wheeldir=args.wheeldir, toxenv=args.toxenv, extras=args.extras, + dependency_groups=args.dependency_groups, generate_extras=args.generate_extras, python3_pkgversion=args.python3_pkgversion, requirement_files=args.requirement_files, diff --git a/pyproject_buildrequires_testcases.yaml b/pyproject_buildrequires_testcases.yaml index 63cf5cc..eccc228 100644 --- a/pyproject_buildrequires_testcases.yaml +++ b/pyproject_buildrequires_testcases.yaml @@ -1278,3 +1278,110 @@ pyproject.toml with self-referencing extras: python3dist(pytest-rerunfailures) python3dist(wurlitzer) result: 0 + +pyproject.toml with dependency-groups not requested: + use_build_system: false + pyproject.toml: | + [dependency-groups] + tests = ["pytest>=5", "pytest-mock"] + docs = ["sphinx", "python-docs-theme"] + expected: "\n" + result: 0 + +pyproject.toml with dependency-groups and build system: + skipif: not SETUPTOOLS_60 + use_build_system: true + installed: + setuptools: 50 + wheel: 1 + tomli: 1 + dependency_groups: + - tests + pyproject.toml: | + [build-system] + requires = ["setuptools"] + build-backend = "setuptools.build_meta" + [project] + name = "my_package" + version = "0.1" + [dependency-groups] + tests = ["pytest>=5", "pytest-mock"] + docs = ["sphinx", "python-docs-theme"] + expected: | + python3dist(setuptools) + python3dist(wheel) + python3dist(pytest) >= 5 + python3dist(pytest-mock) + result: 0 + +pyproject.toml with dependency-groups one requested: + use_build_system: false + dependency_groups: + - tests + pyproject.toml: | + [dependency-groups] + tests = ["pytest>=5", "pytest-mock"] + docs = ["sphinx", "python-docs-theme"] + expected: | + python3dist(pytest) >= 5 + python3dist(pytest-mock) + result: 0 + +pyproject.toml with dependency-groups two requested: + use_build_system: false + dependency_groups: + - tests + - docs + pyproject.toml: | + [dependency-groups] + tests = ["pytest>=5", "pytest-mock"] + docs = ["sphinx", "python-docs-theme"] + expected: | + python3dist(pytest) >= 5 + python3dist(pytest-mock) + python3dist(sphinx) + python3dist(python-docs-theme) + result: 0 + +pyproject.toml with dependency-groups two requested via comma: + use_build_system: false + dependency_groups: + - tests,docs + pyproject.toml: | + [dependency-groups] + tests = ["pytest>=5", "pytest-mock"] + docs = ["sphinx", "python-docs-theme"] + expected: | + python3dist(pytest) >= 5 + python3dist(pytest-mock) + python3dist(sphinx) + python3dist(python-docs-theme) + result: 0 + +pyproject.toml with include-group: + use_build_system: false + dependency_groups: + - tests_docs + pyproject.toml: | + [dependency-groups] + tests = ["pytest>=5", "pytest-mock"] + docs = ["sphinx", "python-docs-theme"] + typing = ["mypy"] + tests-docs = [{include-group = "tests"}, {include-group = "docs"}, "pytest-sphinx"] + expected: | + python3dist(pytest) >= 5 + python3dist(pytest-mock) + python3dist(sphinx) + python3dist(python-docs-theme) + python3dist(pytest-sphinx) + result: 0 + +pyproject.toml with dependency-groups nonexisting requested: + use_build_system: false + dependency_groups: + - typing + pyproject.toml: | + [dependency-groups] + tests = ["pytest>=5", "pytest-mock"] + docs = ["sphinx", "python-docs-theme"] + except: LookupError diff --git a/test_pyproject_buildrequires.py b/test_pyproject_buildrequires.py index e5c3b9b..b20ace0 100644 --- a/test_pyproject_buildrequires.py +++ b/test_pyproject_buildrequires.py @@ -71,6 +71,7 @@ def test_data(case_name, capfd, tmp_path, monkeypatch): build_wheel=case.get('build_wheel', False), wheeldir=str(wheeldir), extras=case.get('extras', []), + dependency_groups=case.get('dependency_groups', []), toxenv=case.get('toxenv', None), generate_extras=case.get('generate_extras', False), requirement_files=requirement_files,