%pyproject_buildrequires: Add support for dependency groups (PEP 735), via the -g flag

README.md

@@ -124,6 +124,14 @@ For example, if upstream suggests installing test dependencies with
     %generate_buildrequires
     %pyproject_buildrequires -x testing
 
+For projects that specify test requirements using [PEP 735] dependency groups,
+these can be added using the `-g` flag.
+Multiple groups can be supplied by repeating the flag or as a comma separated list.
+For example, if upstream uses a dependency group called `tests`, the test deps would be generated by:
+
+    %generate_buildrequires
+    %pyproject_buildrequires -g tests
+
 For projects that specify test requirements in their [tox] configuration,
 these can be added using the `-t` flag (default tox environment)
 or the `-e` flag followed by the tox environment.
@@ -153,10 +161,12 @@ such plugins will be BuildRequired as well.
 Not all plugins are guaranteed to play well with [tox-current-env],
 in worst case, patch/sed the requirement out from the tox configuration.
 
-Note that neither `-x` or `-t` can be used with `-R`,
+Note that neither `-x` or `-t` can be used with `-R` or `-N`,
 because runtime dependencies are always required for testing.
 You can only use those options if the build backend  supports the [prepare-metadata-for-build-wheel hook],
 or together with `-p` or `-w`.
+However, using `-g` with `-R` or `-N` is supported because dependency groups don't need to be used for testing
+and can be obtained by reading `pyproject.toml` only.
 
 [tox]: https://tox.readthedocs.io/
 [tox-current-env]: https://github.com/fedora-python/tox-current-env/
@@ -520,6 +530,7 @@ so be prepared for problems.
 [PEP 517]: https://www.python.org/dev/peps/pep-0517/
 [PEP 518]: https://www.python.org/dev/peps/pep-0518/
 [PEP 639]: https://www.python.org/dev/peps/pep-0639/
+[PEP 735]: https://www.python.org/dev/peps/pep-0735/
 [pip's documentation]: https://pip.pypa.io/en/stable/cli/pip_install/#vcs-support
 
 

macros.aaa-pyproject-srpm

@@ -4,7 +4,7 @@
 # this macro will cause the package with the real macro to be installed.
 # When macros.pyproject is installed, it overrides this macro.
 # Note: This needs to maintain the same set of options as the real macro.
-%pyproject_buildrequires(rRxtNwpe:C:) echo 'pyproject-rpm-macros' && exit 0
+%pyproject_buildrequires(rRxtNwpe:g:C:) echo 'pyproject-rpm-macros' && exit 0
 
 
 # Declarative buildsystem, requires RPM 4.20+ to work

macros.pyproject

@@ -158,9 +158,16 @@ fi
 %default_toxenv py%{python3_version_nodots}
 %toxenv %{default_toxenv}
 
+%_pyproject_tomlidep %["%{python3_pkgversion}" == "3"\
+    ? "echo '(python%{python3_pkgversion}dist(tomli) if python%{python3_pkgversion}-devel < 3.11)'"\
+    : "%[v"%{python3_pkgversion}" < v"3.11"\
+       ? "echo 'python%{python3_pkgversion}dist(tomli)'"\
+       : "true # will use tomllib, echo nothing"\
+    ]"\
+  ]
 
 # Note: Keep the options in sync with this macro from macros.aaa-pyproject-srpm
-%pyproject_buildrequires(rRxtNwpe:C:) %{expand:\\\
+%pyproject_buildrequires(rRxtNwpe:g:C:) %{expand:\\\
 %_set_pytest_addopts
 # The default flags expect the package note file to exist
 # see https://bugzilla.redhat.com/show_bug.cgi?id=2097535
@@ -181,6 +188,9 @@ fi
 %{-w:%{error:The -N and -w options are mutually exclusive}}
 %{-p:%{error:The -N and -p options are mutually exclusive}}
 %{-C:%{error:The -N and -C options are mutually exclusive}}
+%{-g:if [ -f pyproject.toml ]; then
+  %_pyproject_tomlidep
+fi}
 }
 %{-w:
 %{-p:%{error:The -w and -p options are mutually exclusive}}
@@ -191,13 +201,7 @@ echo 'python%{python3_pkgversion}-devel'
 echo 'python%{python3_pkgversion}dist(packaging)'
 %{!-N:echo 'python%{python3_pkgversion}dist(pip) >= 19'
 if [ -f pyproject.toml ]; then
-  %["%{python3_pkgversion}" == "3"
+  %_pyproject_tomlidep
-    ? "echo '(python%{python3_pkgversion}dist(tomli) if python%{python3_pkgversion}-devel < 3.11)'"
-    : "%[v"%{python3_pkgversion}" < v"3.11"
-       ? "echo 'python%{python3_pkgversion}dist(tomli)'"
-       : "true # will use tomllib, echo nothing"
-    ]"
-  ]
 elif [ -f setup.py ]; then
   # Note: If the default requirements change, also change them in the script!
   echo 'python%{python3_pkgversion}dist(setuptools) >= 40.8'

pyproject-rpm-macros.spec

@@ -14,7 +14,7 @@ License:        MIT
 #   Increment Y and reset Z when new macros or features are added
 #   Increment Z when this is a bugfix or a cosmetic change
 # Dropping support for EOL Fedoras is *not* considered a breaking change
-Version:        1.15.1
+Version:        1.16.0
 Release:        1%{?dist}
 
 # Macro files
@@ -173,6 +173,10 @@ export HOSTNAME="rpmbuild"  # to speedup tox in network-less mock, see rhbz#1856
 
 
 %changelog
+* Mon Nov 04 2024 Miro Hrončok <mhroncok@redhat.com> - 1.16.0-1
+- %%pyproject_buildrequires: Add support for dependency groups (PEP 735), via the -g flag
+- Fixes: rhbz#2318849
+
 * Thu Oct 03 2024 Karolina Surma <ksurma@redhat.com> - 1.15.1-1
 - Fix handling of self-referencing extras when reading pyproject.toml
 

pyproject_buildrequires.py

@@ -468,6 +468,80 @@ def generate_tox_requirements(toxenv, requirements):
                             source=f'tox --print-deps-only: {toxenv}')
 
 
+def generate_dependency_groups(requested_groups, requirements):
+    """Adapted from https://peps.python.org/pep-0735/#reference-implementation (public domain)"""
+    from collections import defaultdict
+
+    def _normalize_name(name: str) -> str:
+        return re.sub(r"[-_.]+", "-", name).lower()
+
+    def _normalize_group_names(dependency_groups: dict) -> dict:
+        original_names = defaultdict(list)
+        normalized_groups = {}
+
+        for group_name, value in dependency_groups.items():
+            normed_group_name = _normalize_name(group_name)
+            original_names[normed_group_name].append(group_name)
+            normalized_groups[normed_group_name] = value
+
+        errors = []
+        for normed_name, names in original_names.items():
+            if len(names) > 1:
+                errors.append(f"{normed_name} ({', '.join(names)})")
+        if errors:
+            raise ValueError(f"Duplicate dependency group names: {', '.join(errors)}")
+
+        return normalized_groups
+
+    def _resolve_dependency_group(
+        dependency_groups: dict, group: str, past_groups: tuple[str, ...] = ()
+    ) -> list[str]:
+        if group in past_groups:
+            raise ValueError(f"Cyclic dependency group include: {group} -> {past_groups}")
+
+        if group not in dependency_groups:
+            raise LookupError(f"Dependency group '{group}' not found")
+
+        raw_group = dependency_groups[group]
+        if not isinstance(raw_group, list):
+            raise ValueError(f"Dependency group '{group}' is not a list")
+
+        realized_group = []
+        for item in raw_group:
+            if isinstance(item, str):
+                realized_group.append(item)
+            elif isinstance(item, dict):
+                if tuple(item.keys()) != ("include-group",):
+                    raise ValueError(f"Invalid dependency group item: {item}")
+
+                include_group = _normalize_name(next(iter(item.values())))
+                realized_group.extend(
+                    _resolve_dependency_group(
+                        dependency_groups, include_group, past_groups + (group,)
+                    )
+                )
+            else:
+                raise ValueError(f"Invalid dependency group item: {item}")
+
+        return realized_group
+
+    def resolve(dependency_groups: dict, group: str) -> list[str]:
+        if not isinstance(dependency_groups, dict):
+            raise TypeError("Dependency Groups table is not a dict")
+        return _resolve_dependency_group(dependency_groups, _normalize_name(group))
+
+    pyproject_data = load_pyproject()
+    dependency_groups_raw = pyproject_data.get("dependency-groups", {})
+    dependency_groups = _normalize_group_names(dependency_groups_raw)
+
+    for group_names in requested_groups:
+        for group_name in group_names.split(","):
+            requirements.extend(
+                resolve(dependency_groups, group_name),
+                source=f"Dependency group {group_name}",
+            )
+
+
 def python3dist(name, op=None, version=None, python3_pkgversion="3"):
     prefix = f"python{python3_pkgversion}dist"
 
@@ -480,7 +554,7 @@ def python3dist(name, op=None, version=None, python3_pkgversion="3"):
 
 
 def generate_requires(
-    *, include_runtime=False, build_wheel=False, wheeldir=None, toxenv=None, extras=None,
+    *, include_runtime=False, build_wheel=False, wheeldir=None, toxenv=None, extras=None, dependency_groups=None,
     get_installed_version=importlib.metadata.version,  # for dep injection
     generate_extras=False, python3_pkgversion="3", requirement_files=None, use_build_system=True,
     read_pyproject_dependencies=False,
@@ -514,7 +588,9 @@ def generate_requires(
             generate_build_requirements(backend, requirements)
         if toxenv:
             include_runtime = True
-            generate_tox_requirements(toxenv, requirements)
+            generate_tox_requirements(toxenv, requirements)  # TODO extend dependency_groups
+        if dependency_groups:
+            generate_dependency_groups(dependency_groups, requirements)
         if include_runtime:
             generate_run_requirements(backend, requirements, build_wheel=build_wheel,
                 read_pyproject_dependencies=read_pyproject_dependencies, wheeldir=wheeldir)
@@ -559,6 +635,11 @@ def main(argv):
         help='comma separated list of "extras" for runtime requirements '
              '(e.g. -x testing,feature-x) (implies --runtime, can be repeated)',
     )
+    parser.add_argument(
+        '-g', '--dependency-groups', metavar='GROUPS', action='append',
+        help='comma separated list of dependency groups (PEP 735) for requirements '
+             '(e.g. -g tests,docs) (can be repeated)',
+    )
     parser.add_argument(
         '-t', '--tox', action='store_true',
         help=('generate test tequirements from tox environment '
@@ -627,6 +708,7 @@ def main(argv):
             wheeldir=args.wheeldir,
             toxenv=args.toxenv,
             extras=args.extras,
+            dependency_groups=args.dependency_groups,
             generate_extras=args.generate_extras,
             python3_pkgversion=args.python3_pkgversion,
             requirement_files=args.requirement_files,

pyproject_buildrequires_testcases.yaml

@@ -1278,3 +1278,110 @@ pyproject.toml with self-referencing extras:
     python3dist(pytest-rerunfailures)
     python3dist(wurlitzer)
   result: 0
+
+pyproject.toml with dependency-groups not requested:
+  use_build_system: false
+  pyproject.toml: |
+    [dependency-groups]
+    tests = ["pytest>=5", "pytest-mock"]
+    docs = ["sphinx", "python-docs-theme"]
+  expected: "\n"
+  result: 0
+
+pyproject.toml with dependency-groups and build system:
+  skipif: not SETUPTOOLS_60
+  use_build_system: true
+  installed:
+    setuptools: 50
+    wheel: 1
+    tomli: 1
+  dependency_groups:
+    - tests
+  pyproject.toml: |
+    [build-system]
+    requires = ["setuptools"]
+    build-backend = "setuptools.build_meta"
+    [project]
+    name = "my_package"
+    version = "0.1"
+    [dependency-groups]
+    tests = ["pytest>=5", "pytest-mock"]
+    docs = ["sphinx", "python-docs-theme"]
+  expected: |
+    python3dist(setuptools)
+    python3dist(wheel)
+    python3dist(pytest) >= 5
+    python3dist(pytest-mock)
+  result: 0
+
+pyproject.toml with dependency-groups one requested:
+  use_build_system: false
+  dependency_groups:
+    - tests
+  pyproject.toml: |
+    [dependency-groups]
+    tests = ["pytest>=5", "pytest-mock"]
+    docs = ["sphinx", "python-docs-theme"]
+  expected: |
+    python3dist(pytest) >= 5
+    python3dist(pytest-mock)
+  result: 0
+
+pyproject.toml with dependency-groups two requested:
+  use_build_system: false
+  dependency_groups:
+    - tests
+    - docs
+  pyproject.toml: |
+    [dependency-groups]
+    tests = ["pytest>=5", "pytest-mock"]
+    docs = ["sphinx", "python-docs-theme"]
+  expected: |
+    python3dist(pytest) >= 5
+    python3dist(pytest-mock)
+    python3dist(sphinx)
+    python3dist(python-docs-theme)
+  result: 0
+
+pyproject.toml with dependency-groups two requested via comma:
+  use_build_system: false
+  dependency_groups:
+    - tests,docs
+  pyproject.toml: |
+    [dependency-groups]
+    tests = ["pytest>=5", "pytest-mock"]
+    docs = ["sphinx", "python-docs-theme"]
+  expected: |
+    python3dist(pytest) >= 5
+    python3dist(pytest-mock)
+    python3dist(sphinx)
+    python3dist(python-docs-theme)
+  result: 0
+
+pyproject.toml with include-group:
+  use_build_system: false
+  dependency_groups:
+    - tests_docs
+  pyproject.toml: |
+    [dependency-groups]
+    tests = ["pytest>=5", "pytest-mock"]
+    docs = ["sphinx", "python-docs-theme"]
+    typing = ["mypy"]
+    tests-docs = [{include-group = "tests"}, {include-group = "docs"}, "pytest-sphinx"]
+  expected: |
+    python3dist(pytest) >= 5
+    python3dist(pytest-mock)
+    python3dist(sphinx)
+    python3dist(python-docs-theme)
+    python3dist(pytest-sphinx)
+  result: 0
+
+pyproject.toml with dependency-groups nonexisting requested:
+  use_build_system: false
+  dependency_groups:
+    - typing
+  pyproject.toml: |
+    [dependency-groups]
+    tests = ["pytest>=5", "pytest-mock"]
+    docs = ["sphinx", "python-docs-theme"]
+  except: LookupError

test_pyproject_buildrequires.py

@@ -71,6 +71,7 @@ def test_data(case_name, capfd, tmp_path, monkeypatch):
             build_wheel=case.get('build_wheel', False),
             wheeldir=str(wheeldir),
             extras=case.get('extras', []),
+            dependency_groups=case.get('dependency_groups', []),
             toxenv=case.get('toxenv', None),
             generate_extras=case.get('generate_extras', False),
             requirement_files=requirement_files,