Handle Python Extras in %pyproject_buildrequires on Fedora 33+

There is a slight problem when reporting that a dependency with extra is satisfied.
In fact, we only check the "base" dependency.
This can lead to a problem when a dependency is wrongly assumed as present
and the script proceeds to the "next stage" without restarting --
if the next stage tries to use (import) the missing dependency,
the script would crash.

However, that might be a very unlikely set of events and if such case ever happens,
we'll workaround it or fix it.

macros.pyproject

@@ -79,10 +79,16 @@ echo 'python%{python3_pkgversion}dist(toml)'
 if [ ! -z "%{?python3_version_nodots}" ] && [ %{python3_version_nodots} -lt 38 ]; then
   echo 'python%{python3_pkgversion}dist(importlib-metadata)'
 fi
+# Check if we can generate dependencies on Python extras
+if [ "%{py_dist_name []}" == "[]" ]; then
+  extras_flag=%{?!_python_no_extras_requires:--generate-extras}
+else
+  extras_flag=
+fi
 # setuptools assumes no pre-existing dist-info
 rm -rfv *.dist-info/ >&2
 if [ -f %{__python3} ]; then
-  RPM_TOXENV="%{toxenv}" HOSTNAME="rpmbuild" %{__python3} -I %{_rpmconfigdir}/redhat/pyproject_buildrequires.py --python3_pkgversion %{python3_pkgversion} %{?**}
+  RPM_TOXENV="%{toxenv}" HOSTNAME="rpmbuild" %{__python3} -I %{_rpmconfigdir}/redhat/pyproject_buildrequires.py $extras_flag --python3_pkgversion %{python3_pkgversion} %{?**}
 fi
 }
 

pyproject-rpm-macros.spec

@@ -6,7 +6,7 @@ License:        MIT
 
 # Keep the version at zero and increment only release
 Version:        0
-Release:        24%{?dist}
+Release:        25%{?dist}
 
 # Macro files
 Source001:      macros.pyproject
@@ -88,6 +88,9 @@ export HOSTNAME="rpmbuild"  # to speedup tox in network-less mock, see rhbz#1856
 %license LICENSE
 
 %changelog
+* Fri Aug 14 2020 Miro Hrončok <mhroncok@redhat.com> - 0-25
+- Handle Python Extras in %%pyproject_buildrequires on Fedora 33+
+
 * Tue Aug 11 2020 Miro Hrončok <mhroncok@redhat.com> - 0-24
 - Allow multiple, comma-separated extras in %%pyproject_buildrequires -x
 

pyproject_buildrequires.py

@@ -48,7 +48,7 @@ def hook_call():
 class Requirements:
     """Requirement printer"""
     def __init__(self, get_installed_version, extras='',
-                 python3_pkgversion='3'):
+                 generate_extras=False, python3_pkgversion='3'):
         self.get_installed_version = get_installed_version
 
         if extras:
@@ -58,6 +58,7 @@ class Requirements:
 
         self.missing_requirements = False
 
+        self.generate_extras = generate_extras
         self.python3_pkgversion = python3_pkgversion
 
     def evaluate_all_environamnets(self, requirement):
@@ -86,6 +87,7 @@ class Requirements:
             return
 
         try:
+            # TODO: check if requirements with extras are satisfied
             installed = self.get_installed_version(requirement.name)
         except importlib_metadata.PackageNotFoundError:
             print_err(f'Requirement not satisfied: {requirement_str}')
@@ -93,38 +95,46 @@ class Requirements:
         if installed and installed in requirement.specifier:
             print_err(f'Requirement satisfied: {requirement_str}')
             print_err(f'   (installed: {requirement.name} {installed})')
+            if requirement.extras:
+                print_err(f'   (extras are currently not checked)')
         else:
             self.missing_requirements = True
 
-        together = []
+        if self.generate_extras:
-        for specifier in sorted(
+            extra_names = [f'{name}[{extra}]' for extra in sorted(requirement.extras)]
-            requirement.specifier,
-            key=lambda s: (s.operator, s.version),
-        ):
-            version = canonicalize_version(specifier.version)
-            if not VERSION_RE.fullmatch(str(specifier.version)):
-                raise ValueError(
-                    f'Unknown character in version: {specifier.version}. '
-                    + '(This is probably a bug in pyproject-rpm-macros.)',
-                )
-            if specifier.operator == '!=':
-                lower = python3dist(name, '<', version,
-                                    self.python3_pkgversion)
-                higher = python3dist(name, '>', f'{version}.0',
-                                     self.python3_pkgversion)
-                together.append(
-                    f'({lower} or {higher})'
-                )
-            else:
-                together.append(python3dist(name, specifier.operator, version,
-                                            self.python3_pkgversion))
-        if len(together) == 0:
-            print(python3dist(name,
-                              python3_pkgversion=self.python3_pkgversion))
-        elif len(together) == 1:
-            print(together[0])
         else:
-            print(f"({' and '.join(together)})")
+            extra_names = []
+
+        for name in [name] + extra_names:
+            together = []
+            for specifier in sorted(
+                requirement.specifier,
+                key=lambda s: (s.operator, s.version),
+            ):
+                version = canonicalize_version(specifier.version)
+                if not VERSION_RE.fullmatch(str(specifier.version)):
+                    raise ValueError(
+                        f'Unknown character in version: {specifier.version}. '
+                        + '(This is probably a bug in pyproject-rpm-macros.)',
+                    )
+                if specifier.operator == '!=':
+                    lower = python3dist(name, '<', version,
+                                        self.python3_pkgversion)
+                    higher = python3dist(name, '>', f'{version}.0',
+                                         self.python3_pkgversion)
+                    together.append(
+                        f'({lower} or {higher})'
+                    )
+                else:
+                    together.append(python3dist(name, specifier.operator, version,
+                                                self.python3_pkgversion))
+            if len(together) == 0:
+                print(python3dist(name,
+                                  python3_pkgversion=self.python3_pkgversion))
+            elif len(together) == 1:
+                print(together[0])
+            else:
+                print(f"({' and '.join(together)})")
 
     def check(self, *, source=None):
         """End current pass if any unsatisfied dependencies were output"""
@@ -259,7 +269,7 @@ def python3dist(name, op=None, version=None, python3_pkgversion="3"):
 def generate_requires(
     *, include_runtime=False, toxenv=None, extras='',
     get_installed_version=importlib_metadata.version,  # for dep injection
-    python3_pkgversion="3",
+    generate_extras=False, python3_pkgversion="3",
 ):
     """Generate the BuildRequires for the project in the current directory
 
@@ -267,6 +277,7 @@ def generate_requires(
     """
     requirements = Requirements(
         get_installed_version, extras=extras,
+        generate_extras=generate_extras,
         python3_pkgversion=python3_pkgversion
     )
 
@@ -305,6 +316,10 @@ def main(argv):
         help='comma separated list of "extras" for runtime requirements '
              '(e.g. -x testing,feature-x) (implies --runtime)',
     )
+    parser.add_argument(
+        '--generate-extras', action='store_true',
+        help='Generate build requirements on Python Extras',
+    )
     parser.add_argument(
         '-p', '--python3_pkgversion', metavar='PYTHON3_PKGVERSION',
         default="3", help=('Python version for pythonXdist()'
@@ -329,6 +344,7 @@ def main(argv):
             include_runtime=args.runtime,
             toxenv=args.toxenv,
             extras=args.extras,
+            generate_extras=args.generate_extras,
             python3_pkgversion=args.python3_pkgversion,
         )
     except Exception:

pyproject_buildrequires_testcases.yaml

@@ -66,7 +66,8 @@ Bad character in version:
     requires = ["pkg == 0.$.^.*"]
   except: ValueError
 
-Build system dependencies in pyproject.toml:
+Build system dependencies in pyproject.toml with extras:
+  generate_extras: true
   installed:
     setuptools: 50
     wheel: 1
@@ -74,27 +75,50 @@ Build system dependencies in pyproject.toml:
     [build-system]
     requires = [
         "foo",
+        "bar[baz] > 5",
         "ne!=1",
         "ge>=1.2",
         "le <= 1.2.3",
         "lt < 1.2.3.4      ",
         "    gt > 1.2.3.4.5",
+        "multi[extras1,extras2] == 6.0",
         "combo >2, <5, != 3.0.0",
         "invalid!!ignored",
         "py2 ; python_version < '2.7'",
         "py3 ; python_version > '3.0'",
-        "pkg [extra-currently-ignored]",
     ]
   expected: |
     python3dist(foo)
+    python3dist(bar) > 5
+    python3dist(bar[baz]) > 5
     (python3dist(ne) < 1 or python3dist(ne) > 1.0)
     python3dist(ge) >= 1.2
     python3dist(le) <= 1.2.3
     python3dist(lt) < 1.2.3.4
     python3dist(gt) > 1.2.3.4.5
+    python3dist(multi) == 6
+    python3dist(multi[extras1]) == 6
+    python3dist(multi[extras2]) == 6
     ((python3dist(combo) < 3 or python3dist(combo) > 3.0) and python3dist(combo) < 5 and python3dist(combo) > 2)
     python3dist(py3)
-    python3dist(pkg)
+    python3dist(setuptools) >= 40.8
+    python3dist(wheel)
+  result: 0
+
+Build system dependencies in pyproject.toml without extras:
+  generate_extras: false
+  installed:
+    setuptools: 50
+    wheel: 1
+  pyproject.toml: |
+    [build-system]
+    requires = [
+        "bar[baz] > 5",
+        "multi[extras1,extras2] == 6.0",
+    ]
+  expected: |
+    python3dist(bar) > 5
+    python3dist(multi) == 6
     python3dist(setuptools) >= 40.8
     python3dist(wheel)
   result: 0

test_pyproject_buildrequires.py

@@ -45,6 +45,7 @@ def test_data(case_name, capsys, tmp_path, monkeypatch):
             include_runtime=case.get('include_runtime', False),
             extras=case.get('extras', ''),
             toxenv=case.get('toxenv', None),
+            generate_extras=case.get('generate_extras', False),
         )
     except SystemExit as e:
         assert e.code == case['result']

tests/python-httpbin.spec

@@ -0,0 +1,66 @@
+Name:           python-httpbin
+Version:        0.7.0
+Release:        0%{?dist}
+Summary:        HTTP Request & Response Service, written in Python + Flask
+License:        MIT
+URL:            https://github.com/Runscope/httpbin
+Source0:        %{url}/archive/v%{version}/httpbin-%{version}.tar.gz
+BuildArch:      noarch
+
+BuildRequires:  python3-devel
+BuildRequires:  pyproject-rpm-macros
+
+%description
+This package buildrequires a package with extra: raven[flask].
+
+
+%package -n python3-httpbin
+Summary:            %{summary}
+
+%if 0%{?fedora} < 33 && 0%{?rhel} < 9
+# Old Fedoras don't understand Python extras yet
+# This package needs raven[flask]
+# So we add the transitive dependencies manually:
+BuildRequires:  %{py3_dist blinker flask}
+Requires:       %{py3_dist blinker flask}
+%endif
+
+%description -n python3-httpbin
+%{summary}.
+
+
+%prep
+%autosetup -n httpbin-%{version}
+
+# brotlipy wrapper is not packaged, httpbin works fine with brotli
+sed -i s/brotlipy/brotli/ setup.py
+
+# update test_httpbin.py to reflect new behavior of werkzeug
+sed -i /Content-Length/d test_httpbin.py
+
+
+%generate_buildrequires
+%pyproject_buildrequires -t
+
+
+%build
+%pyproject_wheel
+
+
+%install
+%pyproject_install
+%pyproject_save_files httpbin
+
+
+%check
+%tox
+
+# Internal check for our macros
+# The runtime dependencies contain raven[flask], we assert we got them.
+# The %%tox above also dies without it, but this makes it more explicit
+%{python3} -c 'import blinker, flask'  # transitive deps
+
+
+%files -n python3-httpbin -f %{pyproject_files}
+%doc README*
+%license LICENSE*

tests/tests.yml

@@ -31,6 +31,9 @@
     - openqa_client:
         dir: .
         run: ./mocktest.sh python-openqa_client
+    - httpbin:
+        dir: .
+        run: ./mocktest.sh python-httpbin
     - ldap:
         dir: .
         run: ./mocktest.sh python-ldap