Refactor and add tests

.gitignore

@@ -0,0 +1 @@
+__pycache__/

README.md

@@ -45,6 +45,10 @@ We plan to preserve existing Python flags in shebangs, but the work is not yet f
 The PEPs don't (yet) define a way to specify test dependencies and test runners.
 That means you still need to handle test dependencies and `%check` on your own.
 
+Extras are currently ignored.
+
+Some valid Python version specifiers are not supported.
+
 
 [PEP 517]: https://www.python.org/dev/peps/pep-0517/
 [PEP 518]: https://www.python.org/dev/peps/pep-0518/

macros.pyproject

@@ -23,6 +23,6 @@ echo 'python3dist(packaging)'
 echo 'python3dist(pip) >= 19'
 echo 'python3dist(pytoml)'
 if [ -f %{__python3} ]; then
-  %{__python3} -I %{_rpmconfigdir}/redhat/pyproject_buildrequires.py
+  %{__python3} -I %{_rpmconfigdir}/redhat/pyproject_buildrequires.py %{?*}
 fi
 }

pyproject-rpm-macros.spec

@@ -12,6 +12,9 @@ Source1:        pyproject_buildrequires.py
 Source8:        README.md
 Source9:        LICENSE
 
+Source10:       test_pyproject_buildrequires.py
+Source11:       testcases.yaml
+
 URL:            https://src.fedoraproject.org/rpms/pyproject-rpm-macros
 
 BuildArch:      noarch
@@ -23,6 +26,14 @@ BuildArch:      noarch
 Requires: python3-pip >= 19
 Requires: python3-devel
 
+# Test dependencies
+BuildRequires: python3dist(pytest)
+BuildRequires: python3dist(pyyaml)
+BuildRequires: python3dist(packaging)
+BuildRequires: python3dist(pytoml)
+BuildRequires: python3dist(pip)
+
+
 %description
 This is a provisional implementation of pyproject RPM macros for Fedora 30+.
 These macros are useful for packaging Python projects that use the PEP 517
@@ -45,6 +56,10 @@ mkdir -p %{buildroot}%{_rpmconfigdir}/redhat
 install -m 644 macros.pyproject %{buildroot}%{_rpmmacrodir}/
 install -m 644 pyproject_buildrequires.py %{buildroot}%{_rpmconfigdir}/redhat/
 
+%check
+%{__python3} -m pytest -vv
+
+
 %files
 %{_rpmmacrodir}/macros.pyproject
 %{_rpmconfigdir}/redhat/pyproject_buildrequires.py

pyproject_buildrequires.py

@@ -1,84 +1,216 @@
 import sys
 import importlib
+import argparse
+import functools
+import traceback
+import contextlib
+from io import StringIO
+import subprocess
+import pathlib
+import re
+
+print_err = functools.partial(print, file=sys.stderr)
+
+# Some valid Python version specifiers are not supported.
+# Whitelist characters we can handle.
+VERSION_RE = re.compile('[a-zA-Z0-9.-]+')
+
+class EndPass(Exception):
+    """End current pass of generating requirements"""
 
 try:
     import pytoml
     from packaging.requirements import Requirement, InvalidRequirement
+    from packaging.version import Version
     from packaging.utils import canonicalize_name, canonicalize_version
-except ImportError:
+    import pip
+except ImportError as e:
+    print_err('Import error:', e)
     # already echoed by the %pyproject_buildrequires macro
     sys.exit(0)
 
 
-try:
+@contextlib.contextmanager
-    f = open("pyproject.toml") as f
+def hook_call():
-except FileNotFoundError:
+    captured_out = StringIO()
+    with contextlib.redirect_stdout(captured_out):
+        yield
+    for line in captured_out.getvalue().splitlines():
+        print_err('HOOK STDOUT:', line)
+
+
+class Requirements:
+    """Requirement printer"""
+    def __init__(self, freeze_output):
+        self.installed_packages = {}
+        for line in freeze_output.splitlines():
+            line = line.strip()
+            if line.startswith('#'):
+                continue
+            name, version = line.split('==')
+            self.installed_packages[name.strip()] = Version(version)
+
+        self.missing_requirements = False
+
+    def add(self, requirement_str, *, source=None):
+        """Output a Python-style requirement string as RPM dep"""
+        print_err(f'Handling {requirement_str} from {source}')
+
+        try:
+            requirement = Requirement(requirement_str)
+        except InvalidRequirement as e:
+            print_err(
+                f'"WARNING: Skipping invalid requirement: {requirement_str}\n'
+                + f'    {e}',
+            )
+            return
+
+        name = canonicalize_name(requirement.name)
+        if requirement.marker is not None and not requirement.marker.evaluate():
+            print_err(f'Ignoring alien requirement:', requirement_str)
+            return
+
+        installed = self.installed_packages.get(requirement.name)
+        if installed and installed in requirement.specifier:
+            print_err(f'Requirement satisfied: {requirement_str}')
+            print_err(f'   (installed: {requirement.name} {installed})')
+        else:
+            self.missing_requirements = True
+
+        together = []
+        for specifier in sorted(
+            requirement.specifier,
+            key=lambda s: (s.operator, s.version),
+        ):
+            version = canonicalize_version(specifier.version)
+            print_err(version)
+            if not VERSION_RE.fullmatch(str(specifier.version)):
+                raise ValueError(
+                    f'Unknown character in version: {specifier.version}. '
+                    + '(This is probably a bug in pyproject-rpm-macros.)',
+                )
+            if specifier.operator == "!=":
+                lower = python3dist(name, '<', version)
+                higher = python3dist(name, '>', f'{version}.0')
+                together.append(
+                    f"({lower} or {higher})"
+                )
+            else:
+                together.append(python3dist(name, specifier.operator, version))
+        if len(together) == 0:
+            print(python3dist(name))
+        elif len(together) == 1:
+            print(together[0])
+        else:
+            print(f"({' and '.join(together)})")
+
+    def check(self, *, source=None):
+        """End current pass if any unsatisfied dependencies were output"""
+        if self.missing_requirements:
+            print_err(f'Exiting dependency generation pass: {source}')
+            raise EndPass(source)
+
+    def extend(self, requirement_strs, *, source=None):
+        """add() several requirements"""
+        for req_str in requirement_strs:
+            self.add(req_str, source=source)
+
+def get_backend(requirements):
+    try:
+        f = open('pyproject.toml')
+    except FileNotFoundError:
         pyproject_data = {}
-else:
+    else:
         with f:
             pyproject_data = pytoml.load(f)
 
-    try:
+    buildsystem_data = pyproject_data.get("build-system", {})
-        backend_name = pyproject_data["build-system"]["build-backend"]
+    requirements.extend(
-    except KeyError:
+        buildsystem_data.get("requires", ()),
-        try:
+        source='build-system.requires',
-            import setuptools.build_meta
-        except ImportError:
-            print("python3dist(setuptools) >= 40.8")
-            print("python3dist(wheel)")
-            sys.exit(0)
-
-        backend = setuptools.build_meta
-    else:
-        try:
-            backend = importlib.import_module(backend_name)
-        except ImportError:
-            backend = None
-
-
-requirements = set()
-rpm_requirements = set()
-
-
-def add_requirement(requirement):
-    try:
-        requirements.add(Requirement(requirement))
-    except InvalidRequirement as e:
-        print(
-            f"WARNING: Skipping invalid requirement: {requirement}\n         {e}",
-            file=sys.stderr,
     )
 
+    backend_name = buildsystem_data.get('build-backend')
+    if not backend_name:
+        requirements.add("setuptools >= 40.8", source='default build backend')
+        requirements.add("wheel", source='default build backend')
 
-if "requires" in pyproject_data.get("build-system", {}):
+        backend_name = 'setuptools.build_meta'
-    for requirement in pyproject_data["build-system"]["requires"]:
+
-        add_requirement(requirement)
+    requirements.check(source='build backend')
+
+    backend_path = buildsystem_data.get('backend-path')
+    if backend_path:
+        sys.path.insert(0, backend_path)
+
+    return importlib.import_module(backend_name)
 
 
-get_requires = getattr(backend, "get_requires_for_build_wheel", None)
+def generate_build_requirements(backend, requirements):
-if get_requires:
+    get_requires = getattr(backend, "get_requires_for_build_wheel", None)
-    for requirement in get_requires():
+    if get_requires:
-        add_requirement(requirement)
+        with hook_call():
+            new_reqs = get_requires()
+        requirements.extend(new_reqs, source='get_requires_for_build_wheel')
 
-for requirement in requirements:
+
-    name = canonicalize_name(requirement.name)
+def python3dist(name, op=None, version=None):
-    if requirement.marker is not None and not requirement.marker.evaluate():
+    if op is None:
-        continue
+        if version is not None:
-    together = []
+            raise AssertionError('op and version go together')
-    for specifier in requirement.specifier:
+        return f'python3dist({name})'
-        version = canonicalize_version(specifier.version)
-        if specifier.operator == "!=":
-            together.append(
-                f"(python3dist({name}) < {version} or python3dist({name}) >= {version}.0)"
-            )
     else:
-            together.append(f"python3dist({name}) {specifier.operator} {version}")
+        return f'python3dist({name}) {op} {version}'
-    if len(together) == 0:
-        rpm_requirements.add(f"python3dist({name})")
-    if len(together) == 1:
-        rpm_requirements.add(together[0])
-    elif len(together) > 1:
-        rpm_requirements.add(f"({' and '.join(together)})")
 
 
-print(*sorted(rpm_requirements), sep="\n")
+def generate_requires(freeze_output):
+    requirements = Requirements(freeze_output)
+
+    try:
+        backend = get_backend(requirements)
+        generate_build_requirements(backend, requirements)
+    except EndPass:
+        return 0
+
+
+def main(argv):
+    parser = argparse.ArgumentParser(
+        description='Generate BuildRequires for a Python project.'
+    )
+    parser.add_argument(
+        '--runtime', action='store_true',
+        help='Generate run-time requirements (not implemented)',
+    )
+    parser.add_argument(
+        '--toxenv', metavar='TOXENVS',
+        help='generate test tequirements from tox environment '
+            + '(not implemented; implies --runtime)',
+    )
+    parser.add_argument(
+        '--pyproject-file', default='pyproject.toml',
+        help='override project file (default: pyproject.toml)',
+    )
+
+    args = parser.parse_args(argv)
+    if args.toxenv:
+        args.runtime = True
+    if args.runtime:
+        print_err('--runtime is not implemented')
+        exit(1)
+
+    freeze_output = subprocess.run(
+        ['pip', 'freeze', '--all'],
+        stdout=subprocess.PIPE,
+        check=True,
+    ).stdout
+
+    try:
+        generate_requires(freeze_output)
+    except Exception as e:
+        # Log the traceback explicitly (it's useful debug info)
+        traceback.print_exc()
+        exit(1)
+
+
+if __name__ == '__main__':
+    main(sys.argv[1:])

test_pyproject_buildrequires.py

@@ -0,0 +1,40 @@
+from pathlib import Path
+import io
+
+import pytest
+import yaml
+
+from pyproject_buildrequires import generate_requires
+
+testcases = {}
+with Path(__file__).parent.joinpath('testcases.yaml').open() as f:
+    testcases = yaml.safe_load(f)
+
+
+@pytest.mark.parametrize('case_name', testcases)
+def test_data(case_name, capsys, tmp_path, monkeypatch):
+    case = testcases[case_name]
+
+    cwd = tmp_path.joinpath('cwd')
+    cwd.mkdir()
+    monkeypatch.chdir(cwd)
+
+    if 'pyproject.toml' in case:
+        cwd.joinpath('pyproject.toml').write_text(case['pyproject.toml'])
+
+    if 'setup.py' in case:
+        cwd.joinpath('setup.py').write_text(case['setup.py'])
+
+    try:
+        generate_requires(
+            case['freeze_output'],
+        )
+    except SystemExit as e:
+        assert e.code == case['result']
+    except Exception as e:
+        assert type(e).__name__ == case['except']
+    else:
+        assert 0 == case['result']
+
+        captured = capsys.readouterr()
+        assert captured.out == case['expected']

testcases.yaml

@@ -0,0 +1,119 @@
+No pyproject.toml, nothing installed:
+  freeze_output: |
+    # empty
+  expected: |
+    python3dist(setuptools) >= 40.8
+    python3dist(wheel)
+  result: 0
+
+Nothing installed yet:
+  freeze_output: |
+    # empty
+  pyproject.toml: |
+    # empty
+  expected: |
+    python3dist(setuptools) >= 40.8
+    python3dist(wheel)
+  result: 0
+
+Insufficient version of setuptools:
+  freeze_output: |
+    setuptools==5
+    wheel==1
+  pyproject.toml: |
+    # empty
+  expected: |
+    python3dist(setuptools) >= 40.8
+    python3dist(wheel)
+  result: 0
+
+Empty pyproject.toml, empty setup.py:
+  freeze_output: |
+    setuptools==50
+    wheel==1
+  setup.py: |
+  expected: |
+    python3dist(setuptools) >= 40.8
+    python3dist(wheel)
+    python3dist(wheel)
+  result: 0
+
+Default build system, empty setup.py:
+  freeze_output: |
+    setuptools==50
+    wheel==1
+  pyproject.toml: |
+    # empty
+  setup.py: |
+  expected: |
+    python3dist(setuptools) >= 40.8
+    python3dist(wheel)
+    python3dist(wheel)
+  result: 0
+
+Erroring setup.py:
+  freeze_output: |
+    setuptools==50
+    wheel==1
+  setup.py: |
+    exit(77)
+  result: 77
+
+Bad character in version:
+  freeze_output: |
+  pyproject.toml: |
+    [build-system]
+    requires = ["pkg == 0.$.^.*"]
+  except: ValueError
+
+Build system dependencies in pyproject.toml:
+  freeze_output: |
+    setuptools==50
+    wheel==1
+  pyproject.toml: |
+    [build-system]
+    requires = [
+        "foo",
+        "ne!=1",
+        "ge>=1.2",
+        "le <= 1.2.3",
+        "lt < 1.2.3.4      ",
+        "    gt > 1.2.3.4.5",
+        "combo >2, <5, != 3.0.0",
+        "invalid!!ignored",
+        "py2 ; python_version < '2.7'",
+        "py3 ; python_version > '3.0'",
+        "pkg [extra-currently-ignored]",
+    ]
+  expected: |
+    python3dist(foo)
+    (python3dist(ne) < 1 or python3dist(ne) > 1.0)
+    python3dist(ge) >= 1.2
+    python3dist(le) <= 1.2.3
+    python3dist(lt) < 1.2.3.4
+    python3dist(gt) > 1.2.3.4.5
+    ((python3dist(combo) < 3 or python3dist(combo) > 3.0) and python3dist(combo) < 5 and python3dist(combo) > 2)
+    python3dist(py3)
+    python3dist(pkg)
+    python3dist(setuptools) >= 40.8
+    python3dist(wheel)
+  result: 0
+
+Default build system, dependencies in setup.py:
+  freeze_output: |
+    setuptools==50
+    wheel==1
+  setup.py: |
+    from setuptools import setup
+    setup(
+        name='test',
+        version='0.1',
+        setup_requires=['foo', 'bar!=2'],
+    )
+  expected: |
+    python3dist(setuptools) >= 40.8
+    python3dist(wheel)
+    python3dist(wheel)
+    python3dist(foo)
+    (python3dist(bar) < 2 or python3dist(bar) > 2.0)
+  result: 0