Generate BuildRequires from file

%pyproject_buildrequires macro now accepts multiple file names to load
additional dependencies from them.

New option -N was added to disable automatical generation of requirements
in case package does not use build system. Option -N cannot be used in
combination with options -r, -e, -t, -x.

Co-authored-by: Miro Hrončok <miro@hroncok.cz>

README.md

@@ -122,6 +122,15 @@ because runtime dependencies are always required for testing.
 [tox]: https://tox.readthedocs.io/
 [tox-current-env]: https://github.com/fedora-python/tox-current-env/
 
+Additionaly to generated requirements you can supply multiple file names to `%pyproject_buildrequires` macro.
+Dependencies will be loaded from them:
+
+    %pyproject_buildrequires -r requirements/tests.in requirements/docs.in requirements/dev.in
+
+For packages not using build system you can use `-N` to entirely skip automatical
+generation of requirements and install requirements only from manually specified files.
+`-N` option cannot be used in combination with other options mentioned above
+(`-r`, `-e`, `-t`, `-x`).
 
 Running tox based tests
 -----------------------

macros.pyproject

@@ -84,18 +84,24 @@ fi
 %toxenv %{default_toxenv}
 
 
-%pyproject_buildrequires(rxte:) %{expand:\\\
+%pyproject_buildrequires(rxtNe:) %{expand:\\\
+%{-N:
+%{-r:%{error:The -N and -r options are mutually exclusive}}
+%{-x:%{error:The -N and -x options are mutually exclusive}}
+%{-e:%{error:The -N and -e options are mutually exclusive}}
+%{-t:%{error:The -N and -t options are mutually exclusive}}
+}
 %{-e:%{expand:%global toxenv %(%{__python3} -s %{_rpmconfigdir}/redhat/pyproject_construct_toxenv.py %{?**})}}
 echo 'python%{python3_pkgversion}-devel'
 echo 'python%{python3_pkgversion}dist(pip) >= 19'
 echo 'python%{python3_pkgversion}dist(packaging)'
-if [ -f pyproject.toml ]; then
+%{!-N:if [ -f pyproject.toml ]; then
   echo 'python%{python3_pkgversion}dist(toml)'
 else
   # Note: If the default requirements change, also change them in the script!
   echo 'python%{python3_pkgversion}dist(setuptools) >= 40.8'
   echo 'python%{python3_pkgversion}dist(wheel)'
-fi
+fi}
 # Check if we can generate dependencies on Python extras
 if [ "%{py_dist_name []}" == "[]" ]; then
   extras_flag=%{?!_python_no_extras_requires:--generate-extras}

pyproject-rpm-macros.spec

@@ -6,7 +6,7 @@ License:        MIT
 
 # Keep the version at zero and increment only release
 Version:        0
-Release:        42%{?dist}
+Release:        43%{?dist}
 
 # Macro files
 Source001:      macros.pyproject
@@ -104,6 +104,10 @@ export HOSTNAME="rpmbuild"  # to speedup tox in network-less mock, see rhbz#1856
 %license LICENSE
 
 %changelog
+* Thu Jul 01 2021 Tomas Hrnciar <thrnciar@redhat.com> - 0-43
+- Generate BuildRequires from file
+- Fixes: rhbz#1936448
+
 * Tue Jun 29 2021 Miro Hrončok <mhroncok@redhat.com> - 0-42
 - Don't accidentally treat "~= X.0" requirement as "~= X"
 - Fixes rhzb#1977060

pyproject_buildrequires.py

@@ -11,6 +11,7 @@ import subprocess
 import re
 import tempfile
 import email.parser
+import pathlib
 
 print_err = functools.partial(print, file=sys.stderr)
 
@@ -228,14 +229,23 @@ def generate_run_requirements(backend, requirements):
         requirements.extend(requires, source=f'wheel metadata: {key}')
 
 
-def parse_tox_requires_lines(lines):
+def parse_requirements_lines(lines, path=None):
     packages = []
     for line in lines:
+        line, _, comment = line.partition('#')
+        if comment.startswith('egg='):
+            # not a real comment
+            # e.g. git+https://github.com/monty/spam.git@master#egg=spam&...
+            egg, *_ = comment.strip().partition(' ')
+            egg, *_ = egg.strip().partition('&')
+            line = egg[4:]
         line = line.strip()
         if line.startswith('-r'):
-            path = line[2:]
-            with open(path) as f:
-                packages.extend(parse_tox_requires_lines(f.read().splitlines()))
+            recursed_path = line[2:].strip()
+            if path:
+                recursed_path = path.parent / recursed_path
+            with open(recursed_path) as f:
+                packages.extend(parse_requirements_lines(f.read().splitlines(), recursed_path))
         elif line.startswith('-'):
             print_err(
                 f'WARNING: Skipping dependency line: {line}\n'
@@ -284,7 +294,7 @@ def generate_tox_requirements(toxenv, requirements):
             r.check_returncode()
 
         deplines = deps.read().splitlines()
-        packages = parse_tox_requires_lines(deplines)
+        packages = parse_requirements_lines(deplines)
         requirements.add_extras(*extras.read().splitlines())
         requirements.extend(packages,
                             source=f'tox --print-deps-only: {toxenv}')
@@ -304,7 +314,7 @@ def python3dist(name, op=None, version=None, python3_pkgversion="3"):
 def generate_requires(
     *, include_runtime=False, toxenv=None, extras=None,
     get_installed_version=importlib.metadata.version,  # for dep injection
-    generate_extras=False, python3_pkgversion="3",
+    generate_extras=False, python3_pkgversion="3", requirement_files=None, use_build_system=True
 ):
     """Generate the BuildRequires for the project in the current directory
 
@@ -317,6 +327,16 @@ def generate_requires(
     )
 
     try:
+        if (include_runtime or toxenv) and not use_build_system:
+            raise ValueError('-N option cannot be used in combination with -r, -e, -t, -x options')
+        if requirement_files:
+            for req_file in requirement_files:
+                lines = req_file.read().splitlines()
+                packages = parse_requirements_lines(lines, pathlib.Path(req_file.name))
+                requirements.extend(packages,
+                                    source=f'requirements file {req_file.name}')
+            requirements.check(source='all requirement files')
+        if use_build_system:
             backend = get_backend(requirements)
             generate_build_requirements(backend, requirements)
         if toxenv:
@@ -360,6 +380,14 @@ def main(argv):
         default="3", help=('Python version for pythonXdist()'
                            'or pythonX.Ydist() requirements'),
     )
+    parser.add_argument(
+        '-N', '--no-use-build-system', dest='use_build_system',
+        action='store_false', help='Use -N to indicate that project does not use any build system',
+    )
+    parser.add_argument(
+       'requirement_files', nargs='*', type=argparse.FileType('r'),
+        help=('Add buildrequires from file'),
+    )
 
     args = parser.parse_args(argv)
 
@@ -382,6 +410,8 @@ def main(argv):
             extras=args.extras,
             generate_extras=args.generate_extras,
             python3_pkgversion=args.python3_pkgversion,
+            requirement_files=args.requirement_files,
+            use_build_system=args.use_build_system,
         )
     except Exception:
         # Log the traceback explicitly (it's useful debug info)

pyproject_buildrequires_testcases.yaml

@@ -446,3 +446,156 @@ Tox provision satisfied:
     python3dist(toxdep2)
     python3dist(inst)
   result: 0
+
+Default build system, unmet deps in requirements file:
+  installed:
+    setuptools: 50
+    wheel: 1
+  setup.py: |
+    from setuptools import setup
+    setup(
+        name='test',
+        version='0.1',
+    )
+  requirements.txt: |
+    lxml
+    ncclient
+    cryptography
+    paramiko
+    SQLAlchemy
+  requirement_files:
+    - requirements.txt
+  expected: |
+    python3dist(lxml)
+    python3dist(ncclient)
+    python3dist(cryptography)
+    python3dist(paramiko)
+    python3dist(sqlalchemy)
+  result: 0
+
+Default build system, met deps in requirements file:
+  installed:
+    setuptools: 50
+    wheel: 1
+    lxml: 3.9
+    ncclient: 1
+    cryptography: 2
+    paramiko: 1
+    SQLAlchemy: 1.0.90
+  setup.py: |
+    from setuptools import setup
+    setup(
+        name='test',
+        version='0.1',
+    )
+  requirements.txt: |
+    lxml!=3.7.0,>=2.3    # OF-Config
+    ncclient  # OF-Config
+    cryptography!=1.5.2  # Required by paramiko
+    paramiko  # NETCONF, BGP speaker (SSH console)
+    SQLAlchemy>=1.0.10,<1.1.0  # Zebra protocol service
+  requirement_files:
+    - requirements.txt
+  expected: |
+    ((python3dist(lxml) < 3.7 or python3dist(lxml) > 3.7) with python3dist(lxml) >= 2.3)
+    python3dist(ncclient)
+    (python3dist(cryptography) < 1.5.2 or python3dist(cryptography) > 1.5.2)
+    python3dist(paramiko)
+    (python3dist(sqlalchemy) < 1.1 with python3dist(sqlalchemy) >= 1.0.10)
+    python3dist(setuptools) >= 40.8
+    python3dist(wheel)
+    python3dist(wheel)
+  result: 0
+
+With pyproject.toml, requirements file and with -N option:
+  use_build_system: false
+  installed:
+    setuptools: 50
+    wheel: 1
+    toml: 1
+    lxml: 3.9
+    ncclient: 1
+    cryptography: 2
+    paramiko: 1
+    SQLAlchemy: 1.0.90
+  pyproject.toml: |
+    [build-system]
+    requires = [
+        "foo",
+    ]
+    build-backend = "foo.build"
+  requirements.txt: |
+    lxml
+    ncclient
+    cryptography
+    paramiko
+    SQLAlchemy
+    git+https://github.com/monty/spam.git@master#egg=spam
+  requirement_files:
+    - requirements.txt
+  expected: |
+    python3dist(lxml)
+    python3dist(ncclient)
+    python3dist(cryptography)
+    python3dist(paramiko)
+    python3dist(sqlalchemy)
+    python3dist(spam)
+  result: 0
+
+With pyproject.toml, requirements file and without -N option:
+  use_build_system: true
+  installed:
+    setuptools: 50
+    wheel: 1
+    toml: 1
+    lxml: 3.9
+    ncclient: 1
+    cryptography: 2
+    paramiko: 1
+    SQLAlchemy: 1.0.90
+    argcomplete: 1
+    hypothesis: 1
+  pyproject.toml: |
+    [build-system]
+    requires = [
+        "foo",
+    ]
+    build-backend = "foo.build"
+  requirements.txt: |
+    lxml
+    ncclient
+    cryptography
+    paramiko
+    SQLAlchemy
+  requirements1.in: |
+    argcomplete
+    hypothesis
+  requirement_files:
+    - requirements.txt
+    - requirements1.in
+  expected: |
+    python3dist(lxml)
+    python3dist(ncclient)
+    python3dist(cryptography)
+    python3dist(paramiko)
+    python3dist(sqlalchemy)
+    python3dist(argcomplete)
+    python3dist(hypothesis)
+    python3dist(foo)
+  result: 0
+
+Value error if -N and -r arguments are present:
+  installed:
+    # empty
+  include_runtime: true
+  use_build_system: false
+  except: ValueError
+
+Value error if -N and -e arguments are present:
+  installed:
+    # empty
+  toxenv:
+    - py3
+  use_build_system: false
+  except: ValueError
+

test_pyproject_buildrequires.py

@@ -24,8 +24,9 @@ def test_data(case_name, capsys, tmp_path, monkeypatch):
     if case.get('xfail'):
         pytest.xfail(case.get('xfail'))
 
-    for filename in 'pyproject.toml', 'setup.py', 'tox.ini':
-        if filename in case:
+    for filename in case:
+        file_types = ('.toml', '.py', '.in', '.ini', '.txt')
+        if filename.endswith(file_types):
             cwd.joinpath(filename).write_text(case[filename])
 
     def get_installed_version(dist_name):
@@ -35,7 +36,8 @@ def test_data(case_name, capsys, tmp_path, monkeypatch):
             raise importlib.metadata.PackageNotFoundError(
                 f'info not found for {dist_name}'
             )
-
+    requirement_files = case.get('requirement_files', [])
+    requirement_files = [open(f) for f in requirement_files]
     try:
         generate_requires(
             get_installed_version=get_installed_version,
@@ -43,6 +45,8 @@ def test_data(case_name, capsys, tmp_path, monkeypatch):
             extras=case.get('extras', []),
             toxenv=case.get('toxenv', None),
             generate_extras=case.get('generate_extras', False),
+            requirement_files=requirement_files,
+            use_build_system=case.get('use_build_system', True),
         )
     except SystemExit as e:
         assert e.code == case['result']
@@ -55,3 +59,6 @@ def test_data(case_name, capsys, tmp_path, monkeypatch):
 
         captured = capsys.readouterr()
         assert captured.out == case['expected']
+    finally:
+        for req in requirement_files:
+            req.close()

tests/fake-requirements.spec

@@ -0,0 +1,28 @@
+Name:           fake-requirements
+Version:        0
+Release:        0%{?dist}
+
+Summary:        ...
+License:        MIT
+
+BuildRequires:  pyproject-rpm-macros
+
+
+%description 
+Fake spec file to test %%pyproject_buildrequires -N works as expected
+
+%prep
+cat > requirements.txt <<EOF
+click!=5.0.0,>=4.1 # comment to increase test complexity
+toml>=0.10.0
+EOF
+
+%generate_buildrequires
+%pyproject_buildrequires requirements.txt -N
+
+
+%check
+pip show toml click
+! pip show setuptools
+! pip show wheel
+

tests/python-markupsafe.spec

@@ -0,0 +1,55 @@
+Name:           python-markupsafe
+Version:        2.0.1
+Release:        0%{?dist}
+Summary:        Implements a XML/HTML/XHTML Markup safe string for Python
+License:        BSD
+URL:            https://github.com/pallets/markupsafe
+Source0:        %{url}/archive/%{version}/MarkupSafe-%{version}.tar.gz
+
+BuildRequires:  gcc
+BuildRequires:  make
+BuildRequires:  python3-devel
+BuildRequires:  pyproject-rpm-macros
+
+%description
+This package installs test- and docs-requirements from files
+and uses them to run tests and build documentation.
+
+
+%package -n python3-markupsafe
+Summary:        %{summary}
+
+%description -n python3-markupsafe
+...
+
+%prep
+%autosetup -n markupsafe-%{version}
+
+# we don't have pip-tools packaged in Fedora yet
+sed -i /pip-tools/d requirements/dev.in
+
+
+%generate_buildrequires
+# requirements/dev.in recursively includes tests.in and docs.in
+# we also list tests.in manually to verify we can pass multiple arguments,
+# but it should be redundant if this was a real package
+%pyproject_buildrequires -r requirements/dev.in requirements/tests.in
+
+
+%build
+%pyproject_wheel
+%make_build -C docs html SPHINXOPTS='-n %{?_smp_mflags}'
+
+
+%install
+%pyproject_install
+%pyproject_save_files markupsafe
+
+
+%check
+%pytest
+
+
+%files -n python3-markupsafe -f %{pyproject_files}
+%license LICENSE.rst
+%doc CHANGES.rst README.rst

tests/tests.yml

@@ -73,6 +73,12 @@
     - setuptools:
         dir: .
         run: ./mocktest.sh python-setuptools
+    - markupsafe:
+        dir: .
+        run: ./mocktest.sh python-markupsafe
+    - fake_requirements:
+        dir: .
+        run: ./mocktest.sh fake-requirements
     required_packages:
     - mock
     - rpmdevtools