Generate BuildRequires from file

%pyproject_buildrequires macro now accepts multiple file names to load additional dependencies from them. New option -N was added to disable automatical generation of requirements in case package does not use build system. Option -N cannot be used in combination with options -r, -e, -t, -x. Co-authored-by: Miro Hrončok <miro@hroncok.cz> Related: rhbz#1950291
2021-07-08 11:08:04 +00:00 · 2021-07-08 11:08:04 +00:00 · aac0af8401
parent dd25ce59a6
commit aac0af8401
9 changed files with 310 additions and 15 deletions
--- a/README.md
+++ b/README.md
@ -122,6 +122,15 @@ because runtime dependencies are always required for testing.
 [tox]: https://tox.readthedocs.io/
 [tox-current-env]: https://github.com/fedora-python/tox-current-env/

+Additionaly to generated requirements you can supply multiple file names to `%pyproject_buildrequires` macro.
+Dependencies will be loaded from them:
+
+    %pyproject_buildrequires -r requirements/tests.in requirements/docs.in requirements/dev.in
+
+For packages not using build system you can use `-N` to entirely skip automatical
+generation of requirements and install requirements only from manually specified files.
+`-N` option cannot be used in combination with other options mentioned above
+(`-r`, `-e`, `-t`, `-x`).

 Running tox based tests
 -----------------------
--- a/macros.pyproject
+++ b/macros.pyproject
@ -84,18 +84,24 @@ fi
 %toxenv %{default_toxenv}


-%pyproject_buildrequires(rxte:) %{expand:\\\
+%pyproject_buildrequires(rxtNe:) %{expand:\\\
+%{-N:
+%{-r:%{error:The -N and -r options are mutually exclusive}}
+%{-x:%{error:The -N and -x options are mutually exclusive}}
+%{-e:%{error:The -N and -e options are mutually exclusive}}
+%{-t:%{error:The -N and -t options are mutually exclusive}}
+}
 %{-e:%{expand:%global toxenv %(%{__python3} -s %{_rpmconfigdir}/redhat/pyproject_construct_toxenv.py %{?**})}}
 echo 'python%{python3_pkgversion}-devel'
 echo 'python%{python3_pkgversion}dist(pip) >= 19'
 echo 'python%{python3_pkgversion}dist(packaging)'
-if [ -f pyproject.toml ]; then
+%{!-N:if [ -f pyproject.toml ]; then
  echo 'python%{python3_pkgversion}dist(toml)'
 else
  # Note: If the default requirements change, also change them in the script!
  echo 'python%{python3_pkgversion}dist(setuptools) >= 40.8'
  echo 'python%{python3_pkgversion}dist(wheel)'
-fi
+fi}
 # Check if we can generate dependencies on Python extras
 if [ "%{py_dist_name []}" == "[]" ]; then
  extras_flag=%{?!_python_no_extras_requires:--generate-extras}
--- a/pyproject-rpm-macros.spec
+++ b/pyproject-rpm-macros.spec
@ -12,7 +12,7 @@ License:        MIT
 # In other cases, such as backports, increment the point
 # release.
 Version:        0
-Release:        42%{?dist}
+Release:        43%{?dist}

 # Macro files
 Source001:      macros.pyproject
@ -110,6 +110,9 @@ export HOSTNAME="rpmbuild"  # to speedup tox in network-less mock, see rhbz#1856
 %license LICENSE

 %changelog
+* Thu Jul 01 2021 Tomas Hrnciar <thrnciar@redhat.com> - 0-43
+- Generate BuildRequires from file
+
 * Tue Jun 29 2021 Miro Hrončok <mhroncok@redhat.com> - 0-42
 - Don't accidentally treat "~= X.0" requirement as "~= X"
 - Fixes rhzb#1977060
--- a/pyproject_buildrequires.py
+++ b/pyproject_buildrequires.py
@ -11,6 +11,7 @@ import subprocess
 import re
 import tempfile
 import email.parser
+import pathlib

 print_err = functools.partial(print, file=sys.stderr)

@ -228,14 +229,23 @@ def generate_run_requirements(backend, requirements):
        requirements.extend(requires, source=f'wheel metadata: {key}')


-def parse_tox_requires_lines(lines):
+def parse_requirements_lines(lines, path=None):
    packages = []
    for line in lines:
+        line, _, comment = line.partition('#')
+        if comment.startswith('egg='):
+            # not a real comment
+            # e.g. git+https://github.com/monty/spam.git@master#egg=spam&...
+            egg, *_ = comment.strip().partition(' ')
+            egg, *_ = egg.strip().partition('&')
+            line = egg[4:]
        line = line.strip()
        if line.startswith('-r'):
-            path = line[2:]
-            with open(path) as f:
-                packages.extend(parse_tox_requires_lines(f.read().splitlines()))
+            recursed_path = line[2:].strip()
+            if path:
+                recursed_path = path.parent / recursed_path
+            with open(recursed_path) as f:
+                packages.extend(parse_requirements_lines(f.read().splitlines(), recursed_path))
        elif line.startswith('-'):
            print_err(
                f'WARNING: Skipping dependency line: {line}\n'
@ -284,7 +294,7 @@ def generate_tox_requirements(toxenv, requirements):
            r.check_returncode()

        deplines = deps.read().splitlines()
-        packages = parse_tox_requires_lines(deplines)
+        packages = parse_requirements_lines(deplines)
        requirements.add_extras(*extras.read().splitlines())
        requirements.extend(packages,
                            source=f'tox --print-deps-only: {toxenv}')
@ -304,7 +314,7 @@ def python3dist(name, op=None, version=None, python3_pkgversion="3"):
 def generate_requires(
    *, include_runtime=False, toxenv=None, extras=None,
    get_installed_version=importlib.metadata.version,  # for dep injection
-    generate_extras=False, python3_pkgversion="3",
+    generate_extras=False, python3_pkgversion="3", requirement_files=None, use_build_system=True
 ):
    """Generate the BuildRequires for the project in the current directory

@ -317,8 +327,18 @@ def generate_requires(
    )

    try:
-        backend = get_backend(requirements)
-        generate_build_requirements(backend, requirements)
+        if (include_runtime or toxenv) and not use_build_system:
+            raise ValueError('-N option cannot be used in combination with -r, -e, -t, -x options')
+        if requirement_files:
+            for req_file in requirement_files:
+                lines = req_file.read().splitlines()
+                packages = parse_requirements_lines(lines, pathlib.Path(req_file.name))
+                requirements.extend(packages,
+                                    source=f'requirements file {req_file.name}')
+            requirements.check(source='all requirement files')
+        if use_build_system:
+            backend = get_backend(requirements)
+            generate_build_requirements(backend, requirements)
        if toxenv:
            include_runtime = True
            generate_tox_requirements(toxenv, requirements)
@ -360,6 +380,14 @@ def main(argv):
        default="3", help=('Python version for pythonXdist()'
                           'or pythonX.Ydist() requirements'),
    )
+    parser.add_argument(
+        '-N', '--no-use-build-system', dest='use_build_system',
+        action='store_false', help='Use -N to indicate that project does not use any build system',
+    )
+    parser.add_argument(
+       'requirement_files', nargs='*', type=argparse.FileType('r'),
+        help=('Add buildrequires from file'),
+    )

    args = parser.parse_args(argv)

@ -382,6 +410,8 @@ def main(argv):
            extras=args.extras,
            generate_extras=args.generate_extras,
            python3_pkgversion=args.python3_pkgversion,
+            requirement_files=args.requirement_files,
+            use_build_system=args.use_build_system,
        )
    except Exception:
        # Log the traceback explicitly (it's useful debug info)
--- a/pyproject_buildrequires_testcases.yaml
+++ b/pyproject_buildrequires_testcases.yaml
@ -446,3 +446,155 @@ Tox provision satisfied:
    python3dist(toxdep2)
    python3dist(inst)
  result: 0
+
+Default build system, unmet deps in requirements file:
+  installed:
+    setuptools: 50
+    wheel: 1
+  setup.py: |
+    from setuptools import setup
+    setup(
+        name='test',
+        version='0.1',
+    )
+  requirements.txt: |
+    lxml
+    ncclient
+    cryptography
+    paramiko
+    SQLAlchemy
+  requirement_files:
+    - requirements.txt
+  expected: |
+    python3dist(lxml)
+    python3dist(ncclient)
+    python3dist(cryptography)
+    python3dist(paramiko)
+    python3dist(sqlalchemy)
+  result: 0
+
+Default build system, met deps in requirements file:
+  installed:
+    setuptools: 50
+    wheel: 1
+    lxml: 3.9
+    ncclient: 1
+    cryptography: 2
+    paramiko: 1
+    SQLAlchemy: 1.0.90
+  setup.py: |
+    from setuptools import setup
+    setup(
+        name='test',
+        version='0.1',
+    )
+  requirements.txt: |
+    lxml!=3.7.0,>=2.3    # OF-Config
+    ncclient  # OF-Config
+    cryptography!=1.5.2  # Required by paramiko
+    paramiko  # NETCONF, BGP speaker (SSH console)
+    SQLAlchemy>=1.0.10,<1.1.0  # Zebra protocol service
+  requirement_files:
+    - requirements.txt
+  expected: |
+    ((python3dist(lxml) < 3.7 or python3dist(lxml) > 3.7) with python3dist(lxml) >= 2.3)
+    python3dist(ncclient)
+    (python3dist(cryptography) < 1.5.2 or python3dist(cryptography) > 1.5.2)
+    python3dist(paramiko)
+    (python3dist(sqlalchemy) < 1.1 with python3dist(sqlalchemy) >= 1.0.10)
+    python3dist(setuptools) >= 40.8
+    python3dist(wheel)
+    python3dist(wheel)
+  result: 0
+
+With pyproject.toml, requirements file and with -N option:
+  use_build_system: false
+  installed:
+    setuptools: 50
+    wheel: 1
+    toml: 1
+    lxml: 3.9
+    ncclient: 1
+    cryptography: 2
+    paramiko: 1
+    SQLAlchemy: 1.0.90
+  pyproject.toml: |
+    [build-system]
+    requires = [
+        "foo",
+    ]
+    build-backend = "foo.build"
+  requirements.txt: |
+    lxml
+    ncclient
+    cryptography
+    paramiko
+    SQLAlchemy
+    git+https://github.com/monty/spam.git@master#egg=spam
+  requirement_files:
+    - requirements.txt
+  expected: |
+    python3dist(lxml)
+    python3dist(ncclient)
+    python3dist(cryptography)
+    python3dist(paramiko)
+    python3dist(sqlalchemy)
+    python3dist(spam)
+  result: 0
+
+With pyproject.toml, requirements file and without -N option:
+  use_build_system: true
+  installed:
+    setuptools: 50
+    wheel: 1
+    toml: 1
+    lxml: 3.9
+    ncclient: 1
+    cryptography: 2
+    paramiko: 1
+    SQLAlchemy: 1.0.90
+    argcomplete: 1
+    hypothesis: 1
+  pyproject.toml: |
+    [build-system]
+    requires = [
+        "foo",
+    ]
+    build-backend = "foo.build"
+  requirements.txt: |
+    lxml
+    ncclient
+    cryptography
+    paramiko
+    SQLAlchemy
+  requirements1.in: |
+    argcomplete
+    hypothesis
+  requirement_files:
+    - requirements.txt
+    - requirements1.in
+  expected: |
+    python3dist(lxml)
+    python3dist(ncclient)
+    python3dist(cryptography)
+    python3dist(paramiko)
+    python3dist(sqlalchemy)
+    python3dist(argcomplete)
+    python3dist(hypothesis)
+    python3dist(foo)
+  result: 0
+
+Value error if -N and -r arguments are present:
+  installed:
+    # empty
+  include_runtime: true
+  use_build_system: false
+  except: ValueError
+
+Value error if -N and -e arguments are present:
+  installed:
+    # empty
+  toxenv:
+    - py3
+  use_build_system: false
+  except: ValueError
--- a/test_pyproject_buildrequires.py
+++ b/test_pyproject_buildrequires.py
@ -24,8 +24,9 @@ def test_data(case_name, capsys, tmp_path, monkeypatch):
    if case.get('xfail'):
        pytest.xfail(case.get('xfail'))

-    for filename in 'pyproject.toml', 'setup.py', 'tox.ini':
-        if filename in case:
+    for filename in case:
+        file_types = ('.toml', '.py', '.in', '.ini', '.txt')
+        if filename.endswith(file_types):
            cwd.joinpath(filename).write_text(case[filename])

    def get_installed_version(dist_name):
@ -35,7 +36,8 @@ def test_data(case_name, capsys, tmp_path, monkeypatch):
            raise importlib.metadata.PackageNotFoundError(
                f'info not found for {dist_name}'
            )
-
+    requirement_files = case.get('requirement_files', [])
+    requirement_files = [open(f) for f in requirement_files]
    try:
        generate_requires(
            get_installed_version=get_installed_version,
@ -43,6 +45,8 @@ def test_data(case_name, capsys, tmp_path, monkeypatch):
            extras=case.get('extras', []),
            toxenv=case.get('toxenv', None),
            generate_extras=case.get('generate_extras', False),
+            requirement_files=requirement_files,
+            use_build_system=case.get('use_build_system', True),
        )
    except SystemExit as e:
        assert e.code == case['result']
@ -55,3 +59,6 @@ def test_data(case_name, capsys, tmp_path, monkeypatch):

        captured = capsys.readouterr()
        assert captured.out == case['expected']
+    finally:
+        for req in requirement_files:
+            req.close()
--- a/tests/fake-requirements.spec
+++ b/tests/fake-requirements.spec
@ -0,0 +1,27 @@
+Name:           fake-requirements
+Version:        0
+Release:        0%{?dist}
+
+Summary:        ...
+License:        MIT
+
+BuildRequires:  pyproject-rpm-macros
+
+
+%description
+Fake spec file to test %%pyproject_buildrequires -N works as expected
+
+%prep
+cat > requirements.txt <<EOF
+click!=5.0.0,>=4.1 # comment to increase test complexity
+toml>=0.10.0
+EOF
+
+%generate_buildrequires
+%pyproject_buildrequires requirements.txt -N
+
+
+%check
+pip show toml click
+! pip show setuptools
+! pip show wheel
--- a/tests/python-markupsafe.spec
+++ b/tests/python-markupsafe.spec
@ -0,0 +1,55 @@
+Name:           python-markupsafe
+Version:        2.0.1
+Release:        0%{?dist}
+Summary:        Implements a XML/HTML/XHTML Markup safe string for Python
+License:        BSD
+URL:            https://github.com/pallets/markupsafe
+Source0:        %{url}/archive/%{version}/MarkupSafe-%{version}.tar.gz
+
+BuildRequires:  gcc
+BuildRequires:  make
+BuildRequires:  python3-devel
+BuildRequires:  pyproject-rpm-macros
+
+%description
+This package installs test- and docs-requirements from files
+and uses them to run tests and build documentation.
+
+
+%package -n python3-markupsafe
+Summary:        %{summary}
+
+%description -n python3-markupsafe
+...
+
+%prep
+%autosetup -n markupsafe-%{version}
+
+# we don't have pip-tools packaged in Fedora yet
+sed -i /pip-tools/d requirements/dev.in
+
+
+%generate_buildrequires
+# requirements/dev.in recursively includes tests.in and docs.in
+# we also list tests.in manually to verify we can pass multiple arguments,
+# but it should be redundant if this was a real package
+%pyproject_buildrequires -r requirements/dev.in requirements/tests.in
+
+
+%build
+%pyproject_wheel
+%make_build -C docs html SPHINXOPTS='-n %{?_smp_mflags}'
+
+
+%install
+%pyproject_install
+%pyproject_save_files markupsafe
+
+
+%check
+%pytest
+
+
+%files -n python3-markupsafe -f %{pyproject_files}
+%license LICENSE.rst
+%doc CHANGES.rst README.rst
--- a/tests/tests.yml
+++ b/tests/tests.yml
@ -73,6 +73,12 @@
    - setuptools:
        dir: .
        run: ./mocktest.sh python-setuptools
+    - markupsafe:
+        dir: .
+        run: ./mocktest.sh python-markupsafe
+    - fake_requirements:
+        dir: .
+        run: ./mocktest.sh fake-requirements
    required_packages:
    - mock
    - rpmdevtools