diff --git a/README.md b/README.md index 888c144..886c138 100644 --- a/README.md +++ b/README.md @@ -69,8 +69,7 @@ the package's runtime dependencies need to also be included as build requirement Hence, `%pyproject_buildrequires` also generates runtime dependencies by default. -For this to work, the project's build system must support the -[`prepare-metadata-for-build-wheel` hook](https://www.python.org/dev/peps/pep-0517/#prepare-metadata-for-build-wheel). +For this to work, the project's build system must support the [`prepare-metadata-for-build-wheel` hook]. The popular buildsystems (setuptools, flit, poetry) do support it. This behavior can be disabled @@ -80,6 +79,28 @@ using the `-R` flag: %generate_buildrequires %pyproject_buildrequires -R +Alternatively, the runtime dependencies can be obtained by building the wheel and reading the metadata from the built wheel. +This can be enabled by using the `-w` flag. +Support for building wheels with `%pyproject_buildrequires -w` is **provisional** and the behavior might change. +Please subscribe to Fedora's [python-devel list] if you use the option. + + %generate_buildrequires + %pyproject_buildrequires -w + +When this is used, the wheel is going to be built at least twice, +becasue the `%generate_buildrequires` section runs repeatedly. +To avoid accidentally reusing a wheel leaking from a previous (different) build, +it cannot be reused between `%generate_buildrequires` rounds. +Contrarily to that, rebuilding the wheel again in the `%build` section is redundant +and the packager can omit the `%build` section entirely +to reuse the wheel built from the last round of `%generate_buildrequires`. +Be extra careful when attempting to modify the sources after `%pyproject_buildrequires`, +e.g. when running extra commands in the `%build` section: + + %build + cython src/wrong.pyx # this is too late with %%pyproject_buildrequires -w + %pyproject_wheel + For projects that specify test requirements using an [`extra` provide](https://packaging.python.org/specifications/core-metadata/#provides-extra-multiple-use), these can be added using the `-x` flag. @@ -121,9 +142,12 @@ in worst case, patch/sed the requirement out from the tox configuration. Note that both `-x` and `-t` imply `-r`, because runtime dependencies are always required for testing. +You can only use those options if the build backend supports the [`prepare-metadata-for-build-wheel` hook], +or together with `-w`. [tox]: https://tox.readthedocs.io/ [tox-current-env]: https://github.com/fedora-python/tox-current-env/ +[`prepare-metadata-for-build-wheel` hook]: https://www.python.org/dev/peps/pep-0517/#prepare-metadata-for-build-wheel Additionally to generated requirements you can supply multiple file names to `%pyproject_buildrequires` macro. Dependencies will be loaded from them: @@ -133,7 +157,7 @@ Dependencies will be loaded from them: For packages not using build system you can use `-N` to entirely skip automatical generation of requirements and install requirements only from manually specified files. `-N` option cannot be used in combination with other options mentioned above -(`-r`, `-e`, `-t`, `-x`). +(`-r`, `-w`, `-e`, `-t`, `-x`). Running tox based tests ----------------------- diff --git a/macros.pyproject b/macros.pyproject index b6b9747..a3549c5 100644 --- a/macros.pyproject +++ b/macros.pyproject @@ -29,7 +29,7 @@ %_set_pytest_addopts mkdir -p "%{_pyproject_builddir}" CFLAGS="${CFLAGS:-${RPM_OPT_FLAGS}}" LDFLAGS="${LDFLAGS:-${RPM_LD_FLAGS}}" TMPDIR="%{_pyproject_builddir}" \\\ -%{__python3} -m pip wheel --wheel-dir %{_pyproject_wheeldir} --no-deps --use-pep517 --no-build-isolation --disable-pip-version-check --no-clean --progress-bar off --verbose . +%{__python3} -Bs %{_rpmconfigdir}/redhat/pyproject_wheel.py %{_pyproject_wheeldir} } @@ -121,13 +121,21 @@ fi %toxenv %{default_toxenv} -%pyproject_buildrequires(rRxtNe:) %{expand:\\\ -%{-R:%{-r:%{error:The -R and -r options are mutually exclusive}}} +%pyproject_buildrequires(rRxtNwe:) %{expand:\\\ +%_set_pytest_addopts +# The _auto_set_build_flags feature does not do this in %%generate_buildrequires section, +# but we want to get an environment consistent with %%build: +%{?_auto_set_build_flags:%set_build_flags} +%{-R: +%{-r:%{error:The -R and -r options are mutually exclusive}} +%{-w:%{error:The -R and -w options are mutually exclusive}} +} %{-N: %{-r:%{error:The -N and -r options are mutually exclusive}} %{-x:%{error:The -N and -x options are mutually exclusive}} %{-e:%{error:The -N and -e options are mutually exclusive}} %{-t:%{error:The -N and -t options are mutually exclusive}} +%{-w:%{error:The -N and -w options are mutually exclusive}} } %{-e:%{expand:%global toxenv %(%{__python3} -s %{_rpmconfigdir}/redhat/pyproject_construct_toxenv.py %{?**})}} echo 'pyproject-rpm-macros' # we already have this installed, but this way, it's repoqueryable @@ -147,7 +155,9 @@ fi} # setuptools assumes no pre-existing dist-info rm -rfv *.dist-info/ >&2 if [ -f %{__python3} ]; then - RPM_TOXENV="%{toxenv}" HOSTNAME="rpmbuild" %{__python3} -s %{_rpmconfigdir}/redhat/pyproject_buildrequires.py %{?!_python_no_extras_requires:--generate-extras} --python3_pkgversion %{python3_pkgversion} %{?**} + mkdir -p "%{_pyproject_builddir}" + CFLAGS="${CFLAGS:-${RPM_OPT_FLAGS}}" LDFLAGS="${LDFLAGS:-${RPM_LD_FLAGS}}" TMPDIR="%{_pyproject_builddir}" \\\ + RPM_TOXENV="%{toxenv}" HOSTNAME="rpmbuild" %{__python3} -Bs %{_rpmconfigdir}/redhat/pyproject_buildrequires.py %{?!_python_no_extras_requires:--generate-extras} --python3_pkgversion %{python3_pkgversion} --wheeldir %{_pyproject_wheeldir} %{?**} fi } diff --git a/pyproject-rpm-macros.spec b/pyproject-rpm-macros.spec index d4a3bbe..11aa50f 100644 --- a/pyproject-rpm-macros.spec +++ b/pyproject-rpm-macros.spec @@ -12,7 +12,7 @@ License: MIT # Increment Y and reset Z when new macros or features are added # Increment Z when this is a bugfix or a cosmetic change # Dropping support for EOL Fedoras is *not* considered a breaking change -Version: 1.1.0 +Version: 1.2.0 Release: 1%{?dist} # Macro files @@ -25,6 +25,7 @@ Source103: pyproject_convert.py Source104: pyproject_preprocess_record.py Source105: pyproject_construct_toxenv.py Source106: pyproject_requirements_txt.py +Source107: pyproject_wheel.py # Tests Source201: test_pyproject_buildrequires.py @@ -99,6 +100,7 @@ install -m 644 pyproject_save_files.py %{buildroot}%{_rpmconfigdir}/redhat/ install -m 644 pyproject_preprocess_record.py %{buildroot}%{_rpmconfigdir}/redhat/ install -m 644 pyproject_construct_toxenv.py %{buildroot}%{_rpmconfigdir}/redhat/ install -m 644 pyproject_requirements_txt.py %{buildroot}%{_rpmconfigdir}/redhat/ +install -m 644 pyproject_wheel.py %{buildroot}%{_rpmconfigdir}/redhat/ %if %{with tests} %check @@ -118,11 +120,18 @@ export HOSTNAME="rpmbuild" # to speedup tox in network-less mock, see rhbz#1856 %{_rpmconfigdir}/redhat/pyproject_preprocess_record.py %{_rpmconfigdir}/redhat/pyproject_construct_toxenv.py %{_rpmconfigdir}/redhat/pyproject_requirements_txt.py +%{_rpmconfigdir}/redhat/pyproject_wheel.py %doc README.md %license LICENSE %changelog +* Wed Apr 27 2022 Miro Hrončok - 1.2.0-1 +- %%pyproject_buildrequires: Add provisional -w flag for build backends without + prepare_metadata_for_build_wheel hook + When used, the wheel is built in %%pyproject_buildrequires + and information about runtime requires and extras is read from that wheel. + * Tue Apr 12 2022 Miro Hrončok - 1.1.0-1 - %%pyproject_save_files: Support nested directories in dist-info diff --git a/pyproject_buildrequires.py b/pyproject_buildrequires.py index 21e74da..d1f8d81 100644 --- a/pyproject_buildrequires.py +++ b/pyproject_buildrequires.py @@ -1,3 +1,5 @@ +import glob +import io import os import sys import importlib.metadata @@ -11,6 +13,7 @@ import re import tempfile import email.parser import pathlib +import zipfile from pyproject_requirements_txt import convert_requirements_txt @@ -253,21 +256,67 @@ def generate_build_requirements(backend, requirements): requirements.check(source='get_requires_for_build_wheel') -def generate_run_requirements(backend, requirements): +def requires_from_metadata_file(metadata_file): + message = email.parser.Parser().parse(metadata_file, headersonly=True) + return {k: message.get_all(k, ()) for k in ('Requires', 'Requires-Dist')} + + +def generate_run_requirements_hook(backend, requirements): hook_name = 'prepare_metadata_for_build_wheel' prepare_metadata = getattr(backend, hook_name, None) if not prepare_metadata: raise ValueError( - 'build backend cannot provide build metadata ' - + '(incl. runtime requirements) before build' + 'The build backend cannot provide build metadata ' + '(incl. runtime requirements) before build. ' + 'Use the provisional -w flag to build the wheel and parse the metadata from it, ' + 'or use the -R flag not to generate runtime dependencies.' ) with hook_call(): dir_basename = prepare_metadata('.') - with open(dir_basename + '/METADATA') as f: - message = email.parser.Parser().parse(f, headersonly=True) - for key in 'Requires', 'Requires-Dist': - requires = message.get_all(key, ()) - requirements.extend(requires, source=f'wheel metadata: {key}') + with open(dir_basename + '/METADATA') as metadata_file: + for key, requires in requires_from_metadata_file(metadata_file).items(): + requirements.extend(requires, source=f'hook generated metadata: {key}') + + +def find_built_wheel(wheeldir): + wheels = glob.glob(os.path.join(wheeldir, '*.whl')) + if not wheels: + return None + if len(wheels) > 1: + raise RuntimeError('Found multiple wheels in %{_pyproject_wheeldir}, ' + 'this is not supported with %pyproject_buildrequires -w.') + return wheels[0] + + +def generate_run_requirements_wheel(backend, requirements, wheeldir): + # Reuse the wheel from the previous round of %pyproject_buildrequires (if it exists) + wheel = find_built_wheel(wheeldir) + if not wheel: + import pyproject_wheel + returncode = pyproject_wheel.build_wheel(wheeldir=wheeldir, stdout=sys.stderr) + if returncode != 0: + raise RuntimeError('Failed to build the wheel for %pyproject_buildrequires -w.') + wheel = find_built_wheel(wheeldir) + if not wheel: + raise RuntimeError('Cannot locate the built wheel for %pyproject_buildrequires -w.') + + print_err(f'Reading metadata from {wheel}') + with zipfile.ZipFile(wheel) as wheelfile: + for name in wheelfile.namelist(): + if name.count('/') == 1 and name.endswith('.dist-info/METADATA'): + with io.TextIOWrapper(wheelfile.open(name), encoding='utf-8') as metadata_file: + for key, requires in requires_from_metadata_file(metadata_file).items(): + requirements.extend(requires, source=f'built wheel metadata: {key}') + break + else: + raise RuntimeError('Could not find *.dist-info/METADATA in built wheel.') + + +def generate_run_requirements(backend, requirements, *, build_wheel, wheeldir): + if build_wheel: + generate_run_requirements_wheel(backend, requirements, wheeldir) + else: + generate_run_requirements_hook(backend, requirements) def generate_tox_requirements(toxenv, requirements): @@ -326,7 +375,7 @@ def python3dist(name, op=None, version=None, python3_pkgversion="3"): def generate_requires( - *, include_runtime=False, toxenv=None, extras=None, + *, include_runtime=False, build_wheel=False, wheeldir=None, toxenv=None, extras=None, get_installed_version=importlib.metadata.version, # for dep injection generate_extras=False, python3_pkgversion="3", requirement_files=None, use_build_system=True ): @@ -357,7 +406,7 @@ def generate_requires( include_runtime = True generate_tox_requirements(toxenv, requirements) if include_runtime: - generate_run_requirements(backend, requirements) + generate_run_requirements(backend, requirements, build_wheel=build_wheel, wheeldir=wheeldir) except EndPass: return @@ -370,6 +419,15 @@ def main(argv): '-r', '--runtime', action='store_true', default=True, help='Generate run-time requirements (default, disable with -R)', ) + parser.add_argument( + '-w', '--wheel', action='store_true', default=False, + help=('Generate run-time requirements by building the wheel ' + '(useful for build backends without the prepare_metadata_for_build_wheel hook)'), + ) + parser.add_argument( + '--wheeldir', metavar='PATH', default=None, + help='The directory with wheel, used when -w.', + ) parser.add_argument( '-R', '--no-runtime', action='store_false', dest='runtime', help="Don't generate run-time requirements (implied by -N)", @@ -412,6 +470,10 @@ def main(argv): if not args.use_build_system: args.runtime = False + if args.wheel: + if not args.wheeldir: + raise ValueError('--wheeldir must be set when -w.') + if args.toxenv: args.tox = True @@ -427,6 +489,8 @@ def main(argv): try: generate_requires( include_runtime=args.runtime, + build_wheel=args.wheel, + wheeldir=args.wheeldir, toxenv=args.toxenv, extras=args.extras, generate_extras=args.generate_extras, diff --git a/pyproject_buildrequires_testcases.yaml b/pyproject_buildrequires_testcases.yaml index 5e4b5e5..50f9d4e 100644 --- a/pyproject_buildrequires_testcases.yaml +++ b/pyproject_buildrequires_testcases.yaml @@ -268,6 +268,8 @@ Run dependencies with extras (not selected): def main(): setup( + name = "pytest", + version = "6.6.6", setup_requires=["setuptools>=40.0"], # fmt: off extras_require={ @@ -358,6 +360,35 @@ Run dependencies with multiple extras: python3dist(dep1) result: 0 +Run dependencies with extras and build wheel option: + installed: + setuptools: 50 + wheel: 1 + pyyaml: 1 + include_runtime: true + build_wheel: true + extras: + - testing + setup.py: *pytest_setup_py + expected: | + python3dist(setuptools) >= 40.8 + python3dist(wheel) + python3dist(wheel) + python3dist(setuptools) >= 40 + python3dist(py) >= 1.5 + python3dist(six) >= 1.10 + python3dist(setuptools) + python3dist(attrs) >= 17.4 + python3dist(atomicwrites) >= 1 + python3dist(pluggy) >= 0.11 + python3dist(more-itertools) >= 4 + python3dist(argcomplete) + python3dist(hypothesis) >= 3.56 + python3dist(nose) + python3dist(requests) + result: 0 + stderr_contains: "Reading metadata from {wheeldir}/pytest-6.6.6-py3-none-any.whl" + Tox dependencies: installed: setuptools: 50 diff --git a/pyproject_wheel.py b/pyproject_wheel.py new file mode 100644 index 0000000..1936d9c --- /dev/null +++ b/pyproject_wheel.py @@ -0,0 +1,25 @@ +import sys +import subprocess + + +def build_wheel(*, wheeldir, stdout=None): + command = ( + sys.executable, + '-m', 'pip', + 'wheel', + '--wheel-dir', wheeldir, + '--no-deps', + '--use-pep517', + '--no-build-isolation', + '--disable-pip-version-check', + '--no-clean', + '--progress-bar', 'off', + '--verbose', + '.', + ) + cp = subprocess.run(command, stdout=stdout) + return cp.returncode + + +if __name__ == '__main__': + sys.exit(build_wheel(wheeldir=sys.argv[1])) diff --git a/test_pyproject_buildrequires.py b/test_pyproject_buildrequires.py index 15075c5..f9b2650 100644 --- a/test_pyproject_buildrequires.py +++ b/test_pyproject_buildrequires.py @@ -13,12 +13,14 @@ with Path(__file__).parent.joinpath('pyproject_buildrequires_testcases.yaml').op @pytest.mark.parametrize('case_name', testcases) -def test_data(case_name, capsys, tmp_path, monkeypatch): +def test_data(case_name, capfd, tmp_path, monkeypatch): case = testcases[case_name] cwd = tmp_path.joinpath('cwd') cwd.mkdir() monkeypatch.chdir(cwd) + wheeldir = cwd.joinpath('wheeldir') + wheeldir.mkdir() if case.get('xfail'): pytest.xfail(case.get('xfail')) @@ -45,6 +47,8 @@ def test_data(case_name, capsys, tmp_path, monkeypatch): generate_requires( get_installed_version=get_installed_version, include_runtime=case.get('include_runtime', use_build_system), + build_wheel=case.get('build_wheel', False), + wheeldir=str(wheeldir), extras=case.get('extras', []), toxenv=case.get('toxenv', None), generate_extras=case.get('generate_extras', False), @@ -64,7 +68,7 @@ def test_data(case_name, capsys, tmp_path, monkeypatch): # if we ever need to do that, we can remove the check or change it: assert 'expected' in case or 'stderr_contains' in case - out, err = capsys.readouterr() + out, err = capfd.readouterr() if 'expected' in case: assert out == case['expected'] @@ -75,7 +79,7 @@ def test_data(case_name, capsys, tmp_path, monkeypatch): if isinstance(stderr_contains, str): stderr_contains = [stderr_contains] for expected_substring in stderr_contains: - assert expected_substring in err + assert expected_substring.format(**locals()) in err finally: for req in requirement_files: req.close() diff --git a/tests/python-userpath.spec b/tests/python-userpath.spec index eda43be..8b9c071 100644 --- a/tests/python-userpath.spec +++ b/tests/python-userpath.spec @@ -8,15 +8,12 @@ Source: %{pypi_source userpath} BuildArch: noarch BuildRequires: python3-devel -# Manually BuildRequire the runtime dependencies until we have a solution -# for build backends without prepare_metadata_for_build_wheel(): -BuildRequires: python3dist(click) - %description This package uses hatchling as build backend. This package is tested because: - - the prepare_metadata_for_build_wheel hook does not exist + - the prepare_metadata_for_build_wheel hook does not exist, + %%pyproject_buildrequires -w is used https://github.com/ofek/hatch/issues/128 - the licenses are stored in a dist-info subdirectory https://bugzilla.redhat.com/1985340 @@ -39,12 +36,12 @@ sed -Ei '/^(coverage)$/d' requirements-dev.txt %generate_buildrequires -# Cannot use -r (the default) with hatchling, must use -R -%pyproject_buildrequires requirements-dev.txt -R +%pyproject_buildrequires requirements-dev.txt -w -%build -%pyproject_wheel +## %%pyproject_buildrequires -w makes this redundant: +# %%build +# %%pyproject_wheel %install