290941c5f3
The PEP 517 shows an example backend-path like this:
[build-system]
# Defined by PEP 518:
requires = ["flit"]
# Defined by this PEP:
build-backend = "local_backend"
backend-path = ["backend"]
https://www.python.org/dev/peps/pep-0517/#source-trees
See that backend-path is a list. Our code previously only supported string path.
Obviously a string path is wrong, but we keep it to support projects that have
made the mistake, such as flit-core.
Add a small integration test for both cases.
Note that the new spec files deliberately don't do much, to save CI time.
371 lines
12 KiB
Python
371 lines
12 KiB
Python
import os
|
|
import sys
|
|
import importlib
|
|
import argparse
|
|
import functools
|
|
import traceback
|
|
import contextlib
|
|
from io import StringIO
|
|
import subprocess
|
|
import re
|
|
import tempfile
|
|
import email.parser
|
|
|
|
print_err = functools.partial(print, file=sys.stderr)
|
|
|
|
# Some valid Python version specifiers are not supported.
|
|
# Whitelist characters we can handle.
|
|
VERSION_RE = re.compile('[a-zA-Z0-9.-]+')
|
|
|
|
|
|
class EndPass(Exception):
|
|
"""End current pass of generating requirements"""
|
|
|
|
|
|
try:
|
|
from packaging.requirements import Requirement, InvalidRequirement
|
|
from packaging.utils import canonicalize_name, canonicalize_version
|
|
try:
|
|
import importlib.metadata as importlib_metadata
|
|
except ImportError:
|
|
import importlib_metadata
|
|
except ImportError as e:
|
|
print_err('Import error:', e)
|
|
# already echoed by the %pyproject_buildrequires macro
|
|
sys.exit(0)
|
|
|
|
# uses packaging, needs to be imported after packaging is verified to be present
|
|
from pyproject_convert import convert
|
|
|
|
|
|
@contextlib.contextmanager
|
|
def hook_call():
|
|
captured_out = StringIO()
|
|
with contextlib.redirect_stdout(captured_out):
|
|
yield
|
|
for line in captured_out.getvalue().splitlines():
|
|
print_err('HOOK STDOUT:', line)
|
|
|
|
|
|
class Requirements:
|
|
"""Requirement printer"""
|
|
def __init__(self, get_installed_version, extras=None,
|
|
generate_extras=False, python3_pkgversion='3'):
|
|
self.get_installed_version = get_installed_version
|
|
self.extras = set()
|
|
|
|
if extras:
|
|
for extra in extras:
|
|
self.add_extras(*extra.split(','))
|
|
|
|
self.missing_requirements = False
|
|
|
|
self.generate_extras = generate_extras
|
|
self.python3_pkgversion = python3_pkgversion
|
|
|
|
def add_extras(self, *extras):
|
|
self.extras |= set(e.strip() for e in extras)
|
|
|
|
@property
|
|
def marker_envs(self):
|
|
if self.extras:
|
|
return [{'extra': e} for e in sorted(self.extras)]
|
|
return [{'extra': ''}]
|
|
|
|
def evaluate_all_environamnets(self, requirement):
|
|
for marker_env in self.marker_envs:
|
|
if requirement.marker.evaluate(environment=marker_env):
|
|
return True
|
|
return False
|
|
|
|
def add(self, requirement_str, *, source=None):
|
|
"""Output a Python-style requirement string as RPM dep"""
|
|
print_err(f'Handling {requirement_str} from {source}')
|
|
|
|
try:
|
|
requirement = Requirement(requirement_str)
|
|
except InvalidRequirement as e:
|
|
print_err(
|
|
f'WARNING: Skipping invalid requirement: {requirement_str}\n'
|
|
+ f' {e}',
|
|
)
|
|
return
|
|
|
|
name = canonicalize_name(requirement.name)
|
|
if (requirement.marker is not None and
|
|
not self.evaluate_all_environamnets(requirement)):
|
|
print_err(f'Ignoring alien requirement:', requirement_str)
|
|
return
|
|
|
|
try:
|
|
# TODO: check if requirements with extras are satisfied
|
|
installed = self.get_installed_version(requirement.name)
|
|
except importlib_metadata.PackageNotFoundError:
|
|
print_err(f'Requirement not satisfied: {requirement_str}')
|
|
installed = None
|
|
if installed and installed in requirement.specifier:
|
|
print_err(f'Requirement satisfied: {requirement_str}')
|
|
print_err(f' (installed: {requirement.name} {installed})')
|
|
if requirement.extras:
|
|
print_err(f' (extras are currently not checked)')
|
|
else:
|
|
self.missing_requirements = True
|
|
|
|
if self.generate_extras:
|
|
extra_names = [f'{name}[{extra}]' for extra in sorted(requirement.extras)]
|
|
else:
|
|
extra_names = []
|
|
|
|
for name in [name] + extra_names:
|
|
together = []
|
|
for specifier in sorted(
|
|
requirement.specifier,
|
|
key=lambda s: (s.operator, s.version),
|
|
):
|
|
version = canonicalize_version(specifier.version)
|
|
if not VERSION_RE.fullmatch(str(specifier.version)):
|
|
raise ValueError(
|
|
f'Unknown character in version: {specifier.version}. '
|
|
+ '(This is probably a bug in pyproject-rpm-macros.)',
|
|
)
|
|
together.append(convert(python3dist(name, python3_pkgversion=self.python3_pkgversion),
|
|
specifier.operator, version))
|
|
if len(together) == 0:
|
|
print(python3dist(name,
|
|
python3_pkgversion=self.python3_pkgversion))
|
|
elif len(together) == 1:
|
|
print(together[0])
|
|
else:
|
|
print(f"({' with '.join(together)})")
|
|
|
|
def check(self, *, source=None):
|
|
"""End current pass if any unsatisfied dependencies were output"""
|
|
if self.missing_requirements:
|
|
print_err(f'Exiting dependency generation pass: {source}')
|
|
raise EndPass(source)
|
|
|
|
def extend(self, requirement_strs, *, source=None):
|
|
"""add() several requirements"""
|
|
for req_str in requirement_strs:
|
|
self.add(req_str, source=source)
|
|
|
|
|
|
def get_backend(requirements):
|
|
try:
|
|
f = open('pyproject.toml')
|
|
except FileNotFoundError:
|
|
pyproject_data = {}
|
|
else:
|
|
# lazy import toml here, not needed without pyproject.toml
|
|
requirements.add('toml', source='parsing pyproject.toml')
|
|
requirements.check(source='parsing pyproject.toml')
|
|
import toml
|
|
with f:
|
|
pyproject_data = toml.load(f)
|
|
|
|
buildsystem_data = pyproject_data.get('build-system', {})
|
|
requirements.extend(
|
|
buildsystem_data.get('requires', ()),
|
|
source='build-system.requires',
|
|
)
|
|
|
|
backend_name = buildsystem_data.get('build-backend')
|
|
if not backend_name:
|
|
# https://www.python.org/dev/peps/pep-0517/:
|
|
# If the pyproject.toml file is absent, or the build-backend key is
|
|
# missing, the source tree is not using this specification, and tools
|
|
# should revert to the legacy behaviour of running setup.py
|
|
# (either directly, or by implicitly invoking the [following] backend).
|
|
backend_name = 'setuptools.build_meta:__legacy__'
|
|
|
|
requirements.add('setuptools >= 40.8', source='default build backend')
|
|
requirements.add('wheel', source='default build backend')
|
|
|
|
requirements.check(source='build backend')
|
|
|
|
backend_path = buildsystem_data.get('backend-path')
|
|
if backend_path:
|
|
# PEP 517 example shows the path as a list, but some projects don't follow that
|
|
if isinstance(backend_path, str):
|
|
backend_path = [backend_path]
|
|
sys.path = backend_path + sys.path
|
|
|
|
module_name, _, object_name = backend_name.partition(":")
|
|
backend_module = importlib.import_module(module_name)
|
|
|
|
if object_name:
|
|
return getattr(backend_module, object_name)
|
|
|
|
return backend_module
|
|
|
|
|
|
def generate_build_requirements(backend, requirements):
|
|
get_requires = getattr(backend, 'get_requires_for_build_wheel', None)
|
|
if get_requires:
|
|
with hook_call():
|
|
new_reqs = get_requires()
|
|
requirements.extend(new_reqs, source='get_requires_for_build_wheel')
|
|
requirements.check(source='get_requires_for_build_wheel')
|
|
|
|
|
|
def generate_run_requirements(backend, requirements):
|
|
hook_name = 'prepare_metadata_for_build_wheel'
|
|
prepare_metadata = getattr(backend, hook_name, None)
|
|
if not prepare_metadata:
|
|
raise ValueError(
|
|
'build backend cannot provide build metadata '
|
|
+ '(incl. runtime requirements) before buld'
|
|
)
|
|
with hook_call():
|
|
dir_basename = prepare_metadata('.')
|
|
with open(dir_basename + '/METADATA') as f:
|
|
message = email.parser.Parser().parse(f, headersonly=True)
|
|
for key in 'Requires', 'Requires-Dist':
|
|
requires = message.get_all(key, ())
|
|
requirements.extend(requires, source=f'wheel metadata: {key}')
|
|
|
|
|
|
def parse_tox_requires_lines(lines):
|
|
packages = []
|
|
for line in lines:
|
|
line = line.strip()
|
|
if line.startswith('-r'):
|
|
path = line[2:]
|
|
with open(path) as f:
|
|
packages.extend(parse_tox_requires_lines(f.read().splitlines()))
|
|
elif line.startswith('-'):
|
|
print_err(
|
|
f'WARNING: Skipping dependency line: {line}\n'
|
|
+ f' tox deps options other than -r are not supported (yet).',
|
|
)
|
|
elif line:
|
|
packages.append(line)
|
|
return packages
|
|
|
|
|
|
def generate_tox_requirements(toxenv, requirements):
|
|
requirements.add('tox-current-env >= 0.0.3', source='tox itself')
|
|
requirements.check(source='tox itself')
|
|
with tempfile.NamedTemporaryFile('r') as deps, tempfile.NamedTemporaryFile('r') as extras:
|
|
r = subprocess.run(
|
|
[sys.executable, '-m', 'tox',
|
|
'--print-deps-to', deps.name,
|
|
'--print-extras-to', extras.name,
|
|
'-qre', toxenv],
|
|
check=False,
|
|
encoding='utf-8',
|
|
stdout=subprocess.PIPE,
|
|
stderr=subprocess.STDOUT,
|
|
)
|
|
if r.stdout:
|
|
print_err(r.stdout, end='')
|
|
r.check_returncode()
|
|
|
|
deplines = deps.read().splitlines()
|
|
packages = parse_tox_requires_lines(deplines)
|
|
requirements.add_extras(*extras.read().splitlines())
|
|
requirements.extend(packages,
|
|
source=f'tox --print-deps-only: {toxenv}')
|
|
|
|
|
|
def python3dist(name, op=None, version=None, python3_pkgversion="3"):
|
|
prefix = f"python{python3_pkgversion}dist"
|
|
|
|
if op is None:
|
|
if version is not None:
|
|
raise AssertionError('op and version go together')
|
|
return f'{prefix}({name})'
|
|
else:
|
|
return f'{prefix}({name}) {op} {version}'
|
|
|
|
|
|
def generate_requires(
|
|
*, include_runtime=False, toxenv=None, extras=None,
|
|
get_installed_version=importlib_metadata.version, # for dep injection
|
|
generate_extras=False, python3_pkgversion="3",
|
|
):
|
|
"""Generate the BuildRequires for the project in the current directory
|
|
|
|
This is the main Python entry point.
|
|
"""
|
|
requirements = Requirements(
|
|
get_installed_version, extras=extras or [],
|
|
generate_extras=generate_extras,
|
|
python3_pkgversion=python3_pkgversion
|
|
)
|
|
|
|
try:
|
|
backend = get_backend(requirements)
|
|
generate_build_requirements(backend, requirements)
|
|
if toxenv is not None:
|
|
include_runtime = True
|
|
generate_tox_requirements(toxenv, requirements)
|
|
if include_runtime:
|
|
generate_run_requirements(backend, requirements)
|
|
except EndPass:
|
|
return
|
|
|
|
|
|
def main(argv):
|
|
parser = argparse.ArgumentParser(
|
|
description='Generate BuildRequires for a Python project.'
|
|
)
|
|
parser.add_argument(
|
|
'-r', '--runtime', action='store_true',
|
|
help='Generate run-time requirements',
|
|
)
|
|
parser.add_argument(
|
|
'-e', '--toxenv', metavar='TOXENVS', default=None,
|
|
help=('specify tox environments'
|
|
'(implies --tox)'),
|
|
)
|
|
parser.add_argument(
|
|
'-t', '--tox', action='store_true',
|
|
help=('generate test tequirements from tox environment '
|
|
'(implies --runtime)'),
|
|
)
|
|
parser.add_argument(
|
|
'-x', '--extras', metavar='EXTRAS', action='append',
|
|
help='comma separated list of "extras" for runtime requirements '
|
|
'(e.g. -x testing,feature-x) (implies --runtime, can be repeated)',
|
|
)
|
|
parser.add_argument(
|
|
'--generate-extras', action='store_true',
|
|
help='Generate build requirements on Python Extras',
|
|
)
|
|
parser.add_argument(
|
|
'-p', '--python3_pkgversion', metavar='PYTHON3_PKGVERSION',
|
|
default="3", help=('Python version for pythonXdist()'
|
|
'or pythonX.Ydist() requirements'),
|
|
)
|
|
|
|
args = parser.parse_args(argv)
|
|
|
|
if args.toxenv:
|
|
args.tox = True
|
|
|
|
if args.tox:
|
|
args.runtime = True
|
|
args.toxenv = (args.toxenv or os.getenv('RPM_TOXENV') or
|
|
f'py{sys.version_info.major}{sys.version_info.minor}')
|
|
|
|
if args.extras:
|
|
args.runtime = True
|
|
|
|
try:
|
|
generate_requires(
|
|
include_runtime=args.runtime,
|
|
toxenv=args.toxenv,
|
|
extras=args.extras,
|
|
generate_extras=args.generate_extras,
|
|
python3_pkgversion=args.python3_pkgversion,
|
|
)
|
|
except Exception:
|
|
# Log the traceback explicitly (it's useful debug info)
|
|
traceback.print_exc()
|
|
exit(1)
|
|
|
|
|
|
if __name__ == '__main__':
|
|
main(sys.argv[1:])
|