pyproject-rpm-macros/pyproject_buildrequires.py

import os
import sys
import importlib.metadata
import argparse
import traceback
import contextlib
from io import StringIO
import json
import subprocess
import re
import tempfile
import email.parser
import pathlib


# Some valid Python version specifiers are not supported.
# Allow only the forms we know we can handle.
VERSION_RE = re.compile(r'[a-zA-Z0-9.-]+(\.\*)?')


class EndPass(Exception):
    """End current pass of generating requirements"""


# nb: we don't use functools.partial to be able to use pytest's capsys
# see https://github.com/pytest-dev/pytest/issues/8900
def print_err(*args, **kwargs):
    kwargs.setdefault('file', sys.stderr)
    print(*args, **kwargs)


try:
    from packaging.requirements import Requirement, InvalidRequirement
    from packaging.utils import canonicalize_name
except ImportError as e:
    print_err('Import error:', e)
    # already echoed by the %pyproject_buildrequires macro
    sys.exit(0)

# uses packaging, needs to be imported after packaging is verified to be present
from pyproject_convert import convert


@contextlib.contextmanager
def hook_call():
    captured_out = StringIO()
    with contextlib.redirect_stdout(captured_out):
        yield
    for line in captured_out.getvalue().splitlines():
        print_err('HOOK STDOUT:', line)


class Requirements:
    """Requirement printer"""
    def __init__(self, get_installed_version, extras=None,
                 generate_extras=False, python3_pkgversion='3'):
        self.get_installed_version = get_installed_version
        self.extras = set()

        if extras:
            for extra in extras:
                self.add_extras(*extra.split(','))

        self.missing_requirements = False

        self.generate_extras = generate_extras
        self.python3_pkgversion = python3_pkgversion

    def add_extras(self, *extras):
        self.extras |= set(e.strip() for e in extras)

    @property
    def marker_envs(self):
        if self.extras:
            return [{'extra': e} for e in sorted(self.extras)]
        return [{'extra': ''}]

    def evaluate_all_environamnets(self, requirement):
        for marker_env in self.marker_envs:
            if requirement.marker.evaluate(environment=marker_env):
                return True
        return False

    def add(self, requirement_str, *, source=None):
        """Output a Python-style requirement string as RPM dep"""
        print_err(f'Handling {requirement_str} from {source}')

        try:
            requirement = Requirement(requirement_str)
        except InvalidRequirement as e:
            print_err(
                f'WARNING: Skipping invalid requirement: {requirement_str}\n'
                + f'    {e}',
            )
            return

        name = canonicalize_name(requirement.name)
        if (requirement.marker is not None and
                not self.evaluate_all_environamnets(requirement)):
            print_err(f'Ignoring alien requirement:', requirement_str)
            return

        try:
            # TODO: check if requirements with extras are satisfied
            installed = self.get_installed_version(requirement.name)
        except importlib.metadata.PackageNotFoundError:
            print_err(f'Requirement not satisfied: {requirement_str}')
            installed = None
        if installed and installed in requirement.specifier:
            print_err(f'Requirement satisfied: {requirement_str}')
            print_err(f'   (installed: {requirement.name} {installed})')
            if requirement.extras:
                print_err(f'   (extras are currently not checked)')
        else:
            self.missing_requirements = True

        if self.generate_extras:
            extra_names = [f'{name}[{extra.lower()}]' for extra in sorted(requirement.extras)]
        else:
            extra_names = []

        for name in [name] + extra_names:
            together = []
            for specifier in sorted(
                requirement.specifier,
                key=lambda s: (s.operator, s.version),
            ):
                if not VERSION_RE.fullmatch(str(specifier.version)):
                    raise ValueError(
                        f'Unknown character in version: {specifier.version}. '
                        + '(This is probably a bug in pyproject-rpm-macros.)',
                    )
                together.append(convert(python3dist(name, python3_pkgversion=self.python3_pkgversion),
                                        specifier.operator, specifier.version))
            if len(together) == 0:
                print(python3dist(name,
                                  python3_pkgversion=self.python3_pkgversion))
            elif len(together) == 1:
                print(together[0])
            else:
                print(f"({' with '.join(together)})")

    def check(self, *, source=None):
        """End current pass if any unsatisfied dependencies were output"""
        if self.missing_requirements:
            print_err(f'Exiting dependency generation pass: {source}')
            raise EndPass(source)

    def extend(self, requirement_strs, *, source=None):
        """add() several requirements"""
        for req_str in requirement_strs:
            self.add(req_str, source=source)


def get_backend(requirements):
    try:
        f = open('pyproject.toml')
    except FileNotFoundError:
        pyproject_data = {}
    else:
        try:
            # lazy import toml here, not needed without pyproject.toml
            import toml
        except ImportError as e:
            print_err('Import error:', e)
            # already echoed by the %pyproject_buildrequires macro
            sys.exit(0)
        with f:
            pyproject_data = toml.load(f)

    buildsystem_data = pyproject_data.get('build-system', {})
    requirements.extend(
        buildsystem_data.get('requires', ()),
        source='build-system.requires',
    )

    backend_name = buildsystem_data.get('build-backend')
    if not backend_name:
        # https://www.python.org/dev/peps/pep-0517/:
        # If the pyproject.toml file is absent, or the build-backend key is
        # missing, the source tree is not using this specification, and tools
        # should revert to the legacy behaviour of running setup.py
        # (either directly, or by implicitly invoking the [following] backend).
        backend_name = 'setuptools.build_meta:__legacy__'

        # Note: For projects without pyproject.toml, this was already echoed
        # by the %pyproject_buildrequires macro, but this also handles cases
        # with pyproject.toml without a specified build backend.
        # If the default requirements change, also change them in the macro!
        requirements.add('setuptools >= 40.8', source='default build backend')
        requirements.add('wheel', source='default build backend')

    requirements.check(source='build backend')

    backend_path = buildsystem_data.get('backend-path')
    if backend_path:
        # PEP 517 example shows the path as a list, but some projects don't follow that
        if isinstance(backend_path, str):
            backend_path = [backend_path]
        sys.path = backend_path + sys.path

    module_name, _, object_name = backend_name.partition(":")
    backend_module = importlib.import_module(module_name)

    if object_name:
        return getattr(backend_module, object_name)

    return backend_module


def generate_build_requirements(backend, requirements):
    get_requires = getattr(backend, 'get_requires_for_build_wheel', None)
    if get_requires:
        with hook_call():
            new_reqs = get_requires()
        requirements.extend(new_reqs, source='get_requires_for_build_wheel')
        requirements.check(source='get_requires_for_build_wheel')


def generate_run_requirements(backend, requirements):
    hook_name = 'prepare_metadata_for_build_wheel'
    prepare_metadata = getattr(backend, hook_name, None)
    if not prepare_metadata:
        raise ValueError(
            'build backend cannot provide build metadata '
            + '(incl. runtime requirements) before build'
        )
    with hook_call():
        dir_basename = prepare_metadata('.')
    with open(dir_basename + '/METADATA') as f:
        message = email.parser.Parser().parse(f, headersonly=True)
    for key in 'Requires', 'Requires-Dist':
        requires = message.get_all(key, ())
        requirements.extend(requires, source=f'wheel metadata: {key}')


def parse_requirements_lines(lines, path=None):
    packages = []
    for line in lines:
        line, _, comment = line.partition('#')
        if comment.startswith('egg='):
            # not a real comment
            # e.g. git+https://github.com/monty/spam.git@master#egg=spam&...
            egg, *_ = comment.strip().partition(' ')
            egg, *_ = egg.strip().partition('&')
            line = egg[4:]
        line = line.strip()
        if line.startswith('-r'):
            recursed_path = line[2:].strip()
            if path:
                recursed_path = path.parent / recursed_path
            with open(recursed_path) as f:
                packages.extend(parse_requirements_lines(f.read().splitlines(), recursed_path))
        elif line.startswith('-'):
            print_err(
                f'WARNING: Skipping dependency line: {line}\n'
                + f'    tox deps options other than -r are not supported (yet).',
            )
        elif line:
            packages.append(line)
    return packages


def generate_tox_requirements(toxenv, requirements):
    toxenv = ','.join(toxenv)
    requirements.add('tox-current-env >= 0.0.6', source='tox itself')
    requirements.check(source='tox itself')
    with tempfile.NamedTemporaryFile('r') as deps, \
        tempfile.NamedTemporaryFile('r') as extras, \
            tempfile.NamedTemporaryFile('r') as provision:
        r = subprocess.run(
            [sys.executable, '-m', 'tox',
             '--print-deps-to', deps.name,
             '--print-extras-to', extras.name,
             '--no-provision', provision.name,
             '-qre', toxenv],
            check=False,
            encoding='utf-8',
            stdout=subprocess.PIPE,
            stderr=subprocess.STDOUT,
        )
        if r.stdout:
            print_err(r.stdout, end='')

        provision_content = provision.read()
        if provision_content and r.returncode != 0:
            provision_requires = json.loads(provision_content)
            if 'minversion' in provision_requires:
                requirements.add(f'tox >= {provision_requires["minversion"]}',
                                 source='tox provision (minversion)')
            if 'requires' in provision_requires:
                requirements.extend(provision_requires["requires"],
                                    source='tox provision (requires)')
            requirements.check(source='tox provision')  # this terminates the script
            raise RuntimeError(
                'Dependencies requested by tox provisioning appear installed, '
                'but tox disagreed.')
        else:
            r.check_returncode()

        deplines = deps.read().splitlines()
        packages = parse_requirements_lines(deplines)
        requirements.add_extras(*extras.read().splitlines())
        requirements.extend(packages,
                            source=f'tox --print-deps-only: {toxenv}')


def python3dist(name, op=None, version=None, python3_pkgversion="3"):
    prefix = f"python{python3_pkgversion}dist"

    if op is None:
        if version is not None:
            raise AssertionError('op and version go together')
        return f'{prefix}({name})'
    else:
        return f'{prefix}({name}) {op} {version}'


def generate_requires(
    *, include_runtime=False, toxenv=None, extras=None,
    get_installed_version=importlib.metadata.version,  # for dep injection
    generate_extras=False, python3_pkgversion="3", requirement_files=None, use_build_system=True
):
    """Generate the BuildRequires for the project in the current directory

    This is the main Python entry point.
    """
    requirements = Requirements(
        get_installed_version, extras=extras or [],
        generate_extras=generate_extras,
        python3_pkgversion=python3_pkgversion
    )

    try:
        if (include_runtime or toxenv) and not use_build_system:
            raise ValueError('-N option cannot be used in combination with -r, -e, -t, -x options')
        if requirement_files:
            for req_file in requirement_files:
                lines = req_file.read().splitlines()
                packages = parse_requirements_lines(lines, pathlib.Path(req_file.name))
                requirements.extend(packages,
                                    source=f'requirements file {req_file.name}')
            requirements.check(source='all requirement files')
        if use_build_system:
            backend = get_backend(requirements)
            generate_build_requirements(backend, requirements)
        if toxenv:
            include_runtime = True
            generate_tox_requirements(toxenv, requirements)
        if include_runtime:
            generate_run_requirements(backend, requirements)
    except EndPass:
        return


def main(argv):
    parser = argparse.ArgumentParser(
        description='Generate BuildRequires for a Python project.'
    )
    parser.add_argument(
        '-r', '--runtime', action='store_true',
        help='Generate run-time requirements',
    )
    parser.add_argument(
        '-e', '--toxenv', metavar='TOXENVS', action='append',
        help=('specify tox environments (comma separated and/or repeated)'
              '(implies --tox)'),
    )
    parser.add_argument(
        '-t', '--tox', action='store_true',
        help=('generate test tequirements from tox environment '
              '(implies --runtime)'),
    )
    parser.add_argument(
        '-x', '--extras', metavar='EXTRAS', action='append',
        help='comma separated list of "extras" for runtime requirements '
             '(e.g. -x testing,feature-x) (implies --runtime, can be repeated)',
    )
    parser.add_argument(
        '--generate-extras', action='store_true',
        help='Generate build requirements on Python Extras',
    )
    parser.add_argument(
        '-p', '--python3_pkgversion', metavar='PYTHON3_PKGVERSION',
        default="3", help=('Python version for pythonXdist()'
                           'or pythonX.Ydist() requirements'),
    )
    parser.add_argument(
        '-N', '--no-use-build-system', dest='use_build_system',
        action='store_false', help='Use -N to indicate that project does not use any build system',
    )
    parser.add_argument(
       'requirement_files', nargs='*', type=argparse.FileType('r'),
        help=('Add buildrequires from file'),
    )

    args = parser.parse_args(argv)

    if args.toxenv:
        args.tox = True

    if args.tox:
        args.runtime = True
        if not args.toxenv:
            _default = f'py{sys.version_info.major}{sys.version_info.minor}'
            args.toxenv = [os.getenv('RPM_TOXENV', _default)]

    if args.extras:
        args.runtime = True

    try:
        generate_requires(
            include_runtime=args.runtime,
            toxenv=args.toxenv,
            extras=args.extras,
            generate_extras=args.generate_extras,
            python3_pkgversion=args.python3_pkgversion,
            requirement_files=args.requirement_files,
            use_build_system=args.use_build_system,
        )
    except Exception:
        # Log the traceback explicitly (it's useful debug info)
        traceback.print_exc()
        exit(1)


if __name__ == '__main__':
    main(sys.argv[1:])