rpms/pyproject-rpm-macros
7
0
Fork 0
Code Issues Pull Requests Releases Activity

Redirect stdout to stderr via Shell

Browse Source 
Dependencies are recorded to a text file that is catted at the end.

This should prevent subtle bugs like https://bugzilla.redhat.com/2183519 in the future.
This commit is contained in:
Miro Hrončok 2023-03-31 18:57:54 +02:00
parent f8e160d767
commit 456903666c
6 changed files with 37 additions and 50 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
macros.pyproject
View File

@ -19,6 +19,7 @@
%_pyproject_modules %{_builddir}/%{_pyproject_files_prefix}-pyproject-modules
%_pyproject_ghost_distinfo %{_builddir}/%{_pyproject_files_prefix}-pyproject-ghost-distinfo
%_pyproject_record %{_builddir}/%{_pyproject_files_prefix}-pyproject-record
%_pyproject_buildrequires %{_builddir}/%{_pyproject_files_prefix}-pyproject-buildrequires
# Avoid leaking %%{_pyproject_builddir} to pytest collection
# https://bugzilla.redhat.com/show_bug.cgi?id=1935212
@ -169,8 +170,10 @@ fi}
rm -rfv *.dist-info/ >&2
if [ -f %{__python3} ]; then
mkdir -p "%{_pyproject_builddir}"
echo -n > %{_pyproject_buildrequires}
CFLAGS="${CFLAGS:-${RPM_OPT_FLAGS}}" LDFLAGS="${LDFLAGS:-${RPM_LD_FLAGS}}" TMPDIR="%{_pyproject_builddir}" \\\
RPM_TOXENV="%{toxenv}" HOSTNAME="rpmbuild" %{__python3} -Bs %{_rpmconfigdir}/redhat/pyproject_buildrequires.py %{?!_python_no_extras_requires:--generate-extras} --python3_pkgversion %{python3_pkgversion} --wheeldir %{_pyproject_wheeldir} %{?**}
RPM_TOXENV="%{toxenv}" HOSTNAME="rpmbuild" %{__python3} -Bs %{_rpmconfigdir}/redhat/pyproject_buildrequires.py %{?!_python_no_extras_requires:--generate-extras} --python3_pkgversion %{python3_pkgversion} --wheeldir %{_pyproject_wheeldir} --output %{_pyproject_buildrequires} %{?**} >&2
cat %{_pyproject_buildrequires}
fi
# Incomplete .dist-info dir might confuse importlib.metadata
rm -rfv *.dist-info/ >&2

7
pyproject-rpm-macros.spec
View File

@ -10,7 +10,7 @@ License: MIT
# Increment Y and reset Z when new macros or features are added
# Increment Z when this is a bugfix or a cosmetic change
# Dropping support for EOL Fedoras is *not* considered a breaking change
Version: 1.6.3
Version: 1.7.0
Release: 1%{?dist}
# Macro files
@ -147,6 +147,11 @@ export HOSTNAME="rpmbuild" # to speedup tox in network-less mock, see rhbz#1856
%changelog
* Fri Mar 31 2023 Miro Hrončok <mhroncok@redhat.com> - 1.7.0-1
- %%pyproject_buildrequires: Redirect stdout to stderr via Shell
- Dependencies are recorded to a text file that is catted at the end
- Fixes: rhbz#2183519
* Mon Feb 13 2023 Lumír Balhar <lbalhar@redhat.com> - 1.6.3-1
- Remove .dist-info directory at the end of %%pyproject_buildrequires
- An incomplete .dist-info directory in $PWD can confuse tests in %%check

62
pyproject_buildrequires.py
View File

@ -5,7 +5,6 @@ import sys
import importlib.metadata
import argparse
import traceback
import contextlib
import json
import subprocess
import re
@ -45,39 +44,6 @@ except ImportError as e:
from pyproject_convert import convert
@contextlib.contextmanager
def hook_call():
"""Context manager that records all stdout content (on FD level)
and prints it to stderr at the end, with a 'HOOK STDOUT: ' prefix."""
tmpfile = io.TextIOWrapper(
tempfile.TemporaryFile(buffering=0),
encoding='utf-8',
errors='replace',
write_through=True,
)
stdout_fd = 1
stdout_fd_dup = os.dup(stdout_fd)
stdout_orig = sys.stdout
# begin capture
sys.stdout = tmpfile
os.dup2(tmpfile.fileno(), stdout_fd)
try:
yield
finally:
# end capture
sys.stdout = stdout_orig
os.dup2(stdout_fd_dup, stdout_fd)
tmpfile.seek(0) # rewind
for line in tmpfile:
print_err('HOOK STDOUT:', line, end='')
tmpfile.close()
def guess_reason_for_invalid_requirement(requirement_str):
if ':' in requirement_str:
message = (
@ -99,10 +65,11 @@ def guess_reason_for_invalid_requirement(requirement_str):
class Requirements:
"""Requirement printer"""
"""Requirement gatherer. The macro will eventually print out output_lines."""
def __init__(self, get_installed_version, extras=None,
generate_extras=False, python3_pkgversion='3'):
self.get_installed_version = get_installed_version
self.output_lines = []
self.extras = set()
if extras:
@ -191,12 +158,12 @@ class Requirements:
together.append(convert(python3dist(name, python3_pkgversion=self.python3_pkgversion),
specifier.operator, specifier.version))
if len(together) == 0:
print(python3dist(name,
python3_pkgversion=self.python3_pkgversion))
dep = python3dist(name, python3_pkgversion=self.python3_pkgversion)
self.output_lines.append(dep)
elif len(together) == 1:
print(together[0])
self.output_lines.append(together[0])
else:
print(f"({' with '.join(together)})")
self.output_lines.append(f"({' with '.join(together)})")
def check(self, *, source=None):
"""End current pass if any unsatisfied dependencies were output"""
@ -284,8 +251,7 @@ def get_backend(requirements):
def generate_build_requirements(backend, requirements):
get_requires = getattr(backend, 'get_requires_for_build_wheel', None)
if get_requires:
with hook_call():
new_reqs = get_requires()
new_reqs = get_requires()
requirements.extend(new_reqs, source='get_requires_for_build_wheel')
requirements.check(source='get_requires_for_build_wheel')
@ -305,8 +271,7 @@ def generate_run_requirements_hook(backend, requirements):
'Use the provisional -w flag to build the wheel and parse the metadata from it, '
'or use the -R flag not to generate runtime dependencies.'
)
with hook_call():
dir_basename = prepare_metadata('.')
dir_basename = prepare_metadata('.')
with open(dir_basename + '/METADATA') as metadata_file:
for key, requires in requires_from_metadata_file(metadata_file).items():
requirements.extend(requires, source=f'hook generated metadata: {key}')
@ -411,10 +376,13 @@ def python3dist(name, op=None, version=None, python3_pkgversion="3"):
def generate_requires(
*, include_runtime=False, build_wheel=False, wheeldir=None, toxenv=None, extras=None,
get_installed_version=importlib.metadata.version, # for dep injection
generate_extras=False, python3_pkgversion="3", requirement_files=None, use_build_system=True
generate_extras=False, python3_pkgversion="3", requirement_files=None, use_build_system=True,
output,
):
"""Generate the BuildRequires for the project in the current directory
The generated BuildRequires are written to the provided output.
This is the main Python entry point.
"""
requirements = Requirements(
@ -443,6 +411,8 @@ def generate_requires(
generate_run_requirements(backend, requirements, build_wheel=build_wheel, wheeldir=wheeldir)
except EndPass:
return
finally:
output.write_text(os.linesep.join(requirements.output_lines) + os.linesep)
def main(argv):
@ -468,6 +438,9 @@ def main(argv):
'-p', '--python3_pkgversion', metavar='PYTHON3_PKGVERSION',
default="3", help=argparse.SUPPRESS,
)
parser.add_argument(
'--output', type=pathlib.Path, required=True, help=argparse.SUPPRESS,
)
parser.add_argument(
'--wheeldir', metavar='PATH', default=None,
help=argparse.SUPPRESS,
@ -538,6 +511,7 @@ def main(argv):
python3_pkgversion=args.python3_pkgversion,
requirement_files=args.requirement_files,
use_build_system=args.use_build_system,
output=args.output,
)
except Exception:
# Log the traceback explicitly (it's useful debug info)

3
pyproject_buildrequires_testcases.yaml
View File

@ -820,7 +820,7 @@ Pre-releases are accepted:
result: 0
Wrapped subprocess prints to stdout from setup.py:
Stdout from wrapped subprocess does not appear in output:
installed:
setuptools: 50
wheel: 1
@ -834,5 +834,4 @@ Wrapped subprocess prints to stdout from setup.py:
python3dist(setuptools) >= 40.8
python3dist(wheel)
python3dist(wheel)
stderr_contains: "HOOK STDOUT: LEAK?"
result: 0

7
test_pyproject_buildrequires.py
View File

@ -21,6 +21,7 @@ def test_data(case_name, capfd, tmp_path, monkeypatch):
monkeypatch.chdir(cwd)
wheeldir = cwd.joinpath('wheeldir')
wheeldir.mkdir()
output = tmp_path.joinpath('output.txt')
if case.get('xfail'):
pytest.xfail(case.get('xfail'))
@ -54,6 +55,7 @@ def test_data(case_name, capfd, tmp_path, monkeypatch):
generate_extras=case.get('generate_extras', False),
requirement_files=requirement_files,
use_build_system=use_build_system,
output=output,
)
except SystemExit as e:
assert e.code == case['result']
@ -69,14 +71,15 @@ def test_data(case_name, capfd, tmp_path, monkeypatch):
assert 'expected' in case or 'stderr_contains' in case
out, err = capfd.readouterr()
dependencies = output.read_text()
if 'expected' in case:
expected = case['expected']
if isinstance(expected, list):
# at least one of them needs to match
assert any(out == e for e in expected)
assert any(dependencies == e for e in expected)
else:
assert out == expected
assert dependencies == expected
# stderr_contains may be a string or list of strings
stderr_contains = case.get('stderr_contains')

3
tests/python-pytest.spec
View File

@ -37,6 +37,9 @@ Summary: %{summary}
%autosetup -p1 -n %{pypi_name}-%{version}
# remove optional test dependencies we don't like to pull in
sed -E -i '/mock|nose/d' setup.cfg
# internal check for our macros: insert a subprocess echo to setup.py
# to ensure it's not generated as BuildRequires
echo 'import os; os.system("echo if-this-is-generated-the-build-will-fail")' >> setup.py
%generate_buildrequires