From 8f2b3503f12f816029d77466d68b2b55bd58f574 Mon Sep 17 00:00:00 2001 From: Maxwell G Date: Fri, 12 May 2023 02:36:03 +0000 Subject: [PATCH] Add initial Python 3.12 support Fixes: https://bugzilla.redhat.com/2196539 --- ...at-calls-compat-code-removed-in-3.12.patch | 26 ++ ansible-core.spec | 38 ++- ansible-test-replace-pytest-forked.patch | 310 ++++++++++++++++++ avoid-use-of-deprecated-utcnow.patch | 144 ++++++++ fix-galaxy-cli-unit-test-asserts.patch | 35 ++ fix-unit-test-asserts.patch | 40 +++ replace-deprecated-ast.value.s.patch | 124 +++++++ support-Python-3.12-in-ansible-test.patch | 151 +++++++++ urls-remove-deprecated-client-key-calls.patch | 154 +++++++++ 9 files changed, 1019 insertions(+), 3 deletions(-) create mode 100644 Disable-test-that-calls-compat-code-removed-in-3.12.patch create mode 100644 ansible-test-replace-pytest-forked.patch create mode 100644 avoid-use-of-deprecated-utcnow.patch create mode 100644 fix-galaxy-cli-unit-test-asserts.patch create mode 100644 fix-unit-test-asserts.patch create mode 100644 replace-deprecated-ast.value.s.patch create mode 100644 support-Python-3.12-in-ansible-test.patch create mode 100644 urls-remove-deprecated-client-key-calls.patch diff --git a/Disable-test-that-calls-compat-code-removed-in-3.12.patch b/Disable-test-that-calls-compat-code-removed-in-3.12.patch new file mode 100644 index 0000000..89c2c79 --- /dev/null +++ b/Disable-test-that-calls-compat-code-removed-in-3.12.patch @@ -0,0 +1,26 @@ +From 0186f21027682a4c7013ddf75aa35f530fc2a090 Mon Sep 17 00:00:00 2001 +From: Maxwell G +Date: Tue, 13 Jun 2023 13:29:40 +0000 +Subject: [PATCH] Disable test that calls compat code removed in 3.12 + +--- + test/units/utils/collection_loader/test_collection_loader.py | 3 ++- + 1 file changed, 2 insertions(+), 1 deletion(-) + +diff --git a/test/units/utils/collection_loader/test_collection_loader.py b/test/units/utils/collection_loader/test_collection_loader.py +index 9be59307a9..9a27692ea5 100644 +--- a/test/units/utils/collection_loader/test_collection_loader.py ++++ b/test/units/utils/collection_loader/test_collection_loader.py +@@ -29,7 +29,8 @@ def teardown(*args, **kwargs): + # BEGIN STANDALONE TESTS - these exercise behaviors of the individual components without the import machinery + + +-@pytest.mark.skipif(not PY3, reason='Testing Python 2 codepath (find_module) on Python 3') ++@pytest.mark.skipif(not PY3 or sys.version_info[:2] >= (3, 12), ++ reason='Testing Python 2 codepath (find_module) on Python 3') + def test_find_module_py3(): + dir_to_a_file = os.path.dirname(ping_module.__file__) + path_hook_finder = _AnsiblePathHookFinder(_AnsibleCollectionFinder(), dir_to_a_file) +-- +2.40.1 + diff --git a/ansible-core.spec b/ansible-core.spec index e81dbd8..fbd62c9 100644 --- a/ansible-core.spec +++ b/ansible-core.spec @@ -12,14 +12,37 @@ Name: ansible-core Summary: A radically simple IT automation system Version: 2.15.0 %global uversion %{version_no_tilde %{quote:%nil}} -Release: 2%{?dist} +Release: 3%{?dist} # The main license is GPLv3+. Many of the files in lib/ansible/module_utils # are BSD licensed. There are various files scattered throughout the codebase # containing code under different licenses. License: GPL-3.0-or-later AND BSD-2-Clause AND PSF-2.0 AND MIT AND Apache-2.0 + Source0: https://github.com/ansible/ansible/archive/v%{uversion}/%{name}-%{uversion}.tar.gz Source1: build_manpages.py + Patch: https://github.com/ansible/ansible/commit/734f38b2594692707d1fd3cbcfc8dc8a677f4ee3.patch#/GALAXY_COLLECTIONS_PATH_WARNINGS.patch +# These patches are only applied on Rawhide to enable support for Python 3.12 +# See https://bugzilla.redhat.com/2196539 +# +# Essential # +# add Python 3.12 support to ansible-test (#80834) +Patch5000: https://github.com/ansible/ansible/pull/80834.patch#/support-Python-3.12-in-ansible-test.patch +# Fix unit test asserts (#80500) +Patch5001: https://github.com/ansible/ansible/commit/3ec828703f020551241b4169f6a3f07c701e240a.patch#/fix-unit-test-asserts.patch +# Fix galaxy CLI unit test assertions (#80504) +Patch5002: https://github.com/ansible/ansible/commit/43c5cbcaef34aeb0141b8ad24027496bf6ec2acd.patch#/fix-galaxy-cli-unit-test-asserts.patch +Patch5003: Disable-test-that-calls-compat-code-removed-in-3.12.patch +# Deprecations # +# ansible-test - Replace pytest-forked (#80525) +Patch6000: https://github.com/ansible/ansible/commit/676b731e6f7d60ce6fd48c0d1c883fc85f5c6537.patch#/ansible-test-replace-pytest-forked.patch +# ansible-test - Avoid use of deprecated utcnow (#80750) +Patch6001: https://github.com/ansible/ansible/commit/fd341265d001d4e6545ffb2b7d154340cb1f1931.patch#/avoid-use-of-deprecated-utcnow.patch +# urls - remove deprecated client key calls (#80751) +Patch6002: https://github.com/ansible/ansible/commit/0df794e5a4fe4597ee65b0d492fbf0d0989d5ca0.patch#/urls-remove-deprecated-client-key-calls.patch +# replace deprecated ast.value.s with ast.value.value (#80968) +Patch6003: https://github.com/ansible/ansible/commit/742d47fa15a5418f98abf9aaf07edf466e871c81.patch#/replace-deprecated-ast.value.s.patch + Url: https://ansible.com BuildArch: noarch @@ -99,7 +122,13 @@ This package installs extensive documentation for ansible-core %prep -%autosetup -p1 -n ansible-%{uversion} +%autosetup -N -n ansible-%{uversion} +%autopatch -M 4999 -p1 +# Python 3.12 specific patches +# Set `-D '_has_python312 1'` to test locally +%if 0%{?_has_python312} || v"%{python3_version}" >= v"3.12" +%autopatch -m 5000 -p1 +%endif sed -i -s 's|/usr/bin/env python|%{python3}|' \ bin/ansible-test \ @@ -232,7 +261,7 @@ install -Dpm 0644 licenses/* -t %{buildroot}%{_pkglicensedir} %check %if %{with tests} %{python3} bin/ansible-test \ - units --local --python-interpreter %{python3} + units --local --python-interpreter %{python3} -vv %endif @@ -258,6 +287,9 @@ install -Dpm 0644 licenses/* -t %{buildroot}%{_pkglicensedir} %changelog +* Tue Jun 13 2023 Maxwell G - 2.15.0-3 +- Add support for Python 3.12. Fixes rhbz#2196539. + * Tue May 23 2023 Yaakov Selkowitz - 2.15.0-2 - Disable tests in RHEL builds diff --git a/ansible-test-replace-pytest-forked.patch b/ansible-test-replace-pytest-forked.patch new file mode 100644 index 0000000..cbae5a8 --- /dev/null +++ b/ansible-test-replace-pytest-forked.patch @@ -0,0 +1,310 @@ +From 676b731e6f7d60ce6fd48c0d1c883fc85f5c6537 Mon Sep 17 00:00:00 2001 +From: Matt Clay +Date: Fri, 14 Apr 2023 15:13:58 -0700 +Subject: [PATCH] ansible-test - Replace pytest-forked (#80525) + +- Unit tests now report warnings generated during test runs. +- Python 3.12 warnings about `os.fork` usage with threads (due to `pytest-xdist`) are suppressed. +- Added integration tests to verify forked test behavior. +--- + .../fragments/ansible-test-pytest-forked.yml | 5 + + .../ansible-test-units-assertions/runme.sh | 2 +- + .../targets/ansible-test-units-forked/aliases | 5 + + .../plugins/modules/test_ansible_forked.py | 43 ++++++++ + .../ansible-test-units-forked/runme.sh | 45 ++++++++ + .../ansible_test/_data/requirements/units.txt | 1 - + .../_internal/commands/units/__init__.py | 3 +- + .../pylint/config/ansible-test-target.cfg | 2 + + .../target/pytest/plugins/ansible_forked.py | 103 ++++++++++++++++++ + 9 files changed, 206 insertions(+), 3 deletions(-) + create mode 100644 changelogs/fragments/ansible-test-pytest-forked.yml + create mode 100644 test/integration/targets/ansible-test-units-forked/aliases + create mode 100644 test/integration/targets/ansible-test-units-forked/ansible_collections/ns/col/tests/unit/plugins/modules/test_ansible_forked.py + create mode 100755 test/integration/targets/ansible-test-units-forked/runme.sh + create mode 100644 test/lib/ansible_test/_util/target/pytest/plugins/ansible_forked.py + +diff --git a/changelogs/fragments/ansible-test-pytest-forked.yml b/changelogs/fragments/ansible-test-pytest-forked.yml +new file mode 100644 +index 00000000000000..f8fae8139460a3 +--- /dev/null ++++ b/changelogs/fragments/ansible-test-pytest-forked.yml +@@ -0,0 +1,5 @@ ++minor_changes: ++ - ansible-test - Replace the ``pytest-forked`` pytest plugin with a custom plugin. ++bugfixes: ++ - ansible-test - Unit tests now report warnings generated during test runs. ++ Previously only warnings generated during test collection were reported. +diff --git a/test/integration/targets/ansible-test-units-assertions/runme.sh b/test/integration/targets/ansible-test-units-assertions/runme.sh +index 3511e765004a73..86fe5c818112f7 100755 +--- a/test/integration/targets/ansible-test-units-assertions/runme.sh ++++ b/test/integration/targets/ansible-test-units-assertions/runme.sh +@@ -8,7 +8,7 @@ options=$("${TEST_DIR}"/../ansible-test/venv-pythons.py --only-versions) + IFS=', ' read -r -a pythons <<< "${options}" + + for python in "${pythons[@]}"; do +- if ansible-test units --color --truncate 0 --python "${python}" --requirements "${@}" 2>&1 | tee pytest.log; then ++ if ansible-test units --truncate 0 --python "${python}" --requirements "${@}" 2>&1 | tee pytest.log; then + echo "Test did not fail as expected." + exit 1 + fi +diff --git a/test/integration/targets/ansible-test-units-forked/aliases b/test/integration/targets/ansible-test-units-forked/aliases +new file mode 100644 +index 00000000000000..79d7dbd7b09e91 +--- /dev/null ++++ b/test/integration/targets/ansible-test-units-forked/aliases +@@ -0,0 +1,5 @@ ++shippable/posix/group3 # runs in the distro test containers ++shippable/generic/group1 # runs in the default test container ++context/controller ++needs/target/collection ++needs/target/ansible-test +diff --git a/test/integration/targets/ansible-test-units-forked/ansible_collections/ns/col/tests/unit/plugins/modules/test_ansible_forked.py b/test/integration/targets/ansible-test-units-forked/ansible_collections/ns/col/tests/unit/plugins/modules/test_ansible_forked.py +new file mode 100644 +index 00000000000000..828099c65e4cf3 +--- /dev/null ++++ b/test/integration/targets/ansible-test-units-forked/ansible_collections/ns/col/tests/unit/plugins/modules/test_ansible_forked.py +@@ -0,0 +1,43 @@ ++"""Unit tests to verify the functionality of the ansible-forked pytest plugin.""" ++from __future__ import absolute_import, division, print_function ++ ++__metaclass__ = type ++ ++import os ++import pytest ++import signal ++import sys ++import warnings ++ ++ ++warnings.warn("This verifies that warnings generated during test collection are reported.") ++ ++ ++@pytest.mark.xfail ++def test_kill_xfail(): ++ os.kill(os.getpid(), signal.SIGKILL) # causes pytest to report stdout and stderr ++ ++ ++def test_kill(): ++ os.kill(os.getpid(), signal.SIGKILL) # causes pytest to report stdout and stderr ++ ++ ++@pytest.mark.xfail ++def test_exception_xfail(): ++ sys.stdout.write("This stdout message should be hidden due to xfail.") ++ sys.stderr.write("This stderr message should be hidden due to xfail.") ++ raise Exception("This error is expected, but should be hidden due to xfail.") ++ ++ ++def test_exception(): ++ sys.stdout.write("This stdout message should be reported since we're throwing an exception.") ++ sys.stderr.write("This stderr message should be reported since we're throwing an exception.") ++ raise Exception("This error is expected and should be visible.") ++ ++ ++def test_warning(): ++ warnings.warn("This verifies that warnings generated at test time are reported.") ++ ++ ++def test_passed(): ++ pass +diff --git a/test/integration/targets/ansible-test-units-forked/runme.sh b/test/integration/targets/ansible-test-units-forked/runme.sh +new file mode 100755 +index 00000000000000..c39f3c492440e7 +--- /dev/null ++++ b/test/integration/targets/ansible-test-units-forked/runme.sh +@@ -0,0 +1,45 @@ ++#!/usr/bin/env bash ++ ++source ../collection/setup.sh ++ ++set -x ++ ++options=$("${TEST_DIR}"/../ansible-test/venv-pythons.py --only-versions) ++IFS=', ' read -r -a pythons <<< "${options}" ++ ++for python in "${pythons[@]}"; do ++ echo "*** Checking Python ${python} ***" ++ ++ if ansible-test units --truncate 0 --target-python "venv/${python}" "${@}" > output.log 2>&1 ; then ++ cat output.log ++ echo "Unit tests on Python ${python} did not fail as expected. See output above." ++ exit 1 ++ fi ++ ++ cat output.log ++ echo "Unit tests on Python ${python} failed as expected. See output above. Checking for expected output ..." ++ ++ # Verify that the appropriate tests pased, failed or xfailed. ++ grep 'PASSED tests/unit/plugins/modules/test_ansible_forked.py::test_passed' output.log ++ grep 'PASSED tests/unit/plugins/modules/test_ansible_forked.py::test_warning' output.log ++ grep 'XFAIL tests/unit/plugins/modules/test_ansible_forked.py::test_kill_xfail' output.log ++ grep 'FAILED tests/unit/plugins/modules/test_ansible_forked.py::test_kill' output.log ++ grep 'FAILED tests/unit/plugins/modules/test_ansible_forked.py::test_exception' output.log ++ grep 'XFAIL tests/unit/plugins/modules/test_ansible_forked.py::test_exception_xfail' output.log ++ ++ # Verify that warnings are properly surfaced. ++ grep 'UserWarning: This verifies that warnings generated at test time are reported.' output.log ++ grep 'UserWarning: This verifies that warnings generated during test collection are reported.' output.log ++ ++ # Verify there are no unexpected warnings. ++ grep 'Warning' output.log | grep -v 'UserWarning: This verifies that warnings generated ' && exit 1 ++ ++ # Verify that details from failed tests are properly surfaced. ++ grep "^Test CRASHED with exit code -9.$" output.log ++ grep "^This stdout message should be reported since we're throwing an exception.$" output.log ++ grep "^This stderr message should be reported since we're throwing an exception.$" output.log ++ grep '^> *raise Exception("This error is expected and should be visible.")$' output.log ++ grep "^E *Exception: This error is expected and should be visible.$" output.log ++ ++ echo "*** Done Checking Python ${python} ***" ++done +diff --git a/test/lib/ansible_test/_data/requirements/units.txt b/test/lib/ansible_test/_data/requirements/units.txt +index d2f56d35a92a5c..d723a65fc663d0 100644 +--- a/test/lib/ansible_test/_data/requirements/units.txt ++++ b/test/lib/ansible_test/_data/requirements/units.txt +@@ -2,5 +2,4 @@ mock + pytest + pytest-mock + pytest-xdist +-pytest-forked + pyyaml # required by the collection loader (only needed for collections) +diff --git a/test/lib/ansible_test/_internal/commands/units/__init__.py b/test/lib/ansible_test/_internal/commands/units/__init__.py +index 7d192e1be67148..78dd8498156615 100644 +--- a/test/lib/ansible_test/_internal/commands/units/__init__.py ++++ b/test/lib/ansible_test/_internal/commands/units/__init__.py +@@ -253,7 +253,6 @@ def command_units(args: UnitsConfig) -> None: + + cmd = [ + 'pytest', +- '--forked', + '-r', 'a', + '-n', str(args.num_workers) if args.num_workers else 'auto', + '--color', 'yes' if args.color else 'no', +@@ -275,6 +274,8 @@ def command_units(args: UnitsConfig) -> None: + if data_context().content.collection: + plugins.append('ansible_pytest_collections') + ++ plugins.append('ansible_forked') ++ + if plugins: + env['PYTHONPATH'] += ':%s' % os.path.join(ANSIBLE_TEST_TARGET_ROOT, 'pytest/plugins') + env['PYTEST_PLUGINS'] = ','.join(plugins) +diff --git a/test/lib/ansible_test/_util/controller/sanity/pylint/config/ansible-test-target.cfg b/test/lib/ansible_test/_util/controller/sanity/pylint/config/ansible-test-target.cfg +index e35301dd81c1bd..f8a0a8af3ff21c 100644 +--- a/test/lib/ansible_test/_util/controller/sanity/pylint/config/ansible-test-target.cfg ++++ b/test/lib/ansible_test/_util/controller/sanity/pylint/config/ansible-test-target.cfg +@@ -57,3 +57,5 @@ preferred-modules = + # Listing them here makes it possible to enable the import-error check. + ignored-modules = + py, ++ pytest, ++ _pytest.runner, +diff --git a/test/lib/ansible_test/_util/target/pytest/plugins/ansible_forked.py b/test/lib/ansible_test/_util/target/pytest/plugins/ansible_forked.py +new file mode 100644 +index 00000000000000..d00d9e93d1be00 +--- /dev/null ++++ b/test/lib/ansible_test/_util/target/pytest/plugins/ansible_forked.py +@@ -0,0 +1,103 @@ ++"""Run each test in its own fork. PYTEST_DONT_REWRITE""" ++# MIT License (see licenses/MIT-license.txt or https://opensource.org/licenses/MIT) ++# Based on code originally from: ++# https://github.com/pytest-dev/pytest-forked ++# https://github.com/pytest-dev/py ++# TIP: Disable pytest-xdist when debugging internal errors in this plugin. ++from __future__ import absolute_import, division, print_function ++ ++__metaclass__ = type ++ ++import os ++import pickle ++import tempfile ++import warnings ++ ++from pytest import Item, hookimpl ++ ++try: ++ from pytest import TestReport ++except ImportError: ++ from _pytest.runner import TestReport # Backwards compatibility with pytest < 7. Remove once Python 2.7 is not supported. ++ ++from _pytest.runner import runtestprotocol ++ ++ ++@hookimpl(tryfirst=True) ++def pytest_runtest_protocol(item, nextitem): # type: (Item, Item | None) -> object | None ++ """Entry point for enabling this plugin.""" ++ # This is needed because pytest-xdist creates an OS thread (using execnet). ++ # See: https://github.com/pytest-dev/execnet/blob/d6aa1a56773c2e887515d63e50b1d08338cb78a7/execnet/gateway_base.py#L51 ++ warnings.filterwarnings("ignore", "^This process .* is multi-threaded, use of .* may lead to deadlocks in the child.$", DeprecationWarning) ++ ++ item_hook = item.ihook ++ item_hook.pytest_runtest_logstart(nodeid=item.nodeid, location=item.location) ++ ++ reports = run_item(item, nextitem) ++ ++ for report in reports: ++ item_hook.pytest_runtest_logreport(report=report) ++ ++ item_hook.pytest_runtest_logfinish(nodeid=item.nodeid, location=item.location) ++ ++ return True ++ ++ ++def run_item(item, nextitem): # type: (Item, Item | None) -> list[TestReport] ++ """Run the item in a child process and return a list of reports.""" ++ with tempfile.NamedTemporaryFile() as temp_file: ++ pid = os.fork() ++ ++ if not pid: ++ temp_file.delete = False ++ run_child(item, nextitem, temp_file.name) ++ ++ return run_parent(item, pid, temp_file.name) ++ ++ ++def run_child(item, nextitem, result_path): # type: (Item, Item | None, str) -> None ++ """Run the item, record the result and exit. Called in the child process.""" ++ with warnings.catch_warnings(record=True) as captured_warnings: ++ reports = runtestprotocol(item, nextitem=nextitem, log=False) ++ ++ with open(result_path, "wb") as result_file: ++ pickle.dump((reports, captured_warnings), result_file) ++ ++ os._exit(0) # noqa ++ ++ ++def run_parent(item, pid, result_path): # type: (Item, int, str) -> list[TestReport] ++ """Wait for the child process to exit and return the test reports. Called in the parent process.""" ++ exit_code = waitstatus_to_exitcode(os.waitpid(pid, 0)[1]) ++ ++ if exit_code: ++ reason = "Test CRASHED with exit code {}.".format(exit_code) ++ report = TestReport(item.nodeid, item.location, {x: 1 for x in item.keywords}, "failed", reason, "call", user_properties=item.user_properties) ++ ++ if item.get_closest_marker("xfail"): ++ report.outcome = "skipped" ++ report.wasxfail = reason ++ ++ reports = [report] ++ else: ++ with open(result_path, "rb") as result_file: ++ reports, captured_warnings = pickle.load(result_file) # type: list[TestReport], list[warnings.WarningMessage] ++ ++ for warning in captured_warnings: ++ warnings.warn_explicit(warning.message, warning.category, warning.filename, warning.lineno) ++ ++ return reports ++ ++ ++def waitstatus_to_exitcode(status): # type: (int) -> int ++ """Convert a wait status to an exit code.""" ++ # This function was added in Python 3.9. ++ # See: https://docs.python.org/3/library/os.html#os.waitstatus_to_exitcode ++ ++ if os.WIFEXITED(status): ++ return os.WEXITSTATUS(status) ++ ++ if os.WIFSIGNALED(status): ++ return -os.WTERMSIG(status) ++ ++ raise ValueError(status) diff --git a/avoid-use-of-deprecated-utcnow.patch b/avoid-use-of-deprecated-utcnow.patch new file mode 100644 index 0000000..ce8f598 --- /dev/null +++ b/avoid-use-of-deprecated-utcnow.patch @@ -0,0 +1,144 @@ +From fd341265d001d4e6545ffb2b7d154340cb1f1931 Mon Sep 17 00:00:00 2001 +From: Matt Clay +Date: Wed, 10 May 2023 11:26:49 -0700 +Subject: [PATCH] ansible-test - Avoid use of deprecated utcnow (#80750) + +The timestamps are only used by ansible-test, not the junit callback, so this change only impacts ansible-test. +--- + changelogs/fragments/ansible-test-utcnow.yml | 2 ++ + lib/ansible/utils/_junit_xml.py | 6 +++++- + test/lib/ansible_test/_internal/commands/env/__init__.py | 2 +- + .../_internal/commands/integration/__init__.py | 2 +- + .../_internal/commands/integration/cloud/__init__.py | 2 +- + test/lib/ansible_test/_internal/commands/sanity/pylint.py | 8 ++++---- + test/lib/ansible_test/_internal/test.py | 6 ++---- + 7 files changed, 16 insertions(+), 12 deletions(-) + create mode 100644 changelogs/fragments/ansible-test-utcnow.yml + +diff --git a/changelogs/fragments/ansible-test-utcnow.yml b/changelogs/fragments/ansible-test-utcnow.yml +new file mode 100644 +index 00000000000000..0781a0cb48a1f8 +--- /dev/null ++++ b/changelogs/fragments/ansible-test-utcnow.yml +@@ -0,0 +1,2 @@ ++minor_changes: ++ - ansible-test - Use ``datetime.datetime.now`` with ``tz`` specified instead of ``datetime.datetime.utcnow``. +diff --git a/lib/ansible/utils/_junit_xml.py b/lib/ansible/utils/_junit_xml.py +index 6b7c80f4cdf2de..8c4dba013edcb7 100644 +--- a/lib/ansible/utils/_junit_xml.py ++++ b/lib/ansible/utils/_junit_xml.py +@@ -144,6 +144,10 @@ class TestSuite: + system_out: str | None = None + system_err: str | None = None + ++ def __post_init__(self): ++ if self.timestamp and self.timestamp.tzinfo != datetime.timezone.utc: ++ raise ValueError(f'timestamp.tzinfo must be {datetime.timezone.utc!r}') ++ + @property + def disabled(self) -> int: + """The number of disabled test cases.""" +@@ -187,7 +191,7 @@ def get_attributes(self) -> dict[str, str]: + skipped=self.skipped, + tests=self.tests, + time=self.time, +- timestamp=self.timestamp.isoformat(timespec='seconds') if self.timestamp else None, ++ timestamp=self.timestamp.replace(tzinfo=None).isoformat(timespec='seconds') if self.timestamp else None, + ) + + def get_xml_element(self) -> ET.Element: +diff --git a/test/lib/ansible_test/_internal/commands/env/__init__.py b/test/lib/ansible_test/_internal/commands/env/__init__.py +index 44f229f87999d7..6c0510566a052e 100644 +--- a/test/lib/ansible_test/_internal/commands/env/__init__.py ++++ b/test/lib/ansible_test/_internal/commands/env/__init__.py +@@ -85,7 +85,7 @@ def show_dump_env(args: EnvConfig) -> None: + ), + git=get_ci_provider().get_git_details(args), + platform=dict( +- datetime=datetime.datetime.utcnow().strftime('%Y-%m-%dT%H:%M:%SZ'), ++ datetime=datetime.datetime.now(tz=datetime.timezone.utc).strftime('%Y-%m-%dT%H:%M:%SZ'), + platform=platform.platform(), + uname=platform.uname(), + ), +diff --git a/test/lib/ansible_test/_internal/commands/integration/__init__.py b/test/lib/ansible_test/_internal/commands/integration/__init__.py +index 0e5abbb6532193..5bd04407beeec5 100644 +--- a/test/lib/ansible_test/_internal/commands/integration/__init__.py ++++ b/test/lib/ansible_test/_internal/commands/integration/__init__.py +@@ -566,7 +566,7 @@ def command_integration_filtered( + coverage_manager.teardown() + + result_name = '%s-%s.json' % ( +- args.command, re.sub(r'[^0-9]', '-', str(datetime.datetime.utcnow().replace(microsecond=0)))) ++ args.command, re.sub(r'[^0-9]', '-', str(datetime.datetime.now(tz=datetime.timezone.utc).replace(microsecond=0, tzinfo=None)))) + + data = dict( + targets=results, +diff --git a/test/lib/ansible_test/_internal/commands/integration/cloud/__init__.py b/test/lib/ansible_test/_internal/commands/integration/cloud/__init__.py +index cad84a368924b2..eac9265a4f8c53 100644 +--- a/test/lib/ansible_test/_internal/commands/integration/cloud/__init__.py ++++ b/test/lib/ansible_test/_internal/commands/integration/cloud/__init__.py +@@ -170,7 +170,7 @@ def cloud_init(args: IntegrationConfig, targets: tuple[IntegrationTarget, ...]) + + if not args.explain and results: + result_name = '%s-%s.json' % ( +- args.command, re.sub(r'[^0-9]', '-', str(datetime.datetime.utcnow().replace(microsecond=0)))) ++ args.command, re.sub(r'[^0-9]', '-', str(datetime.datetime.now(tz=datetime.timezone.utc).replace(microsecond=0, tzinfo=None)))) + + data = dict( + clouds=results, +diff --git a/test/lib/ansible_test/_internal/commands/sanity/pylint.py b/test/lib/ansible_test/_internal/commands/sanity/pylint.py +index fe5fbac9522fef..c089f834e50f89 100644 +--- a/test/lib/ansible_test/_internal/commands/sanity/pylint.py ++++ b/test/lib/ansible_test/_internal/commands/sanity/pylint.py +@@ -156,19 +156,19 @@ def context_filter(path_to_filter: str) -> bool: + except CollectionDetailError as ex: + display.warning('Skipping pylint collection version checks since collection detail loading failed: %s' % ex.reason) + +- test_start = datetime.datetime.utcnow() ++ test_start = datetime.datetime.now(tz=datetime.timezone.utc) + + for context, context_paths in sorted(contexts): + if not context_paths: + continue + +- context_start = datetime.datetime.utcnow() ++ context_start = datetime.datetime.now(tz=datetime.timezone.utc) + messages += self.pylint(args, context, context_paths, plugin_dir, plugin_names, python, collection_detail) +- context_end = datetime.datetime.utcnow() ++ context_end = datetime.datetime.now(tz=datetime.timezone.utc) + + context_times.append('%s: %d (%s)' % (context, len(context_paths), context_end - context_start)) + +- test_end = datetime.datetime.utcnow() ++ test_end = datetime.datetime.now(tz=datetime.timezone.utc) + + for context_time in context_times: + display.info(context_time, verbosity=4) +diff --git a/test/lib/ansible_test/_internal/test.py b/test/lib/ansible_test/_internal/test.py +index 4814ee3e25e525..fe5ef3f8d3472a 100644 +--- a/test/lib/ansible_test/_internal/test.py ++++ b/test/lib/ansible_test/_internal/test.py +@@ -114,7 +114,7 @@ def save_junit(self, args: TestConfig, test_case: junit_xml.TestCase) -> None: + junit_xml.TestSuite( + name='ansible-test', + cases=[test_case], +- timestamp=datetime.datetime.utcnow(), ++ timestamp=datetime.datetime.now(tz=datetime.timezone.utc), + ), + ], + ) +@@ -153,13 +153,11 @@ def write(self, args: TestConfig) -> None: + + output += '\n\nConsult the console log for additional details on where the timeout occurred.' + +- timestamp = datetime.datetime.utcnow() +- + suites = junit_xml.TestSuites( + suites=[ + junit_xml.TestSuite( + name='ansible-test', +- timestamp=timestamp, ++ timestamp=datetime.datetime.now(tz=datetime.timezone.utc), + cases=[ + junit_xml.TestCase( + name='timeout', diff --git a/fix-galaxy-cli-unit-test-asserts.patch b/fix-galaxy-cli-unit-test-asserts.patch new file mode 100644 index 0000000..b844844 --- /dev/null +++ b/fix-galaxy-cli-unit-test-asserts.patch @@ -0,0 +1,35 @@ +From 43c5cbcaef34aeb0141b8ad24027496bf6ec2acd Mon Sep 17 00:00:00 2001 +From: Matt Clay +Date: Wed, 12 Apr 2023 10:32:59 -0700 +Subject: [PATCH] Fix galaxy CLI unit test assertions (#80504) + +--- + test/units/cli/test_galaxy.py | 8 ++++++-- + 1 file changed, 6 insertions(+), 2 deletions(-) + +diff --git a/test/units/cli/test_galaxy.py b/test/units/cli/test_galaxy.py +index 8f7e891e7d091f..c21d75dcb9a5f1 100644 +--- a/test/units/cli/test_galaxy.py ++++ b/test/units/cli/test_galaxy.py +@@ -168,7 +168,9 @@ def test_exit_without_ignore_without_flag(self): + with patch.object(ansible.utils.display.Display, "display", return_value=None) as mocked_display: + # testing that error expected is raised + self.assertRaises(AnsibleError, gc.run) +- self.assertTrue(mocked_display.called_once_with("- downloading role 'fake_role_name', owned by ")) ++ assert mocked_display.call_count == 2 ++ assert mocked_display.mock_calls[0].args[0] == "Starting galaxy role install process" ++ assert "fake_role_name was NOT installed successfully" in mocked_display.mock_calls[1].args[0] + + def test_exit_without_ignore_with_flag(self): + ''' tests that GalaxyCLI exits without the error specified if the --ignore-errors flag is used ''' +@@ -176,7 +178,9 @@ def test_exit_without_ignore_with_flag(self): + gc = GalaxyCLI(args=["ansible-galaxy", "install", "--server=None", "fake_role_name", "--ignore-errors"]) + with patch.object(ansible.utils.display.Display, "display", return_value=None) as mocked_display: + gc.run() +- self.assertTrue(mocked_display.called_once_with("- downloading role 'fake_role_name', owned by ")) ++ assert mocked_display.call_count == 2 ++ assert mocked_display.mock_calls[0].args[0] == "Starting galaxy role install process" ++ assert "fake_role_name was NOT installed successfully" in mocked_display.mock_calls[1].args[0] + + def test_parse_no_action(self): + ''' testing the options parser when no action is given ''' diff --git a/fix-unit-test-asserts.patch b/fix-unit-test-asserts.patch new file mode 100644 index 0000000..fbeb8c5 --- /dev/null +++ b/fix-unit-test-asserts.patch @@ -0,0 +1,40 @@ +From 3ec828703f020551241b4169f6a3f07c701e240a Mon Sep 17 00:00:00 2001 +From: Matt Clay +Date: Wed, 12 Apr 2023 10:04:42 -0700 +Subject: [PATCH] Fix unit test asserts (#80500) + +--- + test/units/galaxy/test_collection.py | 6 +++--- + 1 file changed, 3 insertions(+), 3 deletions(-) + +diff --git a/test/units/galaxy/test_collection.py b/test/units/galaxy/test_collection.py +index 04a6a43436bbd1..5d6f32456dda89 100644 +--- a/test/units/galaxy/test_collection.py ++++ b/test/units/galaxy/test_collection.py +@@ -1060,7 +1060,7 @@ def test_verify_file_hash_deleted_file(manifest_info): + with patch.object(collection.os.path, 'isfile', MagicMock(return_value=False)) as mock_isfile: + collection._verify_file_hash(b'path/', 'file', digest, error_queue) + +- assert mock_isfile.called_once ++ mock_isfile.assert_called_once() + + assert len(error_queue) == 1 + assert error_queue[0].installed is None +@@ -1083,7 +1083,7 @@ def test_verify_file_hash_matching_hash(manifest_info): + with patch.object(collection.os.path, 'isfile', MagicMock(return_value=True)) as mock_isfile: + collection._verify_file_hash(b'path/', 'file', digest, error_queue) + +- assert mock_isfile.called_once ++ mock_isfile.assert_called_once() + + assert error_queue == [] + +@@ -1105,7 +1105,7 @@ def test_verify_file_hash_mismatching_hash(manifest_info): + with patch.object(collection.os.path, 'isfile', MagicMock(return_value=True)) as mock_isfile: + collection._verify_file_hash(b'path/', 'file', different_digest, error_queue) + +- assert mock_isfile.called_once ++ mock_isfile.assert_called_once() + + assert len(error_queue) == 1 + assert error_queue[0].installed == digest diff --git a/replace-deprecated-ast.value.s.patch b/replace-deprecated-ast.value.s.patch new file mode 100644 index 0000000..d3d3120 --- /dev/null +++ b/replace-deprecated-ast.value.s.patch @@ -0,0 +1,124 @@ +From 742d47fa15a5418f98abf9aaf07edf466e871c81 Mon Sep 17 00:00:00 2001 +From: Sloane Hertel <19572925+s-hertel@users.noreply.github.com> +Date: Tue, 6 Jun 2023 11:22:26 -0400 +Subject: [PATCH] replace deprecated ast.value.s with ast.value.value (#80968) + +* replace deprecated ast.value.s with ast.value.value + +the s attribute is deprecated since Python 3.8 and emits a warning in +3.12 causing some test failures +--- + .../fragments/80968-replace-deprecated-ast-attr.yml | 2 ++ + lib/ansible/parsing/plugin_docs.py | 4 ++-- + lib/ansible/playbook/conditional.py | 6 +++--- + .../sanity/validate-modules/validate_modules/main.py | 12 ++++++------ + .../_util/controller/sanity/yamllint/yamllinter.py | 6 +++--- + 5 files changed, 16 insertions(+), 14 deletions(-) + create mode 100644 changelogs/fragments/80968-replace-deprecated-ast-attr.yml + +diff --git a/changelogs/fragments/80968-replace-deprecated-ast-attr.yml b/changelogs/fragments/80968-replace-deprecated-ast-attr.yml +new file mode 100644 +index 00000000000000..13100ded3d1987 +--- /dev/null ++++ b/changelogs/fragments/80968-replace-deprecated-ast-attr.yml +@@ -0,0 +1,2 @@ ++bugfixes: ++ - Fix ``ast`` deprecation warnings for ``Str`` and ``value.s`` when using Python 3.12. +diff --git a/lib/ansible/parsing/plugin_docs.py b/lib/ansible/parsing/plugin_docs.py +index 7d3dca015cb341..253f62af68e2b1 100644 +--- a/lib/ansible/parsing/plugin_docs.py ++++ b/lib/ansible/parsing/plugin_docs.py +@@ -151,10 +151,10 @@ def read_docstring_from_python_file(filename, verbose=True, ignore_errors=True): + if theid == 'EXAMPLES': + # examples 'can' be yaml, but even if so, we dont want to parse as such here + # as it can create undesired 'objects' that don't display well as docs. +- data[varkey] = to_text(child.value.s) ++ data[varkey] = to_text(child.value.value) + else: + # string should be yaml if already not a dict +- data[varkey] = AnsibleLoader(child.value.s, file_name=filename).get_single_data() ++ data[varkey] = AnsibleLoader(child.value.value, file_name=filename).get_single_data() + + display.debug('Documentation assigned: %s' % varkey) + +diff --git a/lib/ansible/playbook/conditional.py b/lib/ansible/playbook/conditional.py +index 6b685ef6a8555d..163f9129c94935 100644 +--- a/lib/ansible/playbook/conditional.py ++++ b/lib/ansible/playbook/conditional.py +@@ -144,9 +144,9 @@ def generic_visit(self, node, inside_call=False, inside_yield=False): + inside_call = True + elif isinstance(node, ast.Yield): + inside_yield = True +- elif isinstance(node, ast.Str): ++ elif isinstance(node, ast.Constant) and isinstance(node.value, text_type): + if disable_lookups: +- if inside_call and node.s.startswith("__"): ++ if inside_call and node.value.startswith("__"): + # calling things with a dunder is generally bad at this point... + raise AnsibleError( + "Invalid access found in the conditional: '%s'" % conditional +@@ -154,7 +154,7 @@ def generic_visit(self, node, inside_call=False, inside_yield=False): + elif inside_yield: + # we're inside a yield, so recursively parse and traverse the AST + # of the result to catch forbidden syntax from executing +- parsed = ast.parse(node.s, mode='exec') ++ parsed = ast.parse(node.value, mode='exec') + cnv = CleansingNodeVisitor() + cnv.visit(parsed) + # iterate over all child nodes +diff --git a/test/lib/ansible_test/_util/controller/sanity/validate-modules/validate_modules/main.py b/test/lib/ansible_test/_util/controller/sanity/validate-modules/validate_modules/main.py +index fd5ea3ae788e17..2b92a56c2055dd 100644 +--- a/test/lib/ansible_test/_util/controller/sanity/validate-modules/validate_modules/main.py ++++ b/test/lib/ansible_test/_util/controller/sanity/validate-modules/validate_modules/main.py +@@ -808,22 +808,22 @@ def _get_py_docs(self): + continue + + if grandchild.id == 'DOCUMENTATION': +- docs['DOCUMENTATION']['value'] = child.value.s ++ docs['DOCUMENTATION']['value'] = child.value.value + docs['DOCUMENTATION']['lineno'] = child.lineno + docs['DOCUMENTATION']['end_lineno'] = ( +- child.lineno + len(child.value.s.splitlines()) ++ child.lineno + len(child.value.value.splitlines()) + ) + elif grandchild.id == 'EXAMPLES': +- docs['EXAMPLES']['value'] = child.value.s ++ docs['EXAMPLES']['value'] = child.value.value + docs['EXAMPLES']['lineno'] = child.lineno + docs['EXAMPLES']['end_lineno'] = ( +- child.lineno + len(child.value.s.splitlines()) ++ child.lineno + len(child.value.value.splitlines()) + ) + elif grandchild.id == 'RETURN': +- docs['RETURN']['value'] = child.value.s ++ docs['RETURN']['value'] = child.value.value + docs['RETURN']['lineno'] = child.lineno + docs['RETURN']['end_lineno'] = ( +- child.lineno + len(child.value.s.splitlines()) ++ child.lineno + len(child.value.value.splitlines()) + ) + + return docs +diff --git a/test/lib/ansible_test/_util/controller/sanity/yamllint/yamllinter.py b/test/lib/ansible_test/_util/controller/sanity/yamllint/yamllinter.py +index d6de6117b2328a..ed1afcf3a5efc0 100644 +--- a/test/lib/ansible_test/_util/controller/sanity/yamllint/yamllinter.py ++++ b/test/lib/ansible_test/_util/controller/sanity/yamllint/yamllinter.py +@@ -181,15 +181,15 @@ def check_assignment(statement, doc_types=None): + if doc_types and target.id not in doc_types: + continue + +- fmt_match = fmt_re.match(statement.value.s.lstrip()) ++ fmt_match = fmt_re.match(statement.value.value.lstrip()) + fmt = 'yaml' + if fmt_match: + fmt = fmt_match.group(1) + + docs[target.id] = dict( +- yaml=statement.value.s, ++ yaml=statement.value.value, + lineno=statement.lineno, +- end_lineno=statement.lineno + len(statement.value.s.splitlines()), ++ end_lineno=statement.lineno + len(statement.value.value.splitlines()), + fmt=fmt.lower(), + ) + diff --git a/support-Python-3.12-in-ansible-test.patch b/support-Python-3.12-in-ansible-test.patch new file mode 100644 index 0000000..0039483 --- /dev/null +++ b/support-Python-3.12-in-ansible-test.patch @@ -0,0 +1,151 @@ +From c10e3d21fcb54dd0a9dfe44d97355195166510a9 Mon Sep 17 00:00:00 2001 +From: s-hertel <19572925+s-hertel@users.noreply.github.com> +Date: Tue, 16 May 2023 13:48:12 -0400 +Subject: [PATCH] add Python 3.12 support to ansible-test + +skip docs build sanity test on Python 3.12 until aiohttp has a compatible version + +skip Python 3.12 tests on windows/networking until the default container is +updated + +add interpreter fallback? +--- + lib/ansible/config/base.yml | 1 + + setup.cfg | 1 + + .../minimum-build-constraints.txt | 4 +++- + test/lib/ansible_test/_data/requirements/ansible-test.txt | 2 +- + test/lib/ansible_test/_internal/bootstrap.py | 4 ++++ + test/lib/ansible_test/_internal/coverage_util.py | 2 +- + test/lib/ansible_test/_internal/python_requirements.py | 4 ++-- + test/lib/ansible_test/_util/target/common/constants.py | 1 + + test/lib/ansible_test/_util/target/setup/bootstrap.sh | 2 +- + test/sanity/code-smell/docs-build.json | 1 + + 10 files changed, 16 insertions(+), 6 deletions(-) + +diff --git a/lib/ansible/config/base.yml b/lib/ansible/config/base.yml +index 206deb76d2e916..469fbc6846b6d9 100644 +--- a/lib/ansible/config/base.yml ++++ b/lib/ansible/config/base.yml +@@ -1557,6 +1557,7 @@ _INTERPRETER_PYTHON_DISTRO_MAP: + INTERPRETER_PYTHON_FALLBACK: + name: Ordered list of Python interpreters to check for in discovery + default: ++ - python3.12 + - python3.11 + - python3.10 + - python3.9 +diff --git a/setup.cfg b/setup.cfg +index e020ee3b15caea..af79337f91e9de 100644 +--- a/setup.cfg ++++ b/setup.cfg +@@ -30,6 +30,7 @@ classifiers = + Programming Language :: Python :: 3.9 + Programming Language :: Python :: 3.10 + Programming Language :: Python :: 3.11 ++ Programming Language :: Python :: 3.12 + Programming Language :: Python :: 3 :: Only + Topic :: System :: Installation/Setup + Topic :: System :: Systems Administration +diff --git a/test/integration/targets/canonical-pep517-self-packaging/minimum-build-constraints.txt b/test/integration/targets/canonical-pep517-self-packaging/minimum-build-constraints.txt +index 3ba47aeb4b69b4..765ca85c17d8d3 100644 +--- a/test/integration/targets/canonical-pep517-self-packaging/minimum-build-constraints.txt ++++ b/test/integration/targets/canonical-pep517-self-packaging/minimum-build-constraints.txt +@@ -2,8 +2,10 @@ + setuptools == 57.0.0; python_version == "3.9" or python_version == "3.10" + + # Lowest supporting Python 3.11: +-setuptools == 60.0.0; python_version >= "3.11" ++setuptools == 60.0.0; python_version == "3.11" + ++# Lowest supporting Python 3.12: ++setuptools == 66.1.0; python_version >= "3.12" + + # An arbitrary old version that was released before Python 3.9.0: + wheel == 0.33.6 +diff --git a/test/lib/ansible_test/_data/requirements/ansible-test.txt b/test/lib/ansible_test/_data/requirements/ansible-test.txt +index f7cb9c27780856..8b1772fb915bc3 100644 +--- a/test/lib/ansible_test/_data/requirements/ansible-test.txt ++++ b/test/lib/ansible_test/_data/requirements/ansible-test.txt +@@ -1,4 +1,4 @@ + # The test-constraints sanity test verifies this file, but changes must be made manually to keep it in up-to-date. + virtualenv == 16.7.12 ; python_version < '3' +-coverage == 6.5.0 ; python_version >= '3.7' and python_version <= '3.11' ++coverage == 6.5.0 ; python_version >= '3.7' and python_version <= '3.12' + coverage == 4.5.4 ; python_version >= '2.6' and python_version <= '3.6' +diff --git a/test/lib/ansible_test/_internal/bootstrap.py b/test/lib/ansible_test/_internal/bootstrap.py +index b0cfb601d94497..1bd357679bb72c 100644 +--- a/test/lib/ansible_test/_internal/bootstrap.py ++++ b/test/lib/ansible_test/_internal/bootstrap.py +@@ -90,6 +90,10 @@ def get_variables(self) -> dict[str, t.Union[str, list[str]]]: + """The variables to template in the bootstrapping script.""" + variables = super().get_variables() + ++ # remove once the default docker container is updated ++ if self.platform in ('ios', 'vyos', 'windows'): ++ variables['python_versions'] = [version for version in self.python_versions if version != '3.12'] ++ + variables.update( + platform=self.platform, + platform_version=self.platform_version, +diff --git a/test/lib/ansible_test/_internal/coverage_util.py b/test/lib/ansible_test/_internal/coverage_util.py +index 0af1cac4e97c63..f9276d323c50fc 100644 +--- a/test/lib/ansible_test/_internal/coverage_util.py ++++ b/test/lib/ansible_test/_internal/coverage_util.py +@@ -69,7 +69,7 @@ class CoverageVersion: + + COVERAGE_VERSIONS = ( + # IMPORTANT: Keep this in sync with the ansible-test.txt requirements file. +- CoverageVersion('6.5.0', 7, (3, 7), (3, 11)), ++ CoverageVersion('6.5.0', 7, (3, 7), (3, 12)), + CoverageVersion('4.5.4', 0, (2, 6), (3, 6)), + ) + """ +diff --git a/test/lib/ansible_test/_internal/python_requirements.py b/test/lib/ansible_test/_internal/python_requirements.py +index fc88b637c2b242..1a07334d8637c5 100644 +--- a/test/lib/ansible_test/_internal/python_requirements.py ++++ b/test/lib/ansible_test/_internal/python_requirements.py +@@ -434,8 +434,8 @@ def get_venv_packages(python: PythonConfig) -> dict[str, str]: + # See: https://github.com/ansible/base-test-container/blob/main/files/installer.py + + default_packages = dict( +- pip='21.3.1', +- setuptools='60.8.2', ++ pip='23.1.2', ++ setuptools='67.7.2', + wheel='0.37.1', + ) + +diff --git a/test/lib/ansible_test/_util/target/common/constants.py b/test/lib/ansible_test/_util/target/common/constants.py +index 9bddfaf439563e..f3c3857ef97dbf 100644 +--- a/test/lib/ansible_test/_util/target/common/constants.py ++++ b/test/lib/ansible_test/_util/target/common/constants.py +@@ -17,4 +17,5 @@ + '3.9', + '3.10', + '3.11', ++ '3.12', + ) +diff --git a/test/lib/ansible_test/_util/target/setup/bootstrap.sh b/test/lib/ansible_test/_util/target/setup/bootstrap.sh +index ea17dad38751cc..367dcfcb4ce17e 100644 +--- a/test/lib/ansible_test/_util/target/setup/bootstrap.sh ++++ b/test/lib/ansible_test/_util/target/setup/bootstrap.sh +@@ -53,7 +53,7 @@ install_pip() { + pip_bootstrap_url="https://ci-files.testing.ansible.com/ansible-test/get-pip-20.3.4.py" + ;; + *) +- pip_bootstrap_url="https://ci-files.testing.ansible.com/ansible-test/get-pip-21.3.1.py" ++ pip_bootstrap_url="https://ci-files.testing.ansible.com/ansible-test/get-pip-23.1.2.py" + ;; + esac + +diff --git a/test/sanity/code-smell/docs-build.json b/test/sanity/code-smell/docs-build.json +index a43fa923b2b002..eedeca35dcba20 100644 +--- a/test/sanity/code-smell/docs-build.json ++++ b/test/sanity/code-smell/docs-build.json +@@ -1,5 +1,6 @@ + { + "disabled": true, ++ "maximum_python_version": "3.11", + "no_targets": true, + "output": "path-line-column-message" + } diff --git a/urls-remove-deprecated-client-key-calls.patch b/urls-remove-deprecated-client-key-calls.patch new file mode 100644 index 0000000..1e7d9cd --- /dev/null +++ b/urls-remove-deprecated-client-key-calls.patch @@ -0,0 +1,154 @@ +From 0df794e5a4fe4597ee65b0d492fbf0d0989d5ca0 Mon Sep 17 00:00:00 2001 +From: Jordan Borean +Date: Thu, 18 May 2023 08:17:25 +1000 +Subject: [PATCH] urls - remove deprecated client key calls (#80751) + +--- + .../fragments/urls-client-cert-py12.yml | 2 ++ + lib/ansible/module_utils/urls.py | 28 +++++++++++-------- + test/units/module_utils/urls/test_Request.py | 14 ++++------ + 3 files changed, 24 insertions(+), 20 deletions(-) + create mode 100644 changelogs/fragments/urls-client-cert-py12.yml + +diff --git a/changelogs/fragments/urls-client-cert-py12.yml b/changelogs/fragments/urls-client-cert-py12.yml +new file mode 100644 +index 00000000000000..aab129ed96e94b +--- /dev/null ++++ b/changelogs/fragments/urls-client-cert-py12.yml +@@ -0,0 +1,2 @@ ++bugfixes: ++- urls.py - fixed cert_file and key_file parameters when running on Python 3.12 - https://github.com/ansible/ansible/issues/80490 +diff --git a/lib/ansible/module_utils/urls.py b/lib/ansible/module_utils/urls.py +index 0e5fbb74c4fae2..0197d86e1033b2 100644 +--- a/lib/ansible/module_utils/urls.py ++++ b/lib/ansible/module_utils/urls.py +@@ -535,15 +535,18 @@ def __init__(self, message, import_traceback, module=None): + UnixHTTPSConnection = None + if hasattr(httplib, 'HTTPSConnection') and hasattr(urllib_request, 'HTTPSHandler'): + class CustomHTTPSConnection(httplib.HTTPSConnection): # type: ignore[no-redef] +- def __init__(self, *args, **kwargs): ++ def __init__(self, client_cert=None, client_key=None, *args, **kwargs): + httplib.HTTPSConnection.__init__(self, *args, **kwargs) + self.context = None + if HAS_SSLCONTEXT: + self.context = self._context + elif HAS_URLLIB3_PYOPENSSLCONTEXT: + self.context = self._context = PyOpenSSLContext(PROTOCOL) +- if self.context and self.cert_file: +- self.context.load_cert_chain(self.cert_file, self.key_file) ++ ++ self._client_cert = client_cert ++ self._client_key = client_key ++ if self.context and self._client_cert: ++ self.context.load_cert_chain(self._client_cert, self._client_key) + + def connect(self): + "Connect to a host on a given (SSL) port." +@@ -564,10 +567,10 @@ def connect(self): + if HAS_SSLCONTEXT or HAS_URLLIB3_PYOPENSSLCONTEXT: + self.sock = self.context.wrap_socket(sock, server_hostname=server_hostname) + elif HAS_URLLIB3_SSL_WRAP_SOCKET: +- self.sock = ssl_wrap_socket(sock, keyfile=self.key_file, cert_reqs=ssl.CERT_NONE, # pylint: disable=used-before-assignment +- certfile=self.cert_file, ssl_version=PROTOCOL, server_hostname=server_hostname) ++ self.sock = ssl_wrap_socket(sock, keyfile=self._client_key, cert_reqs=ssl.CERT_NONE, # pylint: disable=used-before-assignment ++ certfile=self._client_cert, ssl_version=PROTOCOL, server_hostname=server_hostname) + else: +- self.sock = ssl.wrap_socket(sock, keyfile=self.key_file, certfile=self.cert_file, ssl_version=PROTOCOL) ++ self.sock = ssl.wrap_socket(sock, keyfile=self._client_key, certfile=self._client_cert, ssl_version=PROTOCOL) + + class CustomHTTPSHandler(urllib_request.HTTPSHandler): # type: ignore[no-redef] + +@@ -602,10 +605,6 @@ def https_open(self, req): + return self.do_open(self._build_https_connection, req) + + def _build_https_connection(self, host, **kwargs): +- kwargs.update({ +- 'cert_file': self.client_cert, +- 'key_file': self.client_key, +- }) + try: + kwargs['context'] = self._context + except AttributeError: +@@ -613,7 +612,7 @@ def _build_https_connection(self, host, **kwargs): + if self._unix_socket: + return UnixHTTPSConnection(self._unix_socket)(host, **kwargs) + if not HAS_SSLCONTEXT: +- return CustomHTTPSConnection(host, **kwargs) ++ return CustomHTTPSConnection(host, client_cert=self.client_cert, client_key=self.client_key, **kwargs) + return httplib.HTTPSConnection(host, **kwargs) + + @contextmanager +@@ -979,7 +978,7 @@ def atexit_remove_file(filename): + pass + + +-def make_context(cafile=None, cadata=None, ciphers=None, validate_certs=True): ++def make_context(cafile=None, cadata=None, ciphers=None, validate_certs=True, client_cert=None, client_key=None): + if ciphers is None: + ciphers = [] + +@@ -1006,6 +1005,9 @@ def make_context(cafile=None, cadata=None, ciphers=None, validate_certs=True): + if ciphers: + context.set_ciphers(':'.join(map(to_native, ciphers))) + ++ if client_cert: ++ context.load_cert_chain(client_cert, keyfile=client_key) ++ + return context + + +@@ -1514,6 +1516,8 @@ def open(self, method, url, data=None, headers=None, use_proxy=None, + cadata=cadata, + ciphers=ciphers, + validate_certs=validate_certs, ++ client_cert=client_cert, ++ client_key=client_key, + ) + handlers.append(HTTPSClientAuthHandler(client_cert=client_cert, + client_key=client_key, +diff --git a/test/units/module_utils/urls/test_Request.py b/test/units/module_utils/urls/test_Request.py +index d2c4ea38012a49..a8bc3a0b6bde3b 100644 +--- a/test/units/module_utils/urls/test_Request.py ++++ b/test/units/module_utils/urls/test_Request.py +@@ -33,6 +33,7 @@ def install_opener_mock(mocker): + def test_Request_fallback(urlopen_mock, install_opener_mock, mocker): + here = os.path.dirname(__file__) + pem = os.path.join(here, 'fixtures/client.pem') ++ client_key = os.path.join(here, 'fixtures/client.key') + + cookies = cookiejar.CookieJar() + request = Request( +@@ -46,8 +47,8 @@ def test_Request_fallback(urlopen_mock, install_opener_mock, mocker): + http_agent='ansible-tests', + force_basic_auth=True, + follow_redirects='all', +- client_cert='/tmp/client.pem', +- client_key='/tmp/client.key', ++ client_cert=pem, ++ client_key=client_key, + cookies=cookies, + unix_socket='/foo/bar/baz.sock', + ca_path=pem, +@@ -68,8 +69,8 @@ def test_Request_fallback(urlopen_mock, install_opener_mock, mocker): + call(None, 'ansible-tests'), # http_agent + call(None, True), # force_basic_auth + call(None, 'all'), # follow_redirects +- call(None, '/tmp/client.pem'), # client_cert +- call(None, '/tmp/client.key'), # client_key ++ call(None, pem), # client_cert ++ call(None, client_key), # client_key + call(None, cookies), # cookies + call(None, '/foo/bar/baz.sock'), # unix_socket + call(None, pem), # ca_path +@@ -358,10 +359,7 @@ def test_Request_open_client_cert(urlopen_mock, install_opener_mock): + assert ssl_handler.client_cert == client_cert + assert ssl_handler.client_key == client_key + +- https_connection = ssl_handler._build_https_connection('ansible.com') +- +- assert https_connection.key_file == client_key +- assert https_connection.cert_file == client_cert ++ ssl_handler._build_https_connection('ansible.com') + + + def test_Request_open_cookies(urlopen_mock, install_opener_mock):