rpms/ansible-core
6
0
Fork 0
Code Issues 1 Pull Requests Releases Activity

Add initial Python 3.12 support

Browse Source 
Fixes: https://bugzilla.redhat.com/2196539
This commit is contained in:
Maxwell G 2023-05-12 02:36:03 +00:00
parent 47238f9977
commit 8f2b3503f1
No known key found for this signature in database
GPG Key ID: F79E4E25E8C661F8
9 changed files with 1019 additions and 3 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

26
Disable-test-that-calls-compat-code-removed-in-3.12.patch Normal file
View File

@ -0,0 +1,26 @@
From 0186f21027682a4c7013ddf75aa35f530fc2a090 Mon Sep 17 00:00:00 2001
From: Maxwell G <maxwell@gtmx.me>
Date: Tue, 13 Jun 2023 13:29:40 +0000
Subject: [PATCH] Disable test that calls compat code removed in 3.12
---
test/units/utils/collection_loader/test_collection_loader.py | 3 ++-
1 file changed, 2 insertions(+), 1 deletion(-)
diff --git a/test/units/utils/collection_loader/test_collection_loader.py b/test/units/utils/collection_loader/test_collection_loader.py
index 9be59307a9..9a27692ea5 100644
--- a/test/units/utils/collection_loader/test_collection_loader.py
+++ b/test/units/utils/collection_loader/test_collection_loader.py
@@ -29,7 +29,8 @@ def teardown(*args, **kwargs):
# BEGIN STANDALONE TESTS - these exercise behaviors of the individual components without the import machinery
-@pytest.mark.skipif(not PY3, reason='Testing Python 2 codepath (find_module) on Python 3')
+@pytest.mark.skipif(not PY3 or sys.version_info[:2] >= (3, 12),
+ reason='Testing Python 2 codepath (find_module) on Python 3')
def test_find_module_py3():
dir_to_a_file = os.path.dirname(ping_module.__file__)
path_hook_finder = _AnsiblePathHookFinder(_AnsibleCollectionFinder(), dir_to_a_file)
--
2.40.1

38
ansible-core.spec
View File

@ -12,14 +12,37 @@ Name: ansible-core
Summary: A radically simple IT automation system Summary: A radically simple IT automation system
Version: 2.15.0 Version: 2.15.0
%global uversion %{version_no_tilde %{quote:%nil}} %global uversion %{version_no_tilde %{quote:%nil}}
Release: 2%{?dist} Release: 3%{?dist}
# The main license is GPLv3+. Many of the files in lib/ansible/module_utils # The main license is GPLv3+. Many of the files in lib/ansible/module_utils
# are BSD licensed. There are various files scattered throughout the codebase # are BSD licensed. There are various files scattered throughout the codebase
# containing code under different licenses. # containing code under different licenses.
License: GPL-3.0-or-later AND BSD-2-Clause AND PSF-2.0 AND MIT AND Apache-2.0 License: GPL-3.0-or-later AND BSD-2-Clause AND PSF-2.0 AND MIT AND Apache-2.0
Source0: https://github.com/ansible/ansible/archive/v%{uversion}/%{name}-%{uversion}.tar.gz Source0: https://github.com/ansible/ansible/archive/v%{uversion}/%{name}-%{uversion}.tar.gz
Source1: build_manpages.py Source1: build_manpages.py
Patch: https://github.com/ansible/ansible/commit/734f38b2594692707d1fd3cbcfc8dc8a677f4ee3.patch#/GALAXY_COLLECTIONS_PATH_WARNINGS.patch Patch: https://github.com/ansible/ansible/commit/734f38b2594692707d1fd3cbcfc8dc8a677f4ee3.patch#/GALAXY_COLLECTIONS_PATH_WARNINGS.patch
# These patches are only applied on Rawhide to enable support for Python 3.12
# See https://bugzilla.redhat.com/2196539
#
# Essential #
# add Python 3.12 support to ansible-test (#80834)
Patch5000: https://github.com/ansible/ansible/pull/80834.patch#/support-Python-3.12-in-ansible-test.patch
# Fix unit test asserts (#80500)
Patch5001: https://github.com/ansible/ansible/commit/3ec828703f020551241b4169f6a3f07c701e240a.patch#/fix-unit-test-asserts.patch
# Fix galaxy CLI unit test assertions (#80504)
Patch5002: https://github.com/ansible/ansible/commit/43c5cbcaef34aeb0141b8ad24027496bf6ec2acd.patch#/fix-galaxy-cli-unit-test-asserts.patch
Patch5003: Disable-test-that-calls-compat-code-removed-in-3.12.patch
# Deprecations #
# ansible-test - Replace pytest-forked (#80525)
Patch6000: https://github.com/ansible/ansible/commit/676b731e6f7d60ce6fd48c0d1c883fc85f5c6537.patch#/ansible-test-replace-pytest-forked.patch
# ansible-test - Avoid use of deprecated utcnow (#80750)
Patch6001: https://github.com/ansible/ansible/commit/fd341265d001d4e6545ffb2b7d154340cb1f1931.patch#/avoid-use-of-deprecated-utcnow.patch
# urls - remove deprecated client key calls (#80751)
Patch6002: https://github.com/ansible/ansible/commit/0df794e5a4fe4597ee65b0d492fbf0d0989d5ca0.patch#/urls-remove-deprecated-client-key-calls.patch
# replace deprecated ast.value.s with ast.value.value (#80968)
Patch6003: https://github.com/ansible/ansible/commit/742d47fa15a5418f98abf9aaf07edf466e871c81.patch#/replace-deprecated-ast.value.s.patch
Url: https://ansible.com Url: https://ansible.com
BuildArch: noarch BuildArch: noarch
@ -99,7 +122,13 @@ This package installs extensive documentation for ansible-core
%prep %prep
%autosetup -p1 -n ansible-%{uversion} %autosetup -N -n ansible-%{uversion}
%autopatch -M 4999 -p1
# Python 3.12 specific patches
# Set `-D '_has_python312 1'` to test locally
%if 0%{?_has_python312} || v"%{python3_version}" >= v"3.12"
%autopatch -m 5000 -p1
%endif
sed -i -s 's|/usr/bin/env python|%{python3}|' \ sed -i -s 's|/usr/bin/env python|%{python3}|' \
bin/ansible-test \ bin/ansible-test \
@ -232,7 +261,7 @@ install -Dpm 0644 licenses/* -t %{buildroot}%{_pkglicensedir}
%check %check
%if %{with tests} %if %{with tests}
%{python3} bin/ansible-test \ %{python3} bin/ansible-test \
units --local --python-interpreter %{python3} units --local --python-interpreter %{python3} -vv
%endif %endif
@ -258,6 +287,9 @@ install -Dpm 0644 licenses/* -t %{buildroot}%{_pkglicensedir}
%changelog %changelog
* Tue Jun 13 2023 Maxwell G <maxwell@gtmx.me> - 2.15.0-3
- Add support for Python 3.12. Fixes rhbz#2196539.
* Tue May 23 2023 Yaakov Selkowitz <yselkowi@redhat.com> - 2.15.0-2 * Tue May 23 2023 Yaakov Selkowitz <yselkowi@redhat.com> - 2.15.0-2
- Disable tests in RHEL builds - Disable tests in RHEL builds

310
ansible-test-replace-pytest-forked.patch Normal file
View File

@ -0,0 +1,310 @@
From 676b731e6f7d60ce6fd48c0d1c883fc85f5c6537 Mon Sep 17 00:00:00 2001
From: Matt Clay <matt@mystile.com>
Date: Fri, 14 Apr 2023 15:13:58 -0700
Subject: [PATCH] ansible-test - Replace pytest-forked (#80525)
- Unit tests now report warnings generated during test runs.
- Python 3.12 warnings about `os.fork` usage with threads (due to `pytest-xdist`) are suppressed.
- Added integration tests to verify forked test behavior.
---
.../fragments/ansible-test-pytest-forked.yml | 5 +
.../ansible-test-units-assertions/runme.sh | 2 +-
.../targets/ansible-test-units-forked/aliases | 5 +
.../plugins/modules/test_ansible_forked.py | 43 ++++++++
.../ansible-test-units-forked/runme.sh | 45 ++++++++
.../ansible_test/_data/requirements/units.txt | 1 -
.../_internal/commands/units/__init__.py | 3 +-
.../pylint/config/ansible-test-target.cfg | 2 +
.../target/pytest/plugins/ansible_forked.py | 103 ++++++++++++++++++
9 files changed, 206 insertions(+), 3 deletions(-)
create mode 100644 changelogs/fragments/ansible-test-pytest-forked.yml
create mode 100644 test/integration/targets/ansible-test-units-forked/aliases
create mode 100644 test/integration/targets/ansible-test-units-forked/ansible_collections/ns/col/tests/unit/plugins/modules/test_ansible_forked.py
create mode 100755 test/integration/targets/ansible-test-units-forked/runme.sh
create mode 100644 test/lib/ansible_test/_util/target/pytest/plugins/ansible_forked.py
diff --git a/changelogs/fragments/ansible-test-pytest-forked.yml b/changelogs/fragments/ansible-test-pytest-forked.yml
new file mode 100644
index 00000000000000..f8fae8139460a3
--- /dev/null
+++ b/changelogs/fragments/ansible-test-pytest-forked.yml
@@ -0,0 +1,5 @@
+minor_changes:
+ - ansible-test - Replace the ``pytest-forked`` pytest plugin with a custom plugin.
+bugfixes:
+ - ansible-test - Unit tests now report warnings generated during test runs.
+ Previously only warnings generated during test collection were reported.
diff --git a/test/integration/targets/ansible-test-units-assertions/runme.sh b/test/integration/targets/ansible-test-units-assertions/runme.sh
index 3511e765004a73..86fe5c818112f7 100755
--- a/test/integration/targets/ansible-test-units-assertions/runme.sh
+++ b/test/integration/targets/ansible-test-units-assertions/runme.sh
@@ -8,7 +8,7 @@ options=$("${TEST_DIR}"/../ansible-test/venv-pythons.py --only-versions)
IFS=', ' read -r -a pythons <<< "${options}"
for python in "${pythons[@]}"; do
- if ansible-test units --color --truncate 0 --python "${python}" --requirements "${@}" 2>&1 | tee pytest.log; then
+ if ansible-test units --truncate 0 --python "${python}" --requirements "${@}" 2>&1 | tee pytest.log; then
echo "Test did not fail as expected."
exit 1
fi
diff --git a/test/integration/targets/ansible-test-units-forked/aliases b/test/integration/targets/ansible-test-units-forked/aliases
new file mode 100644
index 00000000000000..79d7dbd7b09e91
--- /dev/null
+++ b/test/integration/targets/ansible-test-units-forked/aliases
@@ -0,0 +1,5 @@
+shippable/posix/group3 # runs in the distro test containers
+shippable/generic/group1 # runs in the default test container
+context/controller
+needs/target/collection
+needs/target/ansible-test
diff --git a/test/integration/targets/ansible-test-units-forked/ansible_collections/ns/col/tests/unit/plugins/modules/test_ansible_forked.py b/test/integration/targets/ansible-test-units-forked/ansible_collections/ns/col/tests/unit/plugins/modules/test_ansible_forked.py
new file mode 100644
index 00000000000000..828099c65e4cf3
--- /dev/null
+++ b/test/integration/targets/ansible-test-units-forked/ansible_collections/ns/col/tests/unit/plugins/modules/test_ansible_forked.py
@@ -0,0 +1,43 @@
+"""Unit tests to verify the functionality of the ansible-forked pytest plugin."""
+from __future__ import absolute_import, division, print_function
+
+__metaclass__ = type
+
+import os
+import pytest
+import signal
+import sys
+import warnings
+
+
+warnings.warn("This verifies that warnings generated during test collection are reported.")
+
+
+@pytest.mark.xfail
+def test_kill_xfail():
+ os.kill(os.getpid(), signal.SIGKILL) # causes pytest to report stdout and stderr
+
+
+def test_kill():
+ os.kill(os.getpid(), signal.SIGKILL) # causes pytest to report stdout and stderr
+
+
+@pytest.mark.xfail
+def test_exception_xfail():
+ sys.stdout.write("This stdout message should be hidden due to xfail.")
+ sys.stderr.write("This stderr message should be hidden due to xfail.")
+ raise Exception("This error is expected, but should be hidden due to xfail.")
+
+
+def test_exception():
+ sys.stdout.write("This stdout message should be reported since we're throwing an exception.")
+ sys.stderr.write("This stderr message should be reported since we're throwing an exception.")
+ raise Exception("This error is expected and should be visible.")
+
+
+def test_warning():
+ warnings.warn("This verifies that warnings generated at test time are reported.")
+
+
+def test_passed():
+ pass
diff --git a/test/integration/targets/ansible-test-units-forked/runme.sh b/test/integration/targets/ansible-test-units-forked/runme.sh
new file mode 100755
index 00000000000000..c39f3c492440e7
--- /dev/null
+++ b/test/integration/targets/ansible-test-units-forked/runme.sh
@@ -0,0 +1,45 @@
+#!/usr/bin/env bash
+
+source ../collection/setup.sh
+
+set -x
+
+options=$("${TEST_DIR}"/../ansible-test/venv-pythons.py --only-versions)
+IFS=', ' read -r -a pythons <<< "${options}"
+
+for python in "${pythons[@]}"; do
+ echo "*** Checking Python ${python} ***"
+
+ if ansible-test units --truncate 0 --target-python "venv/${python}" "${@}" > output.log 2>&1 ; then
+ cat output.log
+ echo "Unit tests on Python ${python} did not fail as expected. See output above."
+ exit 1
+ fi
+
+ cat output.log
+ echo "Unit tests on Python ${python} failed as expected. See output above. Checking for expected output ..."
+
+ # Verify that the appropriate tests pased, failed or xfailed.
+ grep 'PASSED tests/unit/plugins/modules/test_ansible_forked.py::test_passed' output.log
+ grep 'PASSED tests/unit/plugins/modules/test_ansible_forked.py::test_warning' output.log
+ grep 'XFAIL tests/unit/plugins/modules/test_ansible_forked.py::test_kill_xfail' output.log
+ grep 'FAILED tests/unit/plugins/modules/test_ansible_forked.py::test_kill' output.log
+ grep 'FAILED tests/unit/plugins/modules/test_ansible_forked.py::test_exception' output.log
+ grep 'XFAIL tests/unit/plugins/modules/test_ansible_forked.py::test_exception_xfail' output.log
+
+ # Verify that warnings are properly surfaced.
+ grep 'UserWarning: This verifies that warnings generated at test time are reported.' output.log
+ grep 'UserWarning: This verifies that warnings generated during test collection are reported.' output.log
+
+ # Verify there are no unexpected warnings.
+ grep 'Warning' output.log | grep -v 'UserWarning: This verifies that warnings generated ' && exit 1
+
+ # Verify that details from failed tests are properly surfaced.
+ grep "^Test CRASHED with exit code -9.$" output.log
+ grep "^This stdout message should be reported since we're throwing an exception.$" output.log
+ grep "^This stderr message should be reported since we're throwing an exception.$" output.log
+ grep '^> *raise Exception("This error is expected and should be visible.")$' output.log
+ grep "^E *Exception: This error is expected and should be visible.$" output.log
+
+ echo "*** Done Checking Python ${python} ***"
+done
diff --git a/test/lib/ansible_test/_data/requirements/units.txt b/test/lib/ansible_test/_data/requirements/units.txt
index d2f56d35a92a5c..d723a65fc663d0 100644
--- a/test/lib/ansible_test/_data/requirements/units.txt
+++ b/test/lib/ansible_test/_data/requirements/units.txt
@@ -2,5 +2,4 @@ mock
pytest
pytest-mock
pytest-xdist
-pytest-forked
pyyaml # required by the collection loader (only needed for collections)
diff --git a/test/lib/ansible_test/_internal/commands/units/__init__.py b/test/lib/ansible_test/_internal/commands/units/__init__.py
index 7d192e1be67148..78dd8498156615 100644
--- a/test/lib/ansible_test/_internal/commands/units/__init__.py
+++ b/test/lib/ansible_test/_internal/commands/units/__init__.py
@@ -253,7 +253,6 @@ def command_units(args: UnitsConfig) -> None:
cmd = [
'pytest',
- '--forked',
'-r', 'a',
'-n', str(args.num_workers) if args.num_workers else 'auto',
'--color', 'yes' if args.color else 'no',
@@ -275,6 +274,8 @@ def command_units(args: UnitsConfig) -> None:
if data_context().content.collection:
plugins.append('ansible_pytest_collections')
+ plugins.append('ansible_forked')
+
if plugins:
env['PYTHONPATH'] += ':%s' % os.path.join(ANSIBLE_TEST_TARGET_ROOT, 'pytest/plugins')
env['PYTEST_PLUGINS'] = ','.join(plugins)
diff --git a/test/lib/ansible_test/_util/controller/sanity/pylint/config/ansible-test-target.cfg b/test/lib/ansible_test/_util/controller/sanity/pylint/config/ansible-test-target.cfg
index e35301dd81c1bd..f8a0a8af3ff21c 100644
--- a/test/lib/ansible_test/_util/controller/sanity/pylint/config/ansible-test-target.cfg
+++ b/test/lib/ansible_test/_util/controller/sanity/pylint/config/ansible-test-target.cfg
@@ -57,3 +57,5 @@ preferred-modules =
# Listing them here makes it possible to enable the import-error check.
ignored-modules =
py,
+ pytest,
+ _pytest.runner,
diff --git a/test/lib/ansible_test/_util/target/pytest/plugins/ansible_forked.py b/test/lib/ansible_test/_util/target/pytest/plugins/ansible_forked.py
new file mode 100644
index 00000000000000..d00d9e93d1be00
--- /dev/null
+++ b/test/lib/ansible_test/_util/target/pytest/plugins/ansible_forked.py
@@ -0,0 +1,103 @@
+"""Run each test in its own fork. PYTEST_DONT_REWRITE"""
+# MIT License (see licenses/MIT-license.txt or https://opensource.org/licenses/MIT)
+# Based on code originally from:
+# https://github.com/pytest-dev/pytest-forked
+# https://github.com/pytest-dev/py
+# TIP: Disable pytest-xdist when debugging internal errors in this plugin.
+from __future__ import absolute_import, division, print_function
+
+__metaclass__ = type
+
+import os
+import pickle
+import tempfile
+import warnings
+
+from pytest import Item, hookimpl
+
+try:
+ from pytest import TestReport
+except ImportError:
+ from _pytest.runner import TestReport # Backwards compatibility with pytest < 7. Remove once Python 2.7 is not supported.
+
+from _pytest.runner import runtestprotocol
+
+
+@hookimpl(tryfirst=True)
+def pytest_runtest_protocol(item, nextitem): # type: (Item, Item | None) -> object | None
+ """Entry point for enabling this plugin."""
+ # This is needed because pytest-xdist creates an OS thread (using execnet).
+ # See: https://github.com/pytest-dev/execnet/blob/d6aa1a56773c2e887515d63e50b1d08338cb78a7/execnet/gateway_base.py#L51
+ warnings.filterwarnings("ignore", "^This process .* is multi-threaded, use of .* may lead to deadlocks in the child.$", DeprecationWarning)
+
+ item_hook = item.ihook
+ item_hook.pytest_runtest_logstart(nodeid=item.nodeid, location=item.location)
+
+ reports = run_item(item, nextitem)
+
+ for report in reports:
+ item_hook.pytest_runtest_logreport(report=report)
+
+ item_hook.pytest_runtest_logfinish(nodeid=item.nodeid, location=item.location)
+
+ return True
+
+
+def run_item(item, nextitem): # type: (Item, Item | None) -> list[TestReport]
+ """Run the item in a child process and return a list of reports."""
+ with tempfile.NamedTemporaryFile() as temp_file:
+ pid = os.fork()
+
+ if not pid:
+ temp_file.delete = False
+ run_child(item, nextitem, temp_file.name)
+
+ return run_parent(item, pid, temp_file.name)
+
+
+def run_child(item, nextitem, result_path): # type: (Item, Item | None, str) -> None
+ """Run the item, record the result and exit. Called in the child process."""
+ with warnings.catch_warnings(record=True) as captured_warnings:
+ reports = runtestprotocol(item, nextitem=nextitem, log=False)
+
+ with open(result_path, "wb") as result_file:
+ pickle.dump((reports, captured_warnings), result_file)
+
+ os._exit(0) # noqa
+
+
+def run_parent(item, pid, result_path): # type: (Item, int, str) -> list[TestReport]
+ """Wait for the child process to exit and return the test reports. Called in the parent process."""
+ exit_code = waitstatus_to_exitcode(os.waitpid(pid, 0)[1])
+
+ if exit_code:
+ reason = "Test CRASHED with exit code {}.".format(exit_code)
+ report = TestReport(item.nodeid, item.location, {x: 1 for x in item.keywords}, "failed", reason, "call", user_properties=item.user_properties)
+
+ if item.get_closest_marker("xfail"):
+ report.outcome = "skipped"
+ report.wasxfail = reason
+
+ reports = [report]
+ else:
+ with open(result_path, "rb") as result_file:
+ reports, captured_warnings = pickle.load(result_file) # type: list[TestReport], list[warnings.WarningMessage]
+
+ for warning in captured_warnings:
+ warnings.warn_explicit(warning.message, warning.category, warning.filename, warning.lineno)
+
+ return reports
+
+
+def waitstatus_to_exitcode(status): # type: (int) -> int
+ """Convert a wait status to an exit code."""
+ # This function was added in Python 3.9.
+ # See: https://docs.python.org/3/library/os.html#os.waitstatus_to_exitcode
+
+ if os.WIFEXITED(status):
+ return os.WEXITSTATUS(status)
+
+ if os.WIFSIGNALED(status):
+ return -os.WTERMSIG(status)
+
+ raise ValueError(status)

144
avoid-use-of-deprecated-utcnow.patch Normal file
View File

@ -0,0 +1,144 @@
From fd341265d001d4e6545ffb2b7d154340cb1f1931 Mon Sep 17 00:00:00 2001
From: Matt Clay <matt@mystile.com>
Date: Wed, 10 May 2023 11:26:49 -0700
Subject: [PATCH] ansible-test - Avoid use of deprecated utcnow (#80750)
The timestamps are only used by ansible-test, not the junit callback, so this change only impacts ansible-test.
---
changelogs/fragments/ansible-test-utcnow.yml | 2 ++
lib/ansible/utils/_junit_xml.py | 6 +++++-
test/lib/ansible_test/_internal/commands/env/__init__.py | 2 +-
.../_internal/commands/integration/__init__.py | 2 +-
.../_internal/commands/integration/cloud/__init__.py | 2 +-
test/lib/ansible_test/_internal/commands/sanity/pylint.py | 8 ++++----
test/lib/ansible_test/_internal/test.py | 6 ++----
7 files changed, 16 insertions(+), 12 deletions(-)
create mode 100644 changelogs/fragments/ansible-test-utcnow.yml
diff --git a/changelogs/fragments/ansible-test-utcnow.yml b/changelogs/fragments/ansible-test-utcnow.yml
new file mode 100644
index 00000000000000..0781a0cb48a1f8
--- /dev/null
+++ b/changelogs/fragments/ansible-test-utcnow.yml
@@ -0,0 +1,2 @@
+minor_changes:
+ - ansible-test - Use ``datetime.datetime.now`` with ``tz`` specified instead of ``datetime.datetime.utcnow``.
diff --git a/lib/ansible/utils/_junit_xml.py b/lib/ansible/utils/_junit_xml.py
index 6b7c80f4cdf2de..8c4dba013edcb7 100644
--- a/lib/ansible/utils/_junit_xml.py
+++ b/lib/ansible/utils/_junit_xml.py
@@ -144,6 +144,10 @@ class TestSuite:
system_out: str | None = None
system_err: str | None = None
+ def __post_init__(self):
+ if self.timestamp and self.timestamp.tzinfo != datetime.timezone.utc:
+ raise ValueError(f'timestamp.tzinfo must be {datetime.timezone.utc!r}')
+
@property
def disabled(self) -> int:
"""The number of disabled test cases."""
@@ -187,7 +191,7 @@ def get_attributes(self) -> dict[str, str]:
skipped=self.skipped,
tests=self.tests,
time=self.time,
- timestamp=self.timestamp.isoformat(timespec='seconds') if self.timestamp else None,
+ timestamp=self.timestamp.replace(tzinfo=None).isoformat(timespec='seconds') if self.timestamp else None,
)
def get_xml_element(self) -> ET.Element:
diff --git a/test/lib/ansible_test/_internal/commands/env/__init__.py b/test/lib/ansible_test/_internal/commands/env/__init__.py
index 44f229f87999d7..6c0510566a052e 100644
--- a/test/lib/ansible_test/_internal/commands/env/__init__.py
+++ b/test/lib/ansible_test/_internal/commands/env/__init__.py
@@ -85,7 +85,7 @@ def show_dump_env(args: EnvConfig) -> None:
),
git=get_ci_provider().get_git_details(args),
platform=dict(
- datetime=datetime.datetime.utcnow().strftime('%Y-%m-%dT%H:%M:%SZ'),
+ datetime=datetime.datetime.now(tz=datetime.timezone.utc).strftime('%Y-%m-%dT%H:%M:%SZ'),
platform=platform.platform(),
uname=platform.uname(),
),
diff --git a/test/lib/ansible_test/_internal/commands/integration/__init__.py b/test/lib/ansible_test/_internal/commands/integration/__init__.py
index 0e5abbb6532193..5bd04407beeec5 100644
--- a/test/lib/ansible_test/_internal/commands/integration/__init__.py
+++ b/test/lib/ansible_test/_internal/commands/integration/__init__.py
@@ -566,7 +566,7 @@ def command_integration_filtered(
coverage_manager.teardown()
result_name = '%s-%s.json' % (
- args.command, re.sub(r'[^0-9]', '-', str(datetime.datetime.utcnow().replace(microsecond=0))))
+ args.command, re.sub(r'[^0-9]', '-', str(datetime.datetime.now(tz=datetime.timezone.utc).replace(microsecond=0, tzinfo=None))))
data = dict(
targets=results,
diff --git a/test/lib/ansible_test/_internal/commands/integration/cloud/__init__.py b/test/lib/ansible_test/_internal/commands/integration/cloud/__init__.py
index cad84a368924b2..eac9265a4f8c53 100644
--- a/test/lib/ansible_test/_internal/commands/integration/cloud/__init__.py
+++ b/test/lib/ansible_test/_internal/commands/integration/cloud/__init__.py
@@ -170,7 +170,7 @@ def cloud_init(args: IntegrationConfig, targets: tuple[IntegrationTarget, ...])
if not args.explain and results:
result_name = '%s-%s.json' % (
- args.command, re.sub(r'[^0-9]', '-', str(datetime.datetime.utcnow().replace(microsecond=0))))
+ args.command, re.sub(r'[^0-9]', '-', str(datetime.datetime.now(tz=datetime.timezone.utc).replace(microsecond=0, tzinfo=None))))
data = dict(
clouds=results,
diff --git a/test/lib/ansible_test/_internal/commands/sanity/pylint.py b/test/lib/ansible_test/_internal/commands/sanity/pylint.py
index fe5fbac9522fef..c089f834e50f89 100644
--- a/test/lib/ansible_test/_internal/commands/sanity/pylint.py
+++ b/test/lib/ansible_test/_internal/commands/sanity/pylint.py
@@ -156,19 +156,19 @@ def context_filter(path_to_filter: str) -> bool:
except CollectionDetailError as ex:
display.warning('Skipping pylint collection version checks since collection detail loading failed: %s' % ex.reason)
- test_start = datetime.datetime.utcnow()
+ test_start = datetime.datetime.now(tz=datetime.timezone.utc)
for context, context_paths in sorted(contexts):
if not context_paths:
continue
- context_start = datetime.datetime.utcnow()
+ context_start = datetime.datetime.now(tz=datetime.timezone.utc)
messages += self.pylint(args, context, context_paths, plugin_dir, plugin_names, python, collection_detail)
- context_end = datetime.datetime.utcnow()
+ context_end = datetime.datetime.now(tz=datetime.timezone.utc)
context_times.append('%s: %d (%s)' % (context, len(context_paths), context_end - context_start))
- test_end = datetime.datetime.utcnow()
+ test_end = datetime.datetime.now(tz=datetime.timezone.utc)
for context_time in context_times:
display.info(context_time, verbosity=4)
diff --git a/test/lib/ansible_test/_internal/test.py b/test/lib/ansible_test/_internal/test.py
index 4814ee3e25e525..fe5ef3f8d3472a 100644
--- a/test/lib/ansible_test/_internal/test.py
+++ b/test/lib/ansible_test/_internal/test.py
@@ -114,7 +114,7 @@ def save_junit(self, args: TestConfig, test_case: junit_xml.TestCase) -> None:
junit_xml.TestSuite(
name='ansible-test',
cases=[test_case],
- timestamp=datetime.datetime.utcnow(),
+ timestamp=datetime.datetime.now(tz=datetime.timezone.utc),
),
],
)
@@ -153,13 +153,11 @@ def write(self, args: TestConfig) -> None:
output += '\n\nConsult the console log for additional details on where the timeout occurred.'
- timestamp = datetime.datetime.utcnow()
-
suites = junit_xml.TestSuites(
suites=[
junit_xml.TestSuite(
name='ansible-test',
- timestamp=timestamp,
+ timestamp=datetime.datetime.now(tz=datetime.timezone.utc),
cases=[
junit_xml.TestCase(
name='timeout',

35
fix-galaxy-cli-unit-test-asserts.patch Normal file
View File

@ -0,0 +1,35 @@
From 43c5cbcaef34aeb0141b8ad24027496bf6ec2acd Mon Sep 17 00:00:00 2001
From: Matt Clay <matt@mystile.com>
Date: Wed, 12 Apr 2023 10:32:59 -0700
Subject: [PATCH] Fix galaxy CLI unit test assertions (#80504)
---
test/units/cli/test_galaxy.py | 8 ++++++--
1 file changed, 6 insertions(+), 2 deletions(-)
diff --git a/test/units/cli/test_galaxy.py b/test/units/cli/test_galaxy.py
index 8f7e891e7d091f..c21d75dcb9a5f1 100644
--- a/test/units/cli/test_galaxy.py
+++ b/test/units/cli/test_galaxy.py
@@ -168,7 +168,9 @@ def test_exit_without_ignore_without_flag(self):
with patch.object(ansible.utils.display.Display, "display", return_value=None) as mocked_display:
# testing that error expected is raised
self.assertRaises(AnsibleError, gc.run)
- self.assertTrue(mocked_display.called_once_with("- downloading role 'fake_role_name', owned by "))
+ assert mocked_display.call_count == 2
+ assert mocked_display.mock_calls[0].args[0] == "Starting galaxy role install process"
+ assert "fake_role_name was NOT installed successfully" in mocked_display.mock_calls[1].args[0]
def test_exit_without_ignore_with_flag(self):
''' tests that GalaxyCLI exits without the error specified if the --ignore-errors flag is used '''
@@ -176,7 +178,9 @@ def test_exit_without_ignore_with_flag(self):
gc = GalaxyCLI(args=["ansible-galaxy", "install", "--server=None", "fake_role_name", "--ignore-errors"])
with patch.object(ansible.utils.display.Display, "display", return_value=None) as mocked_display:
gc.run()
- self.assertTrue(mocked_display.called_once_with("- downloading role 'fake_role_name', owned by "))
+ assert mocked_display.call_count == 2
+ assert mocked_display.mock_calls[0].args[0] == "Starting galaxy role install process"
+ assert "fake_role_name was NOT installed successfully" in mocked_display.mock_calls[1].args[0]
def test_parse_no_action(self):
''' testing the options parser when no action is given '''

40
fix-unit-test-asserts.patch Normal file
View File

@ -0,0 +1,40 @@
From 3ec828703f020551241b4169f6a3f07c701e240a Mon Sep 17 00:00:00 2001
From: Matt Clay <matt@mystile.com>
Date: Wed, 12 Apr 2023 10:04:42 -0700
Subject: [PATCH] Fix unit test asserts (#80500)
---
test/units/galaxy/test_collection.py | 6 +++---
1 file changed, 3 insertions(+), 3 deletions(-)
diff --git a/test/units/galaxy/test_collection.py b/test/units/galaxy/test_collection.py
index 04a6a43436bbd1..5d6f32456dda89 100644
--- a/test/units/galaxy/test_collection.py
+++ b/test/units/galaxy/test_collection.py
@@ -1060,7 +1060,7 @@ def test_verify_file_hash_deleted_file(manifest_info):
with patch.object(collection.os.path, 'isfile', MagicMock(return_value=False)) as mock_isfile:
collection._verify_file_hash(b'path/', 'file', digest, error_queue)
- assert mock_isfile.called_once
+ mock_isfile.assert_called_once()
assert len(error_queue) == 1
assert error_queue[0].installed is None
@@ -1083,7 +1083,7 @@ def test_verify_file_hash_matching_hash(manifest_info):
with patch.object(collection.os.path, 'isfile', MagicMock(return_value=True)) as mock_isfile:
collection._verify_file_hash(b'path/', 'file', digest, error_queue)
- assert mock_isfile.called_once
+ mock_isfile.assert_called_once()
assert error_queue == []
@@ -1105,7 +1105,7 @@ def test_verify_file_hash_mismatching_hash(manifest_info):
with patch.object(collection.os.path, 'isfile', MagicMock(return_value=True)) as mock_isfile:
collection._verify_file_hash(b'path/', 'file', different_digest, error_queue)
- assert mock_isfile.called_once
+ mock_isfile.assert_called_once()
assert len(error_queue) == 1
assert error_queue[0].installed == digest

124
replace-deprecated-ast.value.s.patch Normal file
View File

@ -0,0 +1,124 @@
From 742d47fa15a5418f98abf9aaf07edf466e871c81 Mon Sep 17 00:00:00 2001
From: Sloane Hertel <19572925+s-hertel@users.noreply.github.com>
Date: Tue, 6 Jun 2023 11:22:26 -0400
Subject: [PATCH] replace deprecated ast.value.s with ast.value.value (#80968)
* replace deprecated ast.value.s with ast.value.value
the s attribute is deprecated since Python 3.8 and emits a warning in
3.12 causing some test failures
---
.../fragments/80968-replace-deprecated-ast-attr.yml | 2 ++
lib/ansible/parsing/plugin_docs.py | 4 ++--
lib/ansible/playbook/conditional.py | 6 +++---
.../sanity/validate-modules/validate_modules/main.py | 12 ++++++------
.../_util/controller/sanity/yamllint/yamllinter.py | 6 +++---
5 files changed, 16 insertions(+), 14 deletions(-)
create mode 100644 changelogs/fragments/80968-replace-deprecated-ast-attr.yml
diff --git a/changelogs/fragments/80968-replace-deprecated-ast-attr.yml b/changelogs/fragments/80968-replace-deprecated-ast-attr.yml
new file mode 100644
index 00000000000000..13100ded3d1987
--- /dev/null
+++ b/changelogs/fragments/80968-replace-deprecated-ast-attr.yml
@@ -0,0 +1,2 @@
+bugfixes:
+ - Fix ``ast`` deprecation warnings for ``Str`` and ``value.s`` when using Python 3.12.
diff --git a/lib/ansible/parsing/plugin_docs.py b/lib/ansible/parsing/plugin_docs.py
index 7d3dca015cb341..253f62af68e2b1 100644
--- a/lib/ansible/parsing/plugin_docs.py
+++ b/lib/ansible/parsing/plugin_docs.py
@@ -151,10 +151,10 @@ def read_docstring_from_python_file(filename, verbose=True, ignore_errors=True):
if theid == 'EXAMPLES':
# examples 'can' be yaml, but even if so, we dont want to parse as such here
# as it can create undesired 'objects' that don't display well as docs.
- data[varkey] = to_text(child.value.s)
+ data[varkey] = to_text(child.value.value)
else:
# string should be yaml if already not a dict
- data[varkey] = AnsibleLoader(child.value.s, file_name=filename).get_single_data()
+ data[varkey] = AnsibleLoader(child.value.value, file_name=filename).get_single_data()
display.debug('Documentation assigned: %s' % varkey)
diff --git a/lib/ansible/playbook/conditional.py b/lib/ansible/playbook/conditional.py
index 6b685ef6a8555d..163f9129c94935 100644
--- a/lib/ansible/playbook/conditional.py
+++ b/lib/ansible/playbook/conditional.py
@@ -144,9 +144,9 @@ def generic_visit(self, node, inside_call=False, inside_yield=False):
inside_call = True
elif isinstance(node, ast.Yield):
inside_yield = True
- elif isinstance(node, ast.Str):
+ elif isinstance(node, ast.Constant) and isinstance(node.value, text_type):
if disable_lookups:
- if inside_call and node.s.startswith("__"):
+ if inside_call and node.value.startswith("__"):
# calling things with a dunder is generally bad at this point...
raise AnsibleError(
"Invalid access found in the conditional: '%s'" % conditional
@@ -154,7 +154,7 @@ def generic_visit(self, node, inside_call=False, inside_yield=False):
elif inside_yield:
# we're inside a yield, so recursively parse and traverse the AST
# of the result to catch forbidden syntax from executing
- parsed = ast.parse(node.s, mode='exec')
+ parsed = ast.parse(node.value, mode='exec')
cnv = CleansingNodeVisitor()
cnv.visit(parsed)
# iterate over all child nodes
diff --git a/test/lib/ansible_test/_util/controller/sanity/validate-modules/validate_modules/main.py b/test/lib/ansible_test/_util/controller/sanity/validate-modules/validate_modules/main.py
index fd5ea3ae788e17..2b92a56c2055dd 100644
--- a/test/lib/ansible_test/_util/controller/sanity/validate-modules/validate_modules/main.py
+++ b/test/lib/ansible_test/_util/controller/sanity/validate-modules/validate_modules/main.py
@@ -808,22 +808,22 @@ def _get_py_docs(self):
continue
if grandchild.id == 'DOCUMENTATION':
- docs['DOCUMENTATION']['value'] = child.value.s
+ docs['DOCUMENTATION']['value'] = child.value.value
docs['DOCUMENTATION']['lineno'] = child.lineno
docs['DOCUMENTATION']['end_lineno'] = (
- child.lineno + len(child.value.s.splitlines())
+ child.lineno + len(child.value.value.splitlines())
)
elif grandchild.id == 'EXAMPLES':
- docs['EXAMPLES']['value'] = child.value.s
+ docs['EXAMPLES']['value'] = child.value.value
docs['EXAMPLES']['lineno'] = child.lineno
docs['EXAMPLES']['end_lineno'] = (
- child.lineno + len(child.value.s.splitlines())
+ child.lineno + len(child.value.value.splitlines())
)
elif grandchild.id == 'RETURN':
- docs['RETURN']['value'] = child.value.s
+ docs['RETURN']['value'] = child.value.value
docs['RETURN']['lineno'] = child.lineno
docs['RETURN']['end_lineno'] = (
- child.lineno + len(child.value.s.splitlines())
+ child.lineno + len(child.value.value.splitlines())
)
return docs
diff --git a/test/lib/ansible_test/_util/controller/sanity/yamllint/yamllinter.py b/test/lib/ansible_test/_util/controller/sanity/yamllint/yamllinter.py
index d6de6117b2328a..ed1afcf3a5efc0 100644
--- a/test/lib/ansible_test/_util/controller/sanity/yamllint/yamllinter.py
+++ b/test/lib/ansible_test/_util/controller/sanity/yamllint/yamllinter.py
@@ -181,15 +181,15 @@ def check_assignment(statement, doc_types=None):
if doc_types and target.id not in doc_types:
continue
- fmt_match = fmt_re.match(statement.value.s.lstrip())
+ fmt_match = fmt_re.match(statement.value.value.lstrip())
fmt = 'yaml'
if fmt_match:
fmt = fmt_match.group(1)
docs[target.id] = dict(
- yaml=statement.value.s,
+ yaml=statement.value.value,
lineno=statement.lineno,
- end_lineno=statement.lineno + len(statement.value.s.splitlines()),
+ end_lineno=statement.lineno + len(statement.value.value.splitlines()),
fmt=fmt.lower(),
)

151
support-Python-3.12-in-ansible-test.patch Normal file
View File

@ -0,0 +1,151 @@
From c10e3d21fcb54dd0a9dfe44d97355195166510a9 Mon Sep 17 00:00:00 2001
From: s-hertel <19572925+s-hertel@users.noreply.github.com>
Date: Tue, 16 May 2023 13:48:12 -0400
Subject: [PATCH] add Python 3.12 support to ansible-test
skip docs build sanity test on Python 3.12 until aiohttp has a compatible version
skip Python 3.12 tests on windows/networking until the default container is
updated
add interpreter fallback?
---
lib/ansible/config/base.yml | 1 +
setup.cfg | 1 +
.../minimum-build-constraints.txt | 4 +++-
test/lib/ansible_test/_data/requirements/ansible-test.txt | 2 +-
test/lib/ansible_test/_internal/bootstrap.py | 4 ++++
test/lib/ansible_test/_internal/coverage_util.py | 2 +-
test/lib/ansible_test/_internal/python_requirements.py | 4 ++--
test/lib/ansible_test/_util/target/common/constants.py | 1 +
test/lib/ansible_test/_util/target/setup/bootstrap.sh | 2 +-
test/sanity/code-smell/docs-build.json | 1 +
10 files changed, 16 insertions(+), 6 deletions(-)
diff --git a/lib/ansible/config/base.yml b/lib/ansible/config/base.yml
index 206deb76d2e916..469fbc6846b6d9 100644
--- a/lib/ansible/config/base.yml
+++ b/lib/ansible/config/base.yml
@@ -1557,6 +1557,7 @@ _INTERPRETER_PYTHON_DISTRO_MAP:
INTERPRETER_PYTHON_FALLBACK:
name: Ordered list of Python interpreters to check for in discovery
default:
+ - python3.12
- python3.11
- python3.10
- python3.9
diff --git a/setup.cfg b/setup.cfg
index e020ee3b15caea..af79337f91e9de 100644
--- a/setup.cfg
+++ b/setup.cfg
@@ -30,6 +30,7 @@ classifiers =
Programming Language :: Python :: 3.9
Programming Language :: Python :: 3.10
Programming Language :: Python :: 3.11
+ Programming Language :: Python :: 3.12
Programming Language :: Python :: 3 :: Only
Topic :: System :: Installation/Setup
Topic :: System :: Systems Administration
diff --git a/test/integration/targets/canonical-pep517-self-packaging/minimum-build-constraints.txt b/test/integration/targets/canonical-pep517-self-packaging/minimum-build-constraints.txt
index 3ba47aeb4b69b4..765ca85c17d8d3 100644
--- a/test/integration/targets/canonical-pep517-self-packaging/minimum-build-constraints.txt
+++ b/test/integration/targets/canonical-pep517-self-packaging/minimum-build-constraints.txt
@@ -2,8 +2,10 @@
setuptools == 57.0.0; python_version == "3.9" or python_version == "3.10"
# Lowest supporting Python 3.11:
-setuptools == 60.0.0; python_version >= "3.11"
+setuptools == 60.0.0; python_version == "3.11"
+# Lowest supporting Python 3.12:
+setuptools == 66.1.0; python_version >= "3.12"
# An arbitrary old version that was released before Python 3.9.0:
wheel == 0.33.6
diff --git a/test/lib/ansible_test/_data/requirements/ansible-test.txt b/test/lib/ansible_test/_data/requirements/ansible-test.txt
index f7cb9c27780856..8b1772fb915bc3 100644
--- a/test/lib/ansible_test/_data/requirements/ansible-test.txt
+++ b/test/lib/ansible_test/_data/requirements/ansible-test.txt
@@ -1,4 +1,4 @@
# The test-constraints sanity test verifies this file, but changes must be made manually to keep it in up-to-date.
virtualenv == 16.7.12 ; python_version < '3'
-coverage == 6.5.0 ; python_version >= '3.7' and python_version <= '3.11'
+coverage == 6.5.0 ; python_version >= '3.7' and python_version <= '3.12'
coverage == 4.5.4 ; python_version >= '2.6' and python_version <= '3.6'
diff --git a/test/lib/ansible_test/_internal/bootstrap.py b/test/lib/ansible_test/_internal/bootstrap.py
index b0cfb601d94497..1bd357679bb72c 100644
--- a/test/lib/ansible_test/_internal/bootstrap.py
+++ b/test/lib/ansible_test/_internal/bootstrap.py
@@ -90,6 +90,10 @@ def get_variables(self) -> dict[str, t.Union[str, list[str]]]:
"""The variables to template in the bootstrapping script."""
variables = super().get_variables()
+ # remove once the default docker container is updated
+ if self.platform in ('ios', 'vyos', 'windows'):
+ variables['python_versions'] = [version for version in self.python_versions if version != '3.12']
+
variables.update(
platform=self.platform,
platform_version=self.platform_version,
diff --git a/test/lib/ansible_test/_internal/coverage_util.py b/test/lib/ansible_test/_internal/coverage_util.py
index 0af1cac4e97c63..f9276d323c50fc 100644
--- a/test/lib/ansible_test/_internal/coverage_util.py
+++ b/test/lib/ansible_test/_internal/coverage_util.py
@@ -69,7 +69,7 @@ class CoverageVersion:
COVERAGE_VERSIONS = (
# IMPORTANT: Keep this in sync with the ansible-test.txt requirements file.
- CoverageVersion('6.5.0', 7, (3, 7), (3, 11)),
+ CoverageVersion('6.5.0', 7, (3, 7), (3, 12)),
CoverageVersion('4.5.4', 0, (2, 6), (3, 6)),
)
"""
diff --git a/test/lib/ansible_test/_internal/python_requirements.py b/test/lib/ansible_test/_internal/python_requirements.py
index fc88b637c2b242..1a07334d8637c5 100644
--- a/test/lib/ansible_test/_internal/python_requirements.py
+++ b/test/lib/ansible_test/_internal/python_requirements.py
@@ -434,8 +434,8 @@ def get_venv_packages(python: PythonConfig) -> dict[str, str]:
# See: https://github.com/ansible/base-test-container/blob/main/files/installer.py
default_packages = dict(
- pip='21.3.1',
- setuptools='60.8.2',
+ pip='23.1.2',
+ setuptools='67.7.2',
wheel='0.37.1',
)
diff --git a/test/lib/ansible_test/_util/target/common/constants.py b/test/lib/ansible_test/_util/target/common/constants.py
index 9bddfaf439563e..f3c3857ef97dbf 100644
--- a/test/lib/ansible_test/_util/target/common/constants.py
+++ b/test/lib/ansible_test/_util/target/common/constants.py
@@ -17,4 +17,5 @@
'3.9',
'3.10',
'3.11',
+ '3.12',
)
diff --git a/test/lib/ansible_test/_util/target/setup/bootstrap.sh b/test/lib/ansible_test/_util/target/setup/bootstrap.sh
index ea17dad38751cc..367dcfcb4ce17e 100644
--- a/test/lib/ansible_test/_util/target/setup/bootstrap.sh
+++ b/test/lib/ansible_test/_util/target/setup/bootstrap.sh
@@ -53,7 +53,7 @@ install_pip() {
pip_bootstrap_url="https://ci-files.testing.ansible.com/ansible-test/get-pip-20.3.4.py"
;;
*)
- pip_bootstrap_url="https://ci-files.testing.ansible.com/ansible-test/get-pip-21.3.1.py"
+ pip_bootstrap_url="https://ci-files.testing.ansible.com/ansible-test/get-pip-23.1.2.py"
;;
esac
diff --git a/test/sanity/code-smell/docs-build.json b/test/sanity/code-smell/docs-build.json
index a43fa923b2b002..eedeca35dcba20 100644
--- a/test/sanity/code-smell/docs-build.json
+++ b/test/sanity/code-smell/docs-build.json
@@ -1,5 +1,6 @@
{
"disabled": true,
+ "maximum_python_version": "3.11",
"no_targets": true,
"output": "path-line-column-message"
}

154
urls-remove-deprecated-client-key-calls.patch Normal file
View File

@ -0,0 +1,154 @@
From 0df794e5a4fe4597ee65b0d492fbf0d0989d5ca0 Mon Sep 17 00:00:00 2001
From: Jordan Borean <jborean93@gmail.com>
Date: Thu, 18 May 2023 08:17:25 +1000
Subject: [PATCH] urls - remove deprecated client key calls (#80751)
---
.../fragments/urls-client-cert-py12.yml | 2 ++
lib/ansible/module_utils/urls.py | 28 +++++++++++--------
test/units/module_utils/urls/test_Request.py | 14 ++++------
3 files changed, 24 insertions(+), 20 deletions(-)
create mode 100644 changelogs/fragments/urls-client-cert-py12.yml
diff --git a/changelogs/fragments/urls-client-cert-py12.yml b/changelogs/fragments/urls-client-cert-py12.yml
new file mode 100644
index 00000000000000..aab129ed96e94b
--- /dev/null
+++ b/changelogs/fragments/urls-client-cert-py12.yml
@@ -0,0 +1,2 @@
+bugfixes:
+- urls.py - fixed cert_file and key_file parameters when running on Python 3.12 - https://github.com/ansible/ansible/issues/80490
diff --git a/lib/ansible/module_utils/urls.py b/lib/ansible/module_utils/urls.py
index 0e5fbb74c4fae2..0197d86e1033b2 100644
--- a/lib/ansible/module_utils/urls.py
+++ b/lib/ansible/module_utils/urls.py
@@ -535,15 +535,18 @@ def __init__(self, message, import_traceback, module=None):
UnixHTTPSConnection = None
if hasattr(httplib, 'HTTPSConnection') and hasattr(urllib_request, 'HTTPSHandler'):
class CustomHTTPSConnection(httplib.HTTPSConnection): # type: ignore[no-redef]
- def __init__(self, *args, **kwargs):
+ def __init__(self, client_cert=None, client_key=None, *args, **kwargs):
httplib.HTTPSConnection.__init__(self, *args, **kwargs)
self.context = None
if HAS_SSLCONTEXT:
self.context = self._context
elif HAS_URLLIB3_PYOPENSSLCONTEXT:
self.context = self._context = PyOpenSSLContext(PROTOCOL)
- if self.context and self.cert_file:
- self.context.load_cert_chain(self.cert_file, self.key_file)
+
+ self._client_cert = client_cert
+ self._client_key = client_key
+ if self.context and self._client_cert:
+ self.context.load_cert_chain(self._client_cert, self._client_key)
def connect(self):
"Connect to a host on a given (SSL) port."
@@ -564,10 +567,10 @@ def connect(self):
if HAS_SSLCONTEXT or HAS_URLLIB3_PYOPENSSLCONTEXT:
self.sock = self.context.wrap_socket(sock, server_hostname=server_hostname)
elif HAS_URLLIB3_SSL_WRAP_SOCKET:
- self.sock = ssl_wrap_socket(sock, keyfile=self.key_file, cert_reqs=ssl.CERT_NONE, # pylint: disable=used-before-assignment
- certfile=self.cert_file, ssl_version=PROTOCOL, server_hostname=server_hostname)
+ self.sock = ssl_wrap_socket(sock, keyfile=self._client_key, cert_reqs=ssl.CERT_NONE, # pylint: disable=used-before-assignment
+ certfile=self._client_cert, ssl_version=PROTOCOL, server_hostname=server_hostname)
else:
- self.sock = ssl.wrap_socket(sock, keyfile=self.key_file, certfile=self.cert_file, ssl_version=PROTOCOL)
+ self.sock = ssl.wrap_socket(sock, keyfile=self._client_key, certfile=self._client_cert, ssl_version=PROTOCOL)
class CustomHTTPSHandler(urllib_request.HTTPSHandler): # type: ignore[no-redef]
@@ -602,10 +605,6 @@ def https_open(self, req):
return self.do_open(self._build_https_connection, req)
def _build_https_connection(self, host, **kwargs):
- kwargs.update({
- 'cert_file': self.client_cert,
- 'key_file': self.client_key,
- })
try:
kwargs['context'] = self._context
except AttributeError:
@@ -613,7 +612,7 @@ def _build_https_connection(self, host, **kwargs):
if self._unix_socket:
return UnixHTTPSConnection(self._unix_socket)(host, **kwargs)
if not HAS_SSLCONTEXT:
- return CustomHTTPSConnection(host, **kwargs)
+ return CustomHTTPSConnection(host, client_cert=self.client_cert, client_key=self.client_key, **kwargs)
return httplib.HTTPSConnection(host, **kwargs)
@contextmanager
@@ -979,7 +978,7 @@ def atexit_remove_file(filename):
pass
-def make_context(cafile=None, cadata=None, ciphers=None, validate_certs=True):
+def make_context(cafile=None, cadata=None, ciphers=None, validate_certs=True, client_cert=None, client_key=None):
if ciphers is None:
ciphers = []
@@ -1006,6 +1005,9 @@ def make_context(cafile=None, cadata=None, ciphers=None, validate_certs=True):
if ciphers:
context.set_ciphers(':'.join(map(to_native, ciphers)))
+ if client_cert:
+ context.load_cert_chain(client_cert, keyfile=client_key)
+
return context
@@ -1514,6 +1516,8 @@ def open(self, method, url, data=None, headers=None, use_proxy=None,
cadata=cadata,
ciphers=ciphers,
validate_certs=validate_certs,
+ client_cert=client_cert,
+ client_key=client_key,
)
handlers.append(HTTPSClientAuthHandler(client_cert=client_cert,
client_key=client_key,
diff --git a/test/units/module_utils/urls/test_Request.py b/test/units/module_utils/urls/test_Request.py
index d2c4ea38012a49..a8bc3a0b6bde3b 100644
--- a/test/units/module_utils/urls/test_Request.py
+++ b/test/units/module_utils/urls/test_Request.py
@@ -33,6 +33,7 @@ def install_opener_mock(mocker):
def test_Request_fallback(urlopen_mock, install_opener_mock, mocker):
here = os.path.dirname(__file__)
pem = os.path.join(here, 'fixtures/client.pem')
+ client_key = os.path.join(here, 'fixtures/client.key')
cookies = cookiejar.CookieJar()
request = Request(
@@ -46,8 +47,8 @@ def test_Request_fallback(urlopen_mock, install_opener_mock, mocker):
http_agent='ansible-tests',
force_basic_auth=True,
follow_redirects='all',
- client_cert='/tmp/client.pem',
- client_key='/tmp/client.key',
+ client_cert=pem,
+ client_key=client_key,
cookies=cookies,
unix_socket='/foo/bar/baz.sock',
ca_path=pem,
@@ -68,8 +69,8 @@ def test_Request_fallback(urlopen_mock, install_opener_mock, mocker):
call(None, 'ansible-tests'), # http_agent
call(None, True), # force_basic_auth
call(None, 'all'), # follow_redirects
- call(None, '/tmp/client.pem'), # client_cert
- call(None, '/tmp/client.key'), # client_key
+ call(None, pem), # client_cert
+ call(None, client_key), # client_key
call(None, cookies), # cookies
call(None, '/foo/bar/baz.sock'), # unix_socket
call(None, pem), # ca_path
@@ -358,10 +359,7 @@ def test_Request_open_client_cert(urlopen_mock, install_opener_mock):
assert ssl_handler.client_cert == client_cert
assert ssl_handler.client_key == client_key
- https_connection = ssl_handler._build_https_connection('ansible.com')
-
- assert https_connection.key_file == client_key
- assert https_connection.cert_file == client_cert
+ ssl_handler._build_https_connection('ansible.com')
def test_Request_open_cookies(urlopen_mock, install_opener_mock):