Compare commits

...

No commits in common. "c8-beta" and "c8s" have entirely different histories.
c8-beta ... c8s

30 changed files with 2011 additions and 1110 deletions

1
.fmf/version Normal file
View File

@ -0,0 +1 @@
1

15
.gitignore vendored
View File

@ -1 +1,14 @@
SOURCES/Python-3.11.7.tar.xz
/Python-3.11.1.tar.xz
/Python-3.11.1.tar.xz.asc
/Python-3.11.2.tar.xz
/Python-3.11.2.tar.xz.asc
/Python-3.11.4.tar.xz
/Python-3.11.4.tar.xz.asc
/Python-3.11.5.tar.xz
/Python-3.11.5.tar.xz.asc
/Python-3.11.7.tar.xz
/Python-3.11.7.tar.xz.asc
/Python-3.11.9.tar.xz
/Python-3.11.9.tar.xz.asc
/Python-3.11.10.tar.xz
/Python-3.11.10.tar.xz.asc

View File

@ -1 +0,0 @@
f2534d591121f3845388fbdd6a121b96dfe305a6 SOURCES/Python-3.11.7.tar.xz

View File

@ -1,4 +1,4 @@
From ecc5137120f471c22ff6dcb1bd128561c31e023c Mon Sep 17 00:00:00 2001
From 4345f8ea8a56a58ef8a48439c0e201702d1012a2 Mon Sep 17 00:00:00 2001
From: Charalampos Stratakis <cstratak@redhat.com>
Date: Thu, 12 Dec 2019 16:58:31 +0100
Subject: [PATCH 1/7] Expose blake2b and blake2s hashes from OpenSSL
@ -205,10 +205,10 @@ index 5d84f4a..011026a 100644
-/*[clinic end generated code: output=69f2374071bff707 input=a9049054013a1b77]*/
+/*[clinic end generated code: output=c6a9af5563972eda input=a9049054013a1b77]*/
--
2.43.0
2.45.0
From 0198d467525e79cb4be4418708719af3eaee7a40 Mon Sep 17 00:00:00 2001
From 1f79be1a11ad6811913c239da980c5bab0f1c538 Mon Sep 17 00:00:00 2001
From: Petr Viktorin <pviktori@redhat.com>
Date: Thu, 1 Aug 2019 17:57:05 +0200
Subject: [PATCH 2/7] Use a stronger hash in multiprocessing handshake
@ -220,7 +220,7 @@ https://bugs.python.org/issue17258
1 file changed, 6 insertions(+), 2 deletions(-)
diff --git a/Lib/multiprocessing/connection.py b/Lib/multiprocessing/connection.py
index 8b81f99..69c0b7e 100644
index 59c61d2..7fc594e 100644
--- a/Lib/multiprocessing/connection.py
+++ b/Lib/multiprocessing/connection.py
@@ -43,6 +43,10 @@ BUFSIZE = 8192
@ -234,7 +234,7 @@ index 8b81f99..69c0b7e 100644
_mmap_counter = itertools.count()
default_family = 'AF_INET'
@@ -752,7 +756,7 @@ def deliver_challenge(connection, authkey):
@@ -753,7 +757,7 @@ def deliver_challenge(connection, authkey):
"Authkey must be bytes, not {0!s}".format(type(authkey)))
message = os.urandom(MESSAGE_LENGTH)
connection.send_bytes(CHALLENGE + message)
@ -243,7 +243,7 @@ index 8b81f99..69c0b7e 100644
response = connection.recv_bytes(256) # reject large message
if response == digest:
connection.send_bytes(WELCOME)
@@ -768,7 +772,7 @@ def answer_challenge(connection, authkey):
@@ -769,7 +773,7 @@ def answer_challenge(connection, authkey):
message = connection.recv_bytes(256) # reject large message
assert message[:len(CHALLENGE)] == CHALLENGE, 'message = %r' % message
message = message[len(CHALLENGE):]
@ -253,10 +253,10 @@ index 8b81f99..69c0b7e 100644
response = connection.recv_bytes(256) # reject large message
if response != WELCOME:
--
2.43.0
2.45.0
From a7822e2e1f21529e9730885bd8c9c6ab7c704d5b Mon Sep 17 00:00:00 2001
From e069ed2dcd0edf0de489eb387267fb35a92ed506 Mon Sep 17 00:00:00 2001
From: Petr Viktorin <pviktori@redhat.com>
Date: Thu, 25 Jul 2019 17:19:06 +0200
Subject: [PATCH 3/7] Disable Python's hash implementations in FIPS mode,
@ -446,10 +446,10 @@ index 56ae7a5..45fb403 100644
+ if (_Py_hashlib_fips_error(exc, name)) return NULL; \
+} while (0)
diff --git a/configure.ac b/configure.ac
index 52d5c1f..56aff78 100644
index 7b4000f..8e2f0ad 100644
--- a/configure.ac
+++ b/configure.ac
@@ -7069,7 +7069,8 @@ PY_STDLIB_MOD([_sha512], [test "$with_builtin_sha512" = yes])
@@ -7070,7 +7070,8 @@ PY_STDLIB_MOD([_sha512], [test "$with_builtin_sha512" = yes])
PY_STDLIB_MOD([_sha3], [test "$with_builtin_sha3" = yes])
PY_STDLIB_MOD([_blake2],
[test "$with_builtin_blake2" = yes], [],
@ -460,10 +460,10 @@ index 52d5c1f..56aff78 100644
PY_STDLIB_MOD([_crypt],
[], [test "$ac_cv_crypt_crypt" = yes],
--
2.43.0
2.45.0
From e9ce6d33544559172dbebbe0c0dfba2757c62331 Mon Sep 17 00:00:00 2001
From 2e0c5086f4a52803595e19795111278c3c80ee2f Mon Sep 17 00:00:00 2001
From: Charalampos Stratakis <cstratak@redhat.com>
Date: Fri, 29 Jan 2021 14:16:21 +0100
Subject: [PATCH 4/7] Use python's fall back crypto implementations only if we
@ -623,10 +623,10 @@ index 01d12f5..a7cdb07 100644
def test_pbkdf2_hmac_py(self):
with warnings_helper.check_warnings():
--
2.43.0
2.45.0
From 641c617775b6973ed84711a2602ba190fe064474 Mon Sep 17 00:00:00 2001
From 0e1d2a67ef66cccc9afa4a515dc34ce587946f22 Mon Sep 17 00:00:00 2001
From: Charalampos Stratakis <cstratak@redhat.com>
Date: Wed, 31 Jul 2019 15:43:43 +0200
Subject: [PATCH 5/7] Test equivalence of hashes for the various digests with
@ -783,21 +783,21 @@ index a7cdb07..c071f28 100644
class KDFTests(unittest.TestCase):
--
2.43.0
2.45.0
From a706c8342f0f9307d44c43c203702e1476fe73b4 Mon Sep 17 00:00:00 2001
From f1c9ecbb2e2f08d792fb0557058824eed23abb7b Mon Sep 17 00:00:00 2001
From: Petr Viktorin <pviktori@redhat.com>
Date: Mon, 26 Aug 2019 19:39:48 +0200
Subject: [PATCH 6/7] Guard against Python HMAC in FIPS mode
---
Lib/hmac.py | 13 +++++++++----
Lib/hmac.py | 12 +++++++++---
Lib/test/test_hmac.py | 10 ++++++++++
2 files changed, 19 insertions(+), 4 deletions(-)
2 files changed, 19 insertions(+), 3 deletions(-)
diff --git a/Lib/hmac.py b/Lib/hmac.py
index 8b4f920..20ef96c 100644
index 8b4eb2f..8930bda 100644
--- a/Lib/hmac.py
+++ b/Lib/hmac.py
@@ -16,8 +16,9 @@ else:
@ -812,16 +812,9 @@ index 8b4f920..20ef96c 100644
# The size of the digests returned by HMAC depends on the underlying
# hashing module used. Use digest_size from the instance of HMAC instead.
@@ -48,17 +49,18 @@ class HMAC:
msg argument. Passing it as a keyword argument is
recommended, though not required for legacy API reasons.
"""
-
if not isinstance(key, (bytes, bytearray)):
raise TypeError("key: expected bytes or bytearray, but got %r" % type(key).__name__)
@@ -55,10 +56,12 @@ class HMAC:
if not digestmod:
raise TypeError("Missing required parameter 'digestmod'.")
raise TypeError("Missing required argument 'digestmod'.")
- if _hashopenssl and isinstance(digestmod, (str, _functype)):
+ if _hashopenssl.get_fips_mode() or (_hashopenssl and isinstance(digestmod, (str, _functype))):
@ -833,7 +826,7 @@ index 8b4f920..20ef96c 100644
self._init_old(key, msg, digestmod)
else:
self._init_old(key, msg, digestmod)
@@ -69,6 +71,9 @@ class HMAC:
@@ -69,6 +72,9 @@ class HMAC:
self.block_size = self._hmac.block_size
def _init_old(self, key, msg, digestmod):
@ -844,7 +837,7 @@ index 8b4f920..20ef96c 100644
digest_cons = digestmod
elif isinstance(digestmod, str):
diff --git a/Lib/test/test_hmac.py b/Lib/test/test_hmac.py
index a39a2c4..0742a1c 100644
index 1502fba..e40ca4b 100644
--- a/Lib/test/test_hmac.py
+++ b/Lib/test/test_hmac.py
@@ -5,6 +5,7 @@ import hashlib
@ -883,7 +876,7 @@ index a39a2c4..0742a1c 100644
@unittest.skipUnless(sha256_module is not None, 'need _sha256')
def test_with_sha256_module(self):
h = hmac.HMAC(b"key", b"hash this!", digestmod=sha256_module.sha256)
@@ -481,6 +489,7 @@ class SanityTestCase(unittest.TestCase):
@@ -489,6 +497,7 @@ class UpdateTestCase(unittest.TestCase):
class CopyTestCase(unittest.TestCase):
@ -891,7 +884,7 @@ index a39a2c4..0742a1c 100644
@hashlib_helper.requires_hashdigest('sha256')
def test_attributes_old(self):
# Testing if attributes are of same type.
@@ -492,6 +501,7 @@ class CopyTestCase(unittest.TestCase):
@@ -500,6 +509,7 @@ class CopyTestCase(unittest.TestCase):
self.assertEqual(type(h1._outer), type(h2._outer),
"Types of outer don't match.")
@ -900,290 +893,43 @@ index a39a2c4..0742a1c 100644
def test_realcopy_old(self):
# Testing if the copy method created a real copy.
--
2.43.0
2.45.0
From 03f1dedfe5d29af20fb3686d76b045384d41d8dd Mon Sep 17 00:00:00 2001
From: Petr Viktorin <encukou@gmail.com>
Date: Wed, 25 Aug 2021 16:44:43 +0200
Subject: [PATCH 7/7] Disable hash-based PYCs in FIPS mode
From a0c3f9ac5a4e60ab22418a3196ae46ba34e9477b Mon Sep 17 00:00:00 2001
From: Nikita Sobolev <mail@sobolevn.me>
Date: Thu, 24 Nov 2022 01:47:31 +0300
Subject: [PATCH 7/7] closes gh-99508: fix `TypeError` in
`Lib/importlib/_bootstrap_external.py` (GH-99635)
If FIPS mode is on, we can't use siphash-based HMAC
(_Py_KeyedHash), so:
- Unchecked hash PYCs can be imported, but not created
- Checked hash PYCs can not be imported nor created
- The default mode is timestamp-based PYCs, even if
SOURCE_DATE_EPOCH is set.
If FIPS mode is off, there are no changes in behavior.
Resolves: https://bugzilla.redhat.com/show_bug.cgi?id=1835169
---
Lib/py_compile.py | 2 ++
Lib/test/support/__init__.py | 14 +++++++++++++
Lib/test/test_cmd_line_script.py | 2 ++
Lib/test/test_compileall.py | 11 +++++++++-
Lib/test/test_imp.py | 2 ++
.../test_importlib/source/test_file_loader.py | 6 ++++++
Lib/test/test_py_compile.py | 11 ++++++++--
Lib/test/test_zipimport.py | 2 ++
Python/import.c | 20 +++++++++++++++++++
9 files changed, 67 insertions(+), 3 deletions(-)
Lib/importlib/_bootstrap_external.py | 3 ++-
.../next/Library/2022-11-21-10-45-54.gh-issue-99508.QqVbby.rst | 2 ++
2 files changed, 4 insertions(+), 1 deletion(-)
create mode 100644 Misc/NEWS.d/next/Library/2022-11-21-10-45-54.gh-issue-99508.QqVbby.rst
diff --git a/Lib/py_compile.py b/Lib/py_compile.py
index db52725..5fca65e 100644
--- a/Lib/py_compile.py
+++ b/Lib/py_compile.py
@@ -70,7 +70,9 @@ class PycInvalidationMode(enum.Enum):
def _get_default_invalidation_mode():
+ import _hashlib
if (os.environ.get('SOURCE_DATE_EPOCH') and not
+ _hashlib.get_fips_mode() and not
os.environ.get('RPM_BUILD_ROOT')):
return PycInvalidationMode.CHECKED_HASH
else:
diff --git a/Lib/test/support/__init__.py b/Lib/test/support/__init__.py
index dc7a6e6..646b328 100644
--- a/Lib/test/support/__init__.py
+++ b/Lib/test/support/__init__.py
@@ -2203,6 +2203,20 @@ def sleeping_retry(timeout, err_msg=None, /,
delay = min(delay * 2, max_delay)
+def fails_in_fips_mode(expected_error):
+ import _hashlib
+ if _hashlib.get_fips_mode():
+ def _decorator(func):
+ def _wrapper(self, *args, **kwargs):
+ with self.assertRaises(expected_error):
+ func(self, *args, **kwargs)
+ return _wrapper
+ else:
+ def _decorator(func):
+ return func
+ return _decorator
+
+
@contextlib.contextmanager
def adjust_int_max_str_digits(max_digits):
"""Temporarily change the integer string conversion length limit."""
diff --git a/Lib/test/test_cmd_line_script.py b/Lib/test/test_cmd_line_script.py
index 7fcd563..476b557 100644
--- a/Lib/test/test_cmd_line_script.py
+++ b/Lib/test/test_cmd_line_script.py
@@ -286,6 +286,7 @@ class CmdLineTest(unittest.TestCase):
self._check_script(zip_name, run_name, zip_name, zip_name, '',
zipimport.zipimporter)
+ @support.fails_in_fips_mode(ImportError)
def test_zipfile_compiled_checked_hash(self):
with os_helper.temp_dir() as script_dir:
script_name = _make_test_script(script_dir, '__main__')
@@ -296,6 +297,7 @@ class CmdLineTest(unittest.TestCase):
self._check_script(zip_name, run_name, zip_name, zip_name, '',
zipimport.zipimporter)
+ @support.fails_in_fips_mode(ImportError)
def test_zipfile_compiled_unchecked_hash(self):
with os_helper.temp_dir() as script_dir:
script_name = _make_test_script(script_dir, '__main__')
diff --git a/Lib/test/test_compileall.py b/Lib/test/test_compileall.py
index 9cd92ad..4ec29a1 100644
--- a/Lib/test/test_compileall.py
+++ b/Lib/test/test_compileall.py
@@ -806,14 +806,23 @@ class CommandLineTestsBase:
out = self.assertRunOK('badfilename')
self.assertRegex(out, b"Can't list 'badfilename'")
- def test_pyc_invalidation_mode(self):
+ @support.fails_in_fips_mode(AssertionError)
+ def test_pyc_invalidation_mode_checked(self):
script_helper.make_script(self.pkgdir, 'f1', '')
pyc = importlib.util.cache_from_source(
os.path.join(self.pkgdir, 'f1.py'))
+
self.assertRunOK('--invalidation-mode=checked-hash', self.pkgdir)
with open(pyc, 'rb') as fp:
data = fp.read()
self.assertEqual(int.from_bytes(data[4:8], 'little'), 0b11)
+
+ @support.fails_in_fips_mode(AssertionError)
+ def test_pyc_invalidation_mode_unchecked(self):
+ script_helper.make_script(self.pkgdir, 'f1', '')
+ pyc = importlib.util.cache_from_source(
+ os.path.join(self.pkgdir, 'f1.py'))
+
self.assertRunOK('--invalidation-mode=unchecked-hash', self.pkgdir)
with open(pyc, 'rb') as fp:
data = fp.read()
diff --git a/Lib/test/test_imp.py b/Lib/test/test_imp.py
index 4062afd..6bc276d 100644
--- a/Lib/test/test_imp.py
+++ b/Lib/test/test_imp.py
@@ -352,6 +352,7 @@ class ImportTests(unittest.TestCase):
import _frozen_importlib
self.assertEqual(_frozen_importlib.__spec__.origin, "frozen")
+ @support.fails_in_fips_mode(ImportError)
def test_source_hash(self):
self.assertEqual(_imp.source_hash(42, b'hi'), b'\xfb\xd9G\x05\xaf$\x9b~')
self.assertEqual(_imp.source_hash(43, b'hi'), b'\xd0/\x87C\xccC\xff\xe2')
@@ -371,6 +372,7 @@ class ImportTests(unittest.TestCase):
res = script_helper.assert_python_ok(*args)
self.assertEqual(res.out.strip().decode('utf-8'), expected)
+ @support.fails_in_fips_mode(ImportError)
def test_find_and_load_checked_pyc(self):
# issue 34056
with os_helper.temp_cwd():
diff --git a/Lib/test/test_importlib/source/test_file_loader.py b/Lib/test/test_importlib/source/test_file_loader.py
index 378dcbe..7b223a1 100644
--- a/Lib/test/test_importlib/source/test_file_loader.py
+++ b/Lib/test/test_importlib/source/test_file_loader.py
@@ -16,6 +16,7 @@ import types
import unittest
import warnings
+from test import support
from test.support.import_helper import make_legacy_pyc, unload
from test.test_py_compile import without_source_date_epoch
@@ -238,6 +239,7 @@ class SimpleTest(abc.LoaderTests):
loader.load_module('bad name')
@util.writes_bytecode_files
+ @support.fails_in_fips_mode(ImportError)
def test_checked_hash_based_pyc(self):
with util.create_modules('_temp') as mapping:
source = mapping['_temp']
@@ -269,6 +271,7 @@ class SimpleTest(abc.LoaderTests):
)
@util.writes_bytecode_files
+ @support.fails_in_fips_mode(ImportError)
def test_overridden_checked_hash_based_pyc(self):
with util.create_modules('_temp') as mapping, \
unittest.mock.patch('_imp.check_hash_based_pycs', 'never'):
@@ -294,6 +297,7 @@ class SimpleTest(abc.LoaderTests):
self.assertEqual(mod.state, 'old')
@util.writes_bytecode_files
+ @support.fails_in_fips_mode(ImportError)
def test_unchecked_hash_based_pyc(self):
with util.create_modules('_temp') as mapping:
source = mapping['_temp']
@@ -324,6 +328,7 @@ class SimpleTest(abc.LoaderTests):
)
@util.writes_bytecode_files
+ @support.fails_in_fips_mode(ImportError)
def test_overridden_unchecked_hash_based_pyc(self):
with util.create_modules('_temp') as mapping, \
unittest.mock.patch('_imp.check_hash_based_pycs', 'always'):
@@ -433,6 +438,7 @@ class BadBytecodeTest:
del_source=del_source)
test('_temp', mapping, bc_path)
+ @support.fails_in_fips_mode(ImportError)
def _test_partial_hash(self, test, *, del_source=False):
with util.create_modules('_temp') as mapping:
bc_path = self.manipulate_bytecode(
diff --git a/Lib/test/test_py_compile.py b/Lib/test/test_py_compile.py
index 9b420d2..dd6460a 100644
--- a/Lib/test/test_py_compile.py
+++ b/Lib/test/test_py_compile.py
@@ -143,13 +143,16 @@ class PyCompileTestsBase:
importlib.util.cache_from_source(bad_coding)))
def test_source_date_epoch(self):
+ import _hashlib
py_compile.compile(self.source_path, self.pyc_path)
self.assertTrue(os.path.exists(self.pyc_path))
self.assertFalse(os.path.exists(self.cache_path))
with open(self.pyc_path, 'rb') as fp:
flags = importlib._bootstrap_external._classify_pyc(
fp.read(), 'test', {})
- if os.environ.get('SOURCE_DATE_EPOCH'):
+ if _hashlib.get_fips_mode():
+ expected_flags = 0b00
+ elif os.environ.get('SOURCE_DATE_EPOCH'):
expected_flags = 0b11
else:
expected_flags = 0b00
@@ -180,7 +183,8 @@ class PyCompileTestsBase:
# Specifying optimized bytecode should lead to a path reflecting that.
self.assertIn('opt-2', py_compile.compile(self.source_path, optimize=2))
- def test_invalidation_mode(self):
+ @support.fails_in_fips_mode(ImportError)
+ def test_invalidation_mode_checked(self):
py_compile.compile(
self.source_path,
invalidation_mode=py_compile.PycInvalidationMode.CHECKED_HASH,
@@ -189,6 +193,9 @@ class PyCompileTestsBase:
flags = importlib._bootstrap_external._classify_pyc(
fp.read(), 'test', {})
self.assertEqual(flags, 0b11)
+
+ @support.fails_in_fips_mode(ImportError)
+ def test_invalidation_mode_unchecked(self):
py_compile.compile(
self.source_path,
invalidation_mode=py_compile.PycInvalidationMode.UNCHECKED_HASH,
diff --git a/Lib/test/test_zipimport.py b/Lib/test/test_zipimport.py
index 59a5200..81fadb3 100644
--- a/Lib/test/test_zipimport.py
+++ b/Lib/test/test_zipimport.py
@@ -190,6 +190,7 @@ class UncompressedZipImportTestCase(ImportHooksBaseTestCase):
TESTMOD + pyc_ext: (NOW, test_pyc)}
self.doTest(pyc_ext, files, TESTMOD)
+ @support.fails_in_fips_mode(ImportError)
def testUncheckedHashBasedPyc(self):
source = b"state = 'old'"
source_hash = importlib.util.source_hash(source)
@@ -204,6 +205,7 @@ class UncompressedZipImportTestCase(ImportHooksBaseTestCase):
self.assertEqual(mod.state, 'old')
self.doTest(None, files, TESTMOD, call=check)
+ @support.fails_in_fips_mode(ImportError)
@unittest.mock.patch('_imp.check_hash_based_pycs', 'always')
def test_checked_hash_based_change_pyc(self):
source = b"state = 'old'"
diff --git a/Python/import.c b/Python/import.c
index 39144d3..b439059 100644
--- a/Python/import.c
+++ b/Python/import.c
@@ -2449,6 +2449,26 @@ static PyObject *
_imp_source_hash_impl(PyObject *module, long key, Py_buffer *source)
/*[clinic end generated code: output=edb292448cf399ea input=9aaad1e590089789]*/
{
+ PyObject *_hashlib = PyImport_ImportModule("_hashlib");
+ if (_hashlib == NULL) {
+ return NULL;
+ }
+ PyObject *fips_mode_obj = PyObject_CallMethod(_hashlib, "get_fips_mode", NULL);
+ Py_DECREF(_hashlib);
+ if (fips_mode_obj == NULL) {
+ return NULL;
+ }
+ int fips_mode = PyObject_IsTrue(fips_mode_obj);
+ Py_DECREF(fips_mode_obj);
+ if (fips_mode < 0) {
+ return NULL;
+ }
+ if (fips_mode) {
+ PyErr_SetString(
+ PyExc_ImportError,
+ "hash-based PYC validation (siphash24) not available in FIPS mode");
+ return NULL;
+ };
union {
uint64_t x;
char data[sizeof(uint64_t)];
diff --git a/Lib/importlib/_bootstrap_external.py b/Lib/importlib/_bootstrap_external.py
index e53f6ac..bdc491e 100644
--- a/Lib/importlib/_bootstrap_external.py
+++ b/Lib/importlib/_bootstrap_external.py
@@ -1077,7 +1077,8 @@ class SourceLoader(_LoaderBasics):
source_mtime is not None):
if hash_based:
if source_hash is None:
- source_hash = _imp.source_hash(source_bytes)
+ source_hash = _imp.source_hash(_RAW_MAGIC_NUMBER,
+ source_bytes)
data = _code_to_hash_pyc(code_object, source_hash, check_source)
else:
data = _code_to_timestamp_pyc(code_object, source_mtime,
diff --git a/Misc/NEWS.d/next/Library/2022-11-21-10-45-54.gh-issue-99508.QqVbby.rst b/Misc/NEWS.d/next/Library/2022-11-21-10-45-54.gh-issue-99508.QqVbby.rst
new file mode 100644
index 0000000..82720d1
--- /dev/null
+++ b/Misc/NEWS.d/next/Library/2022-11-21-10-45-54.gh-issue-99508.QqVbby.rst
@@ -0,0 +1,2 @@
+Fix ``TypeError`` in ``Lib/importlib/_bootstrap_external.py`` while calling
+``_imp.source_hash()``.
--
2.43.0
2.45.0

View File

@ -1,4 +1,4 @@
From 8b70605b594b3831331a9340ba764ff751871612 Mon Sep 17 00:00:00 2001
From 0181d677dd7fd11bc19a211b3eb735ac3ad3d7fb Mon Sep 17 00:00:00 2001
From: Petr Viktorin <encukou@gmail.com>
Date: Mon, 6 Mar 2023 17:24:24 +0100
Subject: [PATCH] CVE-2007-4559, PEP-706: Add filters for tarfile extraction
@ -9,11 +9,11 @@ variable and config file.
---
Lib/tarfile.py | 42 +++++++++++++
Lib/test/test_shutil.py | 3 +-
Lib/test/test_tarfile.py | 128 ++++++++++++++++++++++++++++++++++++++-
3 files changed, 169 insertions(+), 4 deletions(-)
Lib/test/test_tarfile.py | 127 ++++++++++++++++++++++++++++++++++++++-
3 files changed, 168 insertions(+), 4 deletions(-)
diff --git a/Lib/tarfile.py b/Lib/tarfile.py
index 130b5e0..3b7d8d5 100755
index 612217b..dc59fc6 100755
--- a/Lib/tarfile.py
+++ b/Lib/tarfile.py
@@ -72,6 +72,13 @@ __all__ = ["TarFile", "TarInfo", "is_tarfile", "TarError", "ReadError",
@ -30,7 +30,7 @@ index 130b5e0..3b7d8d5 100755
#---------------------------------------------------------
# tar constants
@@ -2211,6 +2218,41 @@ class TarFile(object):
@@ -2219,6 +2226,41 @@ class TarFile(object):
if filter is None:
filter = self.extraction_filter
if filter is None:
@ -73,10 +73,10 @@ index 130b5e0..3b7d8d5 100755
if isinstance(filter, str):
raise TypeError(
diff --git a/Lib/test/test_shutil.py b/Lib/test/test_shutil.py
index 9bf4145..f247b82 100644
index 6728d30..2338b63 100644
--- a/Lib/test/test_shutil.py
+++ b/Lib/test/test_shutil.py
@@ -1665,7 +1665,8 @@ class TestArchives(BaseTest, unittest.TestCase):
@@ -1774,7 +1774,8 @@ class TestArchives(BaseTest, unittest.TestCase):
def check_unpack_tarball(self, format):
self.check_unpack_archive(format, filter='fully_trusted')
self.check_unpack_archive(format, filter='data')
@ -87,10 +87,10 @@ index 9bf4145..f247b82 100644
def test_unpack_archive_tar(self):
diff --git a/Lib/test/test_tarfile.py b/Lib/test/test_tarfile.py
index cdea033..4724285 100644
index 389da7b..5a43f9d 100644
--- a/Lib/test/test_tarfile.py
+++ b/Lib/test/test_tarfile.py
@@ -2,7 +2,7 @@ import sys
@@ -3,7 +3,7 @@ import sys
import os
import io
from hashlib import sha256
@ -99,7 +99,7 @@ index cdea033..4724285 100644
from random import Random
import pathlib
import shutil
@@ -2999,7 +2999,11 @@ class NoneInfoExtractTests(ReadTest):
@@ -3049,7 +3049,11 @@ class NoneInfoExtractTests(ReadTest):
tar = tarfile.open(tarname, mode='r', encoding="iso8859-1")
cls.control_dir = pathlib.Path(TEMPDIR) / "extractall_ctrl"
tar.errorlevel = 0
@ -112,7 +112,7 @@ index cdea033..4724285 100644
tar.close()
cls.control_paths = set(
p.relative_to(cls.control_dir)
@@ -3674,7 +3678,8 @@ class TestExtractionFilters(unittest.TestCase):
@@ -3868,7 +3872,8 @@ class TestExtractionFilters(unittest.TestCase):
"""Ensure the default filter does not warn (like in 3.12)"""
with ArchiveMaker() as arc:
arc.add('foo')
@ -122,10 +122,10 @@ index cdea033..4724285 100644
with self.check_context(arc.open(), None):
self.expect_file('foo')
@@ -3844,6 +3849,123 @@ class TestExtractionFilters(unittest.TestCase):
@@ -4037,6 +4042,122 @@ class TestExtractionFilters(unittest.TestCase):
with self.check_context(arc.open(errorlevel='boo!'), filtererror_filter):
self.expect_exception(TypeError) # errorlevel is not int
+ @contextmanager
+ def rh_config_context(self, config_lines=None):
+ """Set up for testing various ways of overriding the default filter
@ -242,10 +242,9 @@ index cdea033..4724285 100644
+ ):
+ self.check_trusted_default(tar, tempdir)
+
+
def setUpModule():
os_helper.unlink(TEMPDIR)
os.makedirs(TEMPDIR)
--
2.41.0
class OverwriteTests(archiver_tests.OverwriteTests, unittest.TestCase):
testdir = os.path.join(TEMPDIR, "testoverwrite")
--
2.44.0

View File

@ -0,0 +1,250 @@
From 3f01ced0b5051798516fc65f5fac10ffd15dbce6 Mon Sep 17 00:00:00 2001
From: Lumir Balhar <lbalhar@redhat.com>
Date: Wed, 10 Jan 2024 08:53:53 +0100
Subject: [PATCH] Make it possible to disable strict parsing in email module
---
Doc/library/email.utils.rst | 26 +++++++++++
Lib/email/utils.py | 54 +++++++++++++++++++++-
Lib/test/test_email/test_email.py | 74 ++++++++++++++++++++++++++++++-
3 files changed, 150 insertions(+), 4 deletions(-)
diff --git a/Doc/library/email.utils.rst b/Doc/library/email.utils.rst
index 97ddf49..0c9bf53 100644
--- a/Doc/library/email.utils.rst
+++ b/Doc/library/email.utils.rst
@@ -69,6 +69,19 @@ of the new API.
If *strict* is true, use a strict parser which rejects malformed inputs.
+ The default setting for *strict* is set to ``True``, but you can override
+ it by setting the environment variable ``PYTHON_EMAIL_DISABLE_STRICT_ADDR_PARSING``
+ to non-empty string.
+
+ Additionally, you can permanently set the default value for *strict* to
+ ``False`` by creating the configuration file ``/etc/python/email.cfg``
+ with the following content:
+
+ .. code-block:: ini
+
+ [email_addr_parsing]
+ PYTHON_EMAIL_DISABLE_STRICT_ADDR_PARSING = true
+
.. versionchanged:: 3.11.10
Add *strict* optional parameter and reject malformed inputs by default.
@@ -97,6 +110,19 @@ of the new API.
If *strict* is true, use a strict parser which rejects malformed inputs.
+ The default setting for *strict* is set to ``True``, but you can override
+ it by setting the environment variable ``PYTHON_EMAIL_DISABLE_STRICT_ADDR_PARSING``
+ to non-empty string.
+
+ Additionally, you can permanently set the default value for *strict* to
+ ``False`` by creating the configuration file ``/etc/python/email.cfg``
+ with the following content:
+
+ .. code-block:: ini
+
+ [email_addr_parsing]
+ PYTHON_EMAIL_DISABLE_STRICT_ADDR_PARSING = true
+
Here's a simple example that gets all the recipients of a message::
from email.utils import getaddresses
diff --git a/Lib/email/utils.py b/Lib/email/utils.py
index 94ead0e..09a414c 100644
--- a/Lib/email/utils.py
+++ b/Lib/email/utils.py
@@ -48,6 +48,46 @@ TICK = "'"
specialsre = re.compile(r'[][\\()<>@,:;".]')
escapesre = re.compile(r'[\\"]')
+_EMAIL_CONFIG_FILE = "/etc/python/email.cfg"
+_cached_strict_addr_parsing = None
+
+
+def _use_strict_email_parsing():
+ """"Cache implementation for _cached_strict_addr_parsing"""
+ global _cached_strict_addr_parsing
+ if _cached_strict_addr_parsing is None:
+ _cached_strict_addr_parsing = _use_strict_email_parsing_impl()
+ return _cached_strict_addr_parsing
+
+
+def _use_strict_email_parsing_impl():
+ """Returns True if strict email parsing is not disabled by
+ config file or env variable.
+ """
+ disabled = bool(os.environ.get("PYTHON_EMAIL_DISABLE_STRICT_ADDR_PARSING"))
+ if disabled:
+ return False
+
+ try:
+ file = open(_EMAIL_CONFIG_FILE)
+ except FileNotFoundError:
+ pass
+ else:
+ with file:
+ import configparser
+ config = configparser.ConfigParser(
+ interpolation=None,
+ comment_prefixes=('#', ),
+
+ )
+ config.read_file(file)
+ disabled = config.getboolean('email_addr_parsing', "PYTHON_EMAIL_DISABLE_STRICT_ADDR_PARSING", fallback=None)
+
+ if disabled:
+ return False
+
+ return True
+
def _has_surrogates(s):
"""Return True if s may contain surrogate-escaped binary data."""
@@ -149,7 +189,7 @@ def _strip_quoted_realnames(addr):
supports_strict_parsing = True
-def getaddresses(fieldvalues, *, strict=True):
+def getaddresses(fieldvalues, *, strict=None):
"""Return a list of (REALNAME, EMAIL) or ('','') for each fieldvalue.
When parsing fails for a fieldvalue, a 2-tuple of ('', '') is returned in
@@ -158,6 +198,11 @@ def getaddresses(fieldvalues, *, strict=True):
If strict is true, use a strict parser which rejects malformed inputs.
"""
+ # If default is used, it's True unless disabled
+ # by env variable or config file.
+ if strict == None:
+ strict = _use_strict_email_parsing()
+
# If strict is true, if the resulting list of parsed addresses is greater
# than the number of fieldvalues in the input list, a parsing error has
# occurred and consequently a list containing a single empty 2-tuple [('',
@@ -321,7 +366,7 @@ def parsedate_to_datetime(data):
tzinfo=datetime.timezone(datetime.timedelta(seconds=tz)))
-def parseaddr(addr, *, strict=True):
+def parseaddr(addr, *, strict=None):
"""
Parse addr into its constituent realname and email address parts.
@@ -330,6 +375,11 @@ def parseaddr(addr, *, strict=True):
If strict is True, use a strict parser which rejects malformed inputs.
"""
+ # If default is used, it's True unless disabled
+ # by env variable or config file.
+ if strict == None:
+ strict = _use_strict_email_parsing()
+
if not strict:
addrs = _AddressList(addr).addresslist
if not addrs:
diff --git a/Lib/test/test_email/test_email.py b/Lib/test/test_email/test_email.py
index ad60ed3..f85da56 100644
--- a/Lib/test/test_email/test_email.py
+++ b/Lib/test/test_email/test_email.py
@@ -8,6 +8,9 @@ import base64
import unittest
import textwrap
import warnings
+import contextlib
+import tempfile
+import os
from io import StringIO, BytesIO
from itertools import chain
@@ -41,8 +44,8 @@ from email import quoprimime
from email import utils
from test import support
-from test.support import threading_helper
-from test.support.os_helper import unlink
+from test.support import threading_helper, swap_attr
+from test.support.os_helper import unlink, EnvironmentVarGuard
from test.test_email import openfile, TestEmailBase
# These imports are documented to work, but we are testing them using a
@@ -3442,6 +3445,73 @@ Foo
# Test email.utils.supports_strict_parsing attribute
self.assertEqual(email.utils.supports_strict_parsing, True)
+ def test_parsing_errors_strict_set_via_env_var(self):
+ address = 'alice@example.org )Alice('
+ empty = ('', '')
+
+ # Reset cached default value to make the function
+ # reload the config file provided below.
+ utils._cached_strict_addr_parsing = None
+
+ # Strict disabled via env variable, old behavior expected
+ with EnvironmentVarGuard() as environ:
+ environ["PYTHON_EMAIL_DISABLE_STRICT_ADDR_PARSING"] = "1"
+
+ self.assertEqual(utils.getaddresses([address]),
+ [('', 'alice@example.org'), ('', ''), ('', 'Alice')])
+ self.assertEqual(utils.parseaddr([address]), ('', address))
+
+ # Clear cache again
+ utils._cached_strict_addr_parsing = None
+
+ # Default strict=True, empty result expected
+ self.assertEqual(utils.getaddresses([address]), [empty])
+ self.assertEqual(utils.parseaddr([address]), empty)
+
+ # Clear cache again
+ utils._cached_strict_addr_parsing = None
+
+ # Empty string in env variable = strict parsing enabled (default)
+ with EnvironmentVarGuard() as environ:
+ environ["PYTHON_EMAIL_DISABLE_STRICT_ADDR_PARSING"] = ""
+
+ # Default strict=True, empty result expected
+ self.assertEqual(utils.getaddresses([address]), [empty])
+ self.assertEqual(utils.parseaddr([address]), empty)
+
+ @contextlib.contextmanager
+ def _email_strict_parsing_conf(self):
+ """Context for the given email strict parsing configured in config file"""
+ with tempfile.TemporaryDirectory() as tmpdirname:
+ filename = os.path.join(tmpdirname, 'conf.cfg')
+ with swap_attr(utils, "_EMAIL_CONFIG_FILE", filename):
+ with open(filename, 'w') as file:
+ file.write('[email_addr_parsing]\n')
+ file.write('PYTHON_EMAIL_DISABLE_STRICT_ADDR_PARSING = true')
+ utils._EMAIL_CONFIG_FILE = filename
+ yield
+
+ def test_parsing_errors_strict_disabled_via_config_file(self):
+ address = 'alice@example.org )Alice('
+ empty = ('', '')
+
+ # Reset cached default value to make the function
+ # reload the config file provided below.
+ utils._cached_strict_addr_parsing = None
+
+ # Strict disabled via config file, old results expected
+ with self._email_strict_parsing_conf():
+ self.assertEqual(utils.getaddresses([address]),
+ [('', 'alice@example.org'), ('', ''), ('', 'Alice')])
+ self.assertEqual(utils.parseaddr([address]), ('', address))
+
+ # Clear cache again
+ utils._cached_strict_addr_parsing = None
+
+ # Default strict=True, empty result expected
+ self.assertEqual(utils.getaddresses([address]), [empty])
+ self.assertEqual(utils.parseaddr([address]), empty)
+
def test_getaddresses_nasty(self):
for addresses, expected in (
(['"Sürname, Firstname" <to@example.com>'],
--
2.46.0

View File

@ -0,0 +1,75 @@
From 670984c96eea60488c5355b4cf535c1ee3cf081a Mon Sep 17 00:00:00 2001
From: rpm-build <rpm-build>
Date: Wed, 24 Apr 2024 04:24:16 +0200
Subject: [PATCH] Fix xml tests
---
Lib/test/test_pyexpat.py | 3 +++
Lib/test/test_sax.py | 2 ++
Lib/test/test_xml_etree.py | 6 ++++++
3 files changed, 11 insertions(+)
diff --git a/Lib/test/test_pyexpat.py b/Lib/test/test_pyexpat.py
index 44bd1de..5976fa0 100644
--- a/Lib/test/test_pyexpat.py
+++ b/Lib/test/test_pyexpat.py
@@ -3,6 +3,7 @@
import os
import platform
+import pyexpat
import sys
import sysconfig
import unittest
@@ -793,6 +794,8 @@ class ReparseDeferralTest(unittest.TestCase):
self.assertEqual(started, ['doc'])
+ @unittest.skipIf(pyexpat.version_info < (2, 6, 0),
+ "Reparse deferral not defined for libexpat < 2.6.0")
def test_reparse_deferral_disabled(self):
started = []
diff --git a/Lib/test/test_sax.py b/Lib/test/test_sax.py
index 9b3014a..5960de1 100644
--- a/Lib/test/test_sax.py
+++ b/Lib/test/test_sax.py
@@ -1240,6 +1240,8 @@ class ExpatReaderTest(XmlTestBase):
self.assertEqual(result.getvalue(), start + b"<doc></doc>")
+ @unittest.skipIf(pyexpat.version_info < (2, 6, 0),
+ "Reparse deferral not defined for libexpat < 2.6.0")
def test_flush_reparse_deferral_disabled(self):
result = BytesIO()
xmlgen = XMLGenerator(result)
diff --git a/Lib/test/test_xml_etree.py b/Lib/test/test_xml_etree.py
index 8becafb..5e9b6b5 100644
--- a/Lib/test/test_xml_etree.py
+++ b/Lib/test/test_xml_etree.py
@@ -1424,9 +1424,13 @@ class XMLPullParserTest(unittest.TestCase):
self.assert_event_tags(parser, [('end', 'root')])
self.assertIsNone(parser.close())
+ @unittest.skipIf(pyexpat.version_info < (2, 6, 0),
+ "test not compatible with the latest expat security release")
def test_simple_xml_chunk_1(self):
self.test_simple_xml(chunk_size=1, flush=True)
+ @unittest.skipIf(pyexpat.version_info < (2, 6, 0),
+ "test not compatible with the latest expat security release")
def test_simple_xml_chunk_5(self):
self.test_simple_xml(chunk_size=5, flush=True)
@@ -1651,6 +1655,8 @@ class XMLPullParserTest(unittest.TestCase):
self.assert_event_tags(parser, [('end', 'doc')])
+ @unittest.skipIf(pyexpat.version_info < (2, 6, 0),
+ "Reparse deferral not defined for libexpat < 2.6.0")
def test_flush_reparse_deferral_disabled(self):
parser = ET.XMLPullParser(events=('start', 'end'))
--
2.44.0

View File

@ -1,755 +0,0 @@
From d8b0fafb202bf884135a3f7f0ce0b086217a2da2 Mon Sep 17 00:00:00 2001
From: Victor Stinner <vstinner@python.org>
Date: Fri, 15 Dec 2023 16:10:40 +0100
Subject: [PATCH 1/2] 00415: [CVE-2023-27043] gh-102988: Reject malformed
addresses in email.parseaddr() (#111116)
Detect email address parsing errors and return empty tuple to
indicate the parsing error (old API). Add an optional 'strict'
parameter to getaddresses() and parseaddr() functions. Patch by
Thomas Dwyer.
Co-Authored-By: Thomas Dwyer <github@tomd.tel>
---
Doc/library/email.utils.rst | 19 +-
Lib/email/utils.py | 151 ++++++++++++-
Lib/test/test_email/test_email.py | 204 +++++++++++++++++-
...-10-20-15-28-08.gh-issue-102988.dStNO7.rst | 8 +
4 files changed, 361 insertions(+), 21 deletions(-)
create mode 100644 Misc/NEWS.d/next/Library/2023-10-20-15-28-08.gh-issue-102988.dStNO7.rst
diff --git a/Doc/library/email.utils.rst b/Doc/library/email.utils.rst
index 0e266b6..6723dc4 100644
--- a/Doc/library/email.utils.rst
+++ b/Doc/library/email.utils.rst
@@ -60,13 +60,18 @@ of the new API.
begins with angle brackets, they are stripped off.
-.. function:: parseaddr(address)
+.. function:: parseaddr(address, *, strict=True)
Parse address -- which should be the value of some address-containing field such
as :mailheader:`To` or :mailheader:`Cc` -- into its constituent *realname* and
*email address* parts. Returns a tuple of that information, unless the parse
fails, in which case a 2-tuple of ``('', '')`` is returned.
+ If *strict* is true, use a strict parser which rejects malformed inputs.
+
+ .. versionchanged:: 3.13
+ Add *strict* optional parameter and reject malformed inputs by default.
+
.. function:: formataddr(pair, charset='utf-8')
@@ -84,12 +89,15 @@ of the new API.
Added the *charset* option.
-.. function:: getaddresses(fieldvalues)
+.. function:: getaddresses(fieldvalues, *, strict=True)
This method returns a list of 2-tuples of the form returned by ``parseaddr()``.
*fieldvalues* is a sequence of header field values as might be returned by
- :meth:`Message.get_all <email.message.Message.get_all>`. Here's a simple
- example that gets all the recipients of a message::
+ :meth:`Message.get_all <email.message.Message.get_all>`.
+
+ If *strict* is true, use a strict parser which rejects malformed inputs.
+
+ Here's a simple example that gets all the recipients of a message::
from email.utils import getaddresses
@@ -99,6 +107,9 @@ of the new API.
resent_ccs = msg.get_all('resent-cc', [])
all_recipients = getaddresses(tos + ccs + resent_tos + resent_ccs)
+ .. versionchanged:: 3.13
+ Add *strict* optional parameter and reject malformed inputs by default.
+
.. function:: parsedate(date)
diff --git a/Lib/email/utils.py b/Lib/email/utils.py
index cfdfeb3..9522341 100644
--- a/Lib/email/utils.py
+++ b/Lib/email/utils.py
@@ -48,6 +48,7 @@ TICK = "'"
specialsre = re.compile(r'[][\\()<>@,:;".]')
escapesre = re.compile(r'[\\"]')
+
def _has_surrogates(s):
"""Return True if s contains surrogate-escaped binary data."""
# This check is based on the fact that unless there are surrogates, utf8
@@ -106,12 +107,127 @@ def formataddr(pair, charset='utf-8'):
return address
+def _iter_escaped_chars(addr):
+ pos = 0
+ escape = False
+ for pos, ch in enumerate(addr):
+ if escape:
+ yield (pos, '\\' + ch)
+ escape = False
+ elif ch == '\\':
+ escape = True
+ else:
+ yield (pos, ch)
+ if escape:
+ yield (pos, '\\')
+
+
+def _strip_quoted_realnames(addr):
+ """Strip real names between quotes."""
+ if '"' not in addr:
+ # Fast path
+ return addr
+
+ start = 0
+ open_pos = None
+ result = []
+ for pos, ch in _iter_escaped_chars(addr):
+ if ch == '"':
+ if open_pos is None:
+ open_pos = pos
+ else:
+ if start != open_pos:
+ result.append(addr[start:open_pos])
+ start = pos + 1
+ open_pos = None
+
+ if start < len(addr):
+ result.append(addr[start:])
+
+ return ''.join(result)
-def getaddresses(fieldvalues):
- """Return a list of (REALNAME, EMAIL) for each fieldvalue."""
- all = COMMASPACE.join(str(v) for v in fieldvalues)
- a = _AddressList(all)
- return a.addresslist
+
+supports_strict_parsing = True
+
+def getaddresses(fieldvalues, *, strict=True):
+ """Return a list of (REALNAME, EMAIL) or ('','') for each fieldvalue.
+
+ When parsing fails for a fieldvalue, a 2-tuple of ('', '') is returned in
+ its place.
+
+ If strict is true, use a strict parser which rejects malformed inputs.
+ """
+
+ # If strict is true, if the resulting list of parsed addresses is greater
+ # than the number of fieldvalues in the input list, a parsing error has
+ # occurred and consequently a list containing a single empty 2-tuple [('',
+ # '')] is returned in its place. This is done to avoid invalid output.
+ #
+ # Malformed input: getaddresses(['alice@example.com <bob@example.com>'])
+ # Invalid output: [('', 'alice@example.com'), ('', 'bob@example.com')]
+ # Safe output: [('', '')]
+
+ if not strict:
+ all = COMMASPACE.join(str(v) for v in fieldvalues)
+ a = _AddressList(all)
+ return a.addresslist
+
+ fieldvalues = [str(v) for v in fieldvalues]
+ fieldvalues = _pre_parse_validation(fieldvalues)
+ addr = COMMASPACE.join(fieldvalues)
+ a = _AddressList(addr)
+ result = _post_parse_validation(a.addresslist)
+
+ # Treat output as invalid if the number of addresses is not equal to the
+ # expected number of addresses.
+ n = 0
+ for v in fieldvalues:
+ # When a comma is used in the Real Name part it is not a deliminator.
+ # So strip those out before counting the commas.
+ v = _strip_quoted_realnames(v)
+ # Expected number of addresses: 1 + number of commas
+ n += 1 + v.count(',')
+ if len(result) != n:
+ return [('', '')]
+
+ return result
+
+
+def _check_parenthesis(addr):
+ # Ignore parenthesis in quoted real names.
+ addr = _strip_quoted_realnames(addr)
+
+ opens = 0
+ for pos, ch in _iter_escaped_chars(addr):
+ if ch == '(':
+ opens += 1
+ elif ch == ')':
+ opens -= 1
+ if opens < 0:
+ return False
+ return (opens == 0)
+
+
+def _pre_parse_validation(email_header_fields):
+ accepted_values = []
+ for v in email_header_fields:
+ if not _check_parenthesis(v):
+ v = "('', '')"
+ accepted_values.append(v)
+
+ return accepted_values
+
+
+def _post_parse_validation(parsed_email_header_tuples):
+ accepted_values = []
+ # The parser would have parsed a correctly formatted domain-literal
+ # The existence of an [ after parsing indicates a parsing failure
+ for v in parsed_email_header_tuples:
+ if '[' in v[1]:
+ v = ('', '')
+ accepted_values.append(v)
+
+ return accepted_values
def _format_timetuple_and_zone(timetuple, zone):
@@ -205,16 +321,33 @@ def parsedate_to_datetime(data):
tzinfo=datetime.timezone(datetime.timedelta(seconds=tz)))
-def parseaddr(addr):
+def parseaddr(addr, *, strict=True):
"""
Parse addr into its constituent realname and email address parts.
Return a tuple of realname and email address, unless the parse fails, in
which case return a 2-tuple of ('', '').
+
+ If strict is True, use a strict parser which rejects malformed inputs.
"""
- addrs = _AddressList(addr).addresslist
- if not addrs:
- return '', ''
+ if not strict:
+ addrs = _AddressList(addr).addresslist
+ if not addrs:
+ return ('', '')
+ return addrs[0]
+
+ if isinstance(addr, list):
+ addr = addr[0]
+
+ if not isinstance(addr, str):
+ return ('', '')
+
+ addr = _pre_parse_validation([addr])[0]
+ addrs = _post_parse_validation(_AddressList(addr).addresslist)
+
+ if not addrs or len(addrs) > 1:
+ return ('', '')
+
return addrs[0]
diff --git a/Lib/test/test_email/test_email.py b/Lib/test/test_email/test_email.py
index 677f209..20b6779 100644
--- a/Lib/test/test_email/test_email.py
+++ b/Lib/test/test_email/test_email.py
@@ -17,6 +17,7 @@ from unittest.mock import patch
import email
import email.policy
+import email.utils
from email.charset import Charset
from email.generator import Generator, DecodedGenerator, BytesGenerator
@@ -3321,15 +3322,154 @@ Foo
[('Al Person', 'aperson@dom.ain'),
('Bud Person', 'bperson@dom.ain')])
+ def test_getaddresses_comma_in_name(self):
+ """GH-106669 regression test."""
+ self.assertEqual(
+ utils.getaddresses(
+ [
+ '"Bud, Person" <bperson@dom.ain>',
+ 'aperson@dom.ain (Al Person)',
+ '"Mariusz Felisiak" <to@example.com>',
+ ]
+ ),
+ [
+ ('Bud, Person', 'bperson@dom.ain'),
+ ('Al Person', 'aperson@dom.ain'),
+ ('Mariusz Felisiak', 'to@example.com'),
+ ],
+ )
+
+ def test_parsing_errors(self):
+ """Test for parsing errors from CVE-2023-27043 and CVE-2019-16056"""
+ alice = 'alice@example.org'
+ bob = 'bob@example.com'
+ empty = ('', '')
+
+ # Test utils.getaddresses() and utils.parseaddr() on malformed email
+ # addresses: default behavior (strict=True) rejects malformed address,
+ # and strict=False which tolerates malformed address.
+ for invalid_separator, expected_non_strict in (
+ ('(', [(f'<{bob}>', alice)]),
+ (')', [('', alice), empty, ('', bob)]),
+ ('<', [('', alice), empty, ('', bob), empty]),
+ ('>', [('', alice), empty, ('', bob)]),
+ ('[', [('', f'{alice}[<{bob}>]')]),
+ (']', [('', alice), empty, ('', bob)]),
+ ('@', [empty, empty, ('', bob)]),
+ (';', [('', alice), empty, ('', bob)]),
+ (':', [('', alice), ('', bob)]),
+ ('.', [('', alice + '.'), ('', bob)]),
+ ('"', [('', alice), ('', f'<{bob}>')]),
+ ):
+ address = f'{alice}{invalid_separator}<{bob}>'
+ with self.subTest(address=address):
+ self.assertEqual(utils.getaddresses([address]),
+ [empty])
+ self.assertEqual(utils.getaddresses([address], strict=False),
+ expected_non_strict)
+
+ self.assertEqual(utils.parseaddr([address]),
+ empty)
+ self.assertEqual(utils.parseaddr([address], strict=False),
+ ('', address))
+
+ # Comma (',') is treated differently depending on strict parameter.
+ # Comma without quotes.
+ address = f'{alice},<{bob}>'
+ self.assertEqual(utils.getaddresses([address]),
+ [('', alice), ('', bob)])
+ self.assertEqual(utils.getaddresses([address], strict=False),
+ [('', alice), ('', bob)])
+ self.assertEqual(utils.parseaddr([address]),
+ empty)
+ self.assertEqual(utils.parseaddr([address], strict=False),
+ ('', address))
+
+ # Real name between quotes containing comma.
+ address = '"Alice, alice@example.org" <bob@example.com>'
+ expected_strict = ('Alice, alice@example.org', 'bob@example.com')
+ self.assertEqual(utils.getaddresses([address]), [expected_strict])
+ self.assertEqual(utils.getaddresses([address], strict=False), [expected_strict])
+ self.assertEqual(utils.parseaddr([address]), expected_strict)
+ self.assertEqual(utils.parseaddr([address], strict=False),
+ ('', address))
+
+ # Valid parenthesis in comments.
+ address = 'alice@example.org (Alice)'
+ expected_strict = ('Alice', 'alice@example.org')
+ self.assertEqual(utils.getaddresses([address]), [expected_strict])
+ self.assertEqual(utils.getaddresses([address], strict=False), [expected_strict])
+ self.assertEqual(utils.parseaddr([address]), expected_strict)
+ self.assertEqual(utils.parseaddr([address], strict=False),
+ ('', address))
+
+ # Invalid parenthesis in comments.
+ address = 'alice@example.org )Alice('
+ self.assertEqual(utils.getaddresses([address]), [empty])
+ self.assertEqual(utils.getaddresses([address], strict=False),
+ [('', 'alice@example.org'), ('', ''), ('', 'Alice')])
+ self.assertEqual(utils.parseaddr([address]), empty)
+ self.assertEqual(utils.parseaddr([address], strict=False),
+ ('', address))
+
+ # Two addresses with quotes separated by comma.
+ address = '"Jane Doe" <jane@example.net>, "John Doe" <john@example.net>'
+ self.assertEqual(utils.getaddresses([address]),
+ [('Jane Doe', 'jane@example.net'),
+ ('John Doe', 'john@example.net')])
+ self.assertEqual(utils.getaddresses([address], strict=False),
+ [('Jane Doe', 'jane@example.net'),
+ ('John Doe', 'john@example.net')])
+ self.assertEqual(utils.parseaddr([address]), empty)
+ self.assertEqual(utils.parseaddr([address], strict=False),
+ ('', address))
+
+ # Test email.utils.supports_strict_parsing attribute
+ self.assertEqual(email.utils.supports_strict_parsing, True)
+
def test_getaddresses_nasty(self):
- eq = self.assertEqual
- eq(utils.getaddresses(['foo: ;']), [('', '')])
- eq(utils.getaddresses(
- ['[]*-- =~$']),
- [('', ''), ('', ''), ('', '*--')])
- eq(utils.getaddresses(
- ['foo: ;', '"Jason R. Mastaler" <jason@dom.ain>']),
- [('', ''), ('Jason R. Mastaler', 'jason@dom.ain')])
+ for addresses, expected in (
+ (['"Sürname, Firstname" <to@example.com>'],
+ [('Sürname, Firstname', 'to@example.com')]),
+
+ (['foo: ;'],
+ [('', '')]),
+
+ (['foo: ;', '"Jason R. Mastaler" <jason@dom.ain>'],
+ [('', ''), ('Jason R. Mastaler', 'jason@dom.ain')]),
+
+ ([r'Pete(A nice \) chap) <pete(his account)@silly.test(his host)>'],
+ [('Pete (A nice ) chap his account his host)', 'pete@silly.test')]),
+
+ (['(Empty list)(start)Undisclosed recipients :(nobody(I know))'],
+ [('', '')]),
+
+ (['Mary <@machine.tld:mary@example.net>, , jdoe@test . example'],
+ [('Mary', 'mary@example.net'), ('', ''), ('', 'jdoe@test.example')]),
+
+ (['John Doe <jdoe@machine(comment). example>'],
+ [('John Doe (comment)', 'jdoe@machine.example')]),
+
+ (['"Mary Smith: Personal Account" <smith@home.example>'],
+ [('Mary Smith: Personal Account', 'smith@home.example')]),
+
+ (['Undisclosed recipients:;'],
+ [('', '')]),
+
+ ([r'<boss@nil.test>, "Giant; \"Big\" Box" <bob@example.net>'],
+ [('', 'boss@nil.test'), ('Giant; "Big" Box', 'bob@example.net')]),
+ ):
+ with self.subTest(addresses=addresses):
+ self.assertEqual(utils.getaddresses(addresses),
+ expected)
+ self.assertEqual(utils.getaddresses(addresses, strict=False),
+ expected)
+
+ addresses = ['[]*-- =~$']
+ self.assertEqual(utils.getaddresses(addresses),
+ [('', '')])
+ self.assertEqual(utils.getaddresses(addresses, strict=False),
+ [('', ''), ('', ''), ('', '*--')])
def test_getaddresses_embedded_comment(self):
"""Test proper handling of a nested comment"""
@@ -3520,6 +3660,54 @@ multipart/report
m = cls(*constructor, policy=email.policy.default)
self.assertIs(m.policy, email.policy.default)
+ def test_iter_escaped_chars(self):
+ self.assertEqual(list(utils._iter_escaped_chars(r'a\\b\"c\\"d')),
+ [(0, 'a'),
+ (2, '\\\\'),
+ (3, 'b'),
+ (5, '\\"'),
+ (6, 'c'),
+ (8, '\\\\'),
+ (9, '"'),
+ (10, 'd')])
+ self.assertEqual(list(utils._iter_escaped_chars('a\\')),
+ [(0, 'a'), (1, '\\')])
+
+ def test_strip_quoted_realnames(self):
+ def check(addr, expected):
+ self.assertEqual(utils._strip_quoted_realnames(addr), expected)
+
+ check('"Jane Doe" <jane@example.net>, "John Doe" <john@example.net>',
+ ' <jane@example.net>, <john@example.net>')
+ check(r'"Jane \"Doe\"." <jane@example.net>',
+ ' <jane@example.net>')
+
+ # special cases
+ check(r'before"name"after', 'beforeafter')
+ check(r'before"name"', 'before')
+ check(r'b"name"', 'b') # single char
+ check(r'"name"after', 'after')
+ check(r'"name"a', 'a') # single char
+ check(r'"name"', '')
+
+ # no change
+ for addr in (
+ 'Jane Doe <jane@example.net>, John Doe <john@example.net>',
+ 'lone " quote',
+ ):
+ self.assertEqual(utils._strip_quoted_realnames(addr), addr)
+
+
+ def test_check_parenthesis(self):
+ addr = 'alice@example.net'
+ self.assertTrue(utils._check_parenthesis(f'{addr} (Alice)'))
+ self.assertFalse(utils._check_parenthesis(f'{addr} )Alice('))
+ self.assertFalse(utils._check_parenthesis(f'{addr} (Alice))'))
+ self.assertFalse(utils._check_parenthesis(f'{addr} ((Alice)'))
+
+ # Ignore real name between quotes
+ self.assertTrue(utils._check_parenthesis(f'")Alice((" {addr}'))
+
# Test the iterator/generators
class TestIterators(TestEmailBase):
diff --git a/Misc/NEWS.d/next/Library/2023-10-20-15-28-08.gh-issue-102988.dStNO7.rst b/Misc/NEWS.d/next/Library/2023-10-20-15-28-08.gh-issue-102988.dStNO7.rst
new file mode 100644
index 0000000..3d0e9e4
--- /dev/null
+++ b/Misc/NEWS.d/next/Library/2023-10-20-15-28-08.gh-issue-102988.dStNO7.rst
@@ -0,0 +1,8 @@
+:func:`email.utils.getaddresses` and :func:`email.utils.parseaddr` now
+return ``('', '')`` 2-tuples in more situations where invalid email
+addresses are encountered instead of potentially inaccurate values. Add
+optional *strict* parameter to these two functions: use ``strict=False`` to
+get the old behavior, accept malformed inputs.
+``getattr(email.utils, 'supports_strict_parsing', False)`` can be use to check
+if the *strict* paramater is available. Patch by Thomas Dwyer and Victor
+Stinner to improve the CVE-2023-27043 fix.
--
2.43.0
From 6c34f5b95da90bd494e29776c0e807af44689fae Mon Sep 17 00:00:00 2001
From: Lumir Balhar <lbalhar@redhat.com>
Date: Wed, 10 Jan 2024 08:53:53 +0100
Subject: [PATCH 2/2] Make it possible to disable strict parsing in email
module
---
Doc/library/email.utils.rst | 26 +++++++++++
Lib/email/utils.py | 54 +++++++++++++++++++++-
Lib/test/test_email/test_email.py | 74 ++++++++++++++++++++++++++++++-
3 files changed, 150 insertions(+), 4 deletions(-)
diff --git a/Doc/library/email.utils.rst b/Doc/library/email.utils.rst
index 6723dc4..c89602d 100644
--- a/Doc/library/email.utils.rst
+++ b/Doc/library/email.utils.rst
@@ -69,6 +69,19 @@ of the new API.
If *strict* is true, use a strict parser which rejects malformed inputs.
+ The default setting for *strict* is set to ``True``, but you can override
+ it by setting the environment variable ``PYTHON_EMAIL_DISABLE_STRICT_ADDR_PARSING``
+ to non-empty string.
+
+ Additionally, you can permanently set the default value for *strict* to
+ ``False`` by creating the configuration file ``/etc/python/email.cfg``
+ with the following content:
+
+ .. code-block:: ini
+
+ [email_addr_parsing]
+ PYTHON_EMAIL_DISABLE_STRICT_ADDR_PARSING = true
+
.. versionchanged:: 3.13
Add *strict* optional parameter and reject malformed inputs by default.
@@ -97,6 +110,19 @@ of the new API.
If *strict* is true, use a strict parser which rejects malformed inputs.
+ The default setting for *strict* is set to ``True``, but you can override
+ it by setting the environment variable ``PYTHON_EMAIL_DISABLE_STRICT_ADDR_PARSING``
+ to non-empty string.
+
+ Additionally, you can permanently set the default value for *strict* to
+ ``False`` by creating the configuration file ``/etc/python/email.cfg``
+ with the following content:
+
+ .. code-block:: ini
+
+ [email_addr_parsing]
+ PYTHON_EMAIL_DISABLE_STRICT_ADDR_PARSING = true
+
Here's a simple example that gets all the recipients of a message::
from email.utils import getaddresses
diff --git a/Lib/email/utils.py b/Lib/email/utils.py
index 9522341..2e30e09 100644
--- a/Lib/email/utils.py
+++ b/Lib/email/utils.py
@@ -48,6 +48,46 @@ TICK = "'"
specialsre = re.compile(r'[][\\()<>@,:;".]')
escapesre = re.compile(r'[\\"]')
+_EMAIL_CONFIG_FILE = "/etc/python/email.cfg"
+_cached_strict_addr_parsing = None
+
+
+def _use_strict_email_parsing():
+ """"Cache implementation for _cached_strict_addr_parsing"""
+ global _cached_strict_addr_parsing
+ if _cached_strict_addr_parsing is None:
+ _cached_strict_addr_parsing = _use_strict_email_parsing_impl()
+ return _cached_strict_addr_parsing
+
+
+def _use_strict_email_parsing_impl():
+ """Returns True if strict email parsing is not disabled by
+ config file or env variable.
+ """
+ disabled = bool(os.environ.get("PYTHON_EMAIL_DISABLE_STRICT_ADDR_PARSING"))
+ if disabled:
+ return False
+
+ try:
+ file = open(_EMAIL_CONFIG_FILE)
+ except FileNotFoundError:
+ pass
+ else:
+ with file:
+ import configparser
+ config = configparser.ConfigParser(
+ interpolation=None,
+ comment_prefixes=('#', ),
+
+ )
+ config.read_file(file)
+ disabled = config.getboolean('email_addr_parsing', "PYTHON_EMAIL_DISABLE_STRICT_ADDR_PARSING", fallback=None)
+
+ if disabled:
+ return False
+
+ return True
+
def _has_surrogates(s):
"""Return True if s contains surrogate-escaped binary data."""
@@ -149,7 +189,7 @@ def _strip_quoted_realnames(addr):
supports_strict_parsing = True
-def getaddresses(fieldvalues, *, strict=True):
+def getaddresses(fieldvalues, *, strict=None):
"""Return a list of (REALNAME, EMAIL) or ('','') for each fieldvalue.
When parsing fails for a fieldvalue, a 2-tuple of ('', '') is returned in
@@ -158,6 +198,11 @@ def getaddresses(fieldvalues, *, strict=True):
If strict is true, use a strict parser which rejects malformed inputs.
"""
+ # If default is used, it's True unless disabled
+ # by env variable or config file.
+ if strict == None:
+ strict = _use_strict_email_parsing()
+
# If strict is true, if the resulting list of parsed addresses is greater
# than the number of fieldvalues in the input list, a parsing error has
# occurred and consequently a list containing a single empty 2-tuple [('',
@@ -321,7 +366,7 @@ def parsedate_to_datetime(data):
tzinfo=datetime.timezone(datetime.timedelta(seconds=tz)))
-def parseaddr(addr, *, strict=True):
+def parseaddr(addr, *, strict=None):
"""
Parse addr into its constituent realname and email address parts.
@@ -330,6 +375,11 @@ def parseaddr(addr, *, strict=True):
If strict is True, use a strict parser which rejects malformed inputs.
"""
+ # If default is used, it's True unless disabled
+ # by env variable or config file.
+ if strict == None:
+ strict = _use_strict_email_parsing()
+
if not strict:
addrs = _AddressList(addr).addresslist
if not addrs:
diff --git a/Lib/test/test_email/test_email.py b/Lib/test/test_email/test_email.py
index 20b6779..d7d99f0 100644
--- a/Lib/test/test_email/test_email.py
+++ b/Lib/test/test_email/test_email.py
@@ -8,6 +8,9 @@ import base64
import unittest
import textwrap
import warnings
+import contextlib
+import tempfile
+import os
from io import StringIO, BytesIO
from itertools import chain
@@ -41,8 +44,8 @@ from email import quoprimime
from email import utils
from test import support
-from test.support import threading_helper
-from test.support.os_helper import unlink
+from test.support import threading_helper, swap_attr
+from test.support.os_helper import unlink, EnvironmentVarGuard
from test.test_email import openfile, TestEmailBase
# These imports are documented to work, but we are testing them using a
@@ -3427,6 +3430,73 @@ Foo
# Test email.utils.supports_strict_parsing attribute
self.assertEqual(email.utils.supports_strict_parsing, True)
+ def test_parsing_errors_strict_set_via_env_var(self):
+ address = 'alice@example.org )Alice('
+ empty = ('', '')
+
+ # Reset cached default value to make the function
+ # reload the config file provided below.
+ utils._cached_strict_addr_parsing = None
+
+ # Strict disabled via env variable, old behavior expected
+ with EnvironmentVarGuard() as environ:
+ environ["PYTHON_EMAIL_DISABLE_STRICT_ADDR_PARSING"] = "1"
+
+ self.assertEqual(utils.getaddresses([address]),
+ [('', 'alice@example.org'), ('', ''), ('', 'Alice')])
+ self.assertEqual(utils.parseaddr([address]), ('', address))
+
+ # Clear cache again
+ utils._cached_strict_addr_parsing = None
+
+ # Default strict=True, empty result expected
+ self.assertEqual(utils.getaddresses([address]), [empty])
+ self.assertEqual(utils.parseaddr([address]), empty)
+
+ # Clear cache again
+ utils._cached_strict_addr_parsing = None
+
+ # Empty string in env variable = strict parsing enabled (default)
+ with EnvironmentVarGuard() as environ:
+ environ["PYTHON_EMAIL_DISABLE_STRICT_ADDR_PARSING"] = ""
+
+ # Default strict=True, empty result expected
+ self.assertEqual(utils.getaddresses([address]), [empty])
+ self.assertEqual(utils.parseaddr([address]), empty)
+
+ @contextlib.contextmanager
+ def _email_strict_parsing_conf(self):
+ """Context for the given email strict parsing configured in config file"""
+ with tempfile.TemporaryDirectory() as tmpdirname:
+ filename = os.path.join(tmpdirname, 'conf.cfg')
+ with swap_attr(utils, "_EMAIL_CONFIG_FILE", filename):
+ with open(filename, 'w') as file:
+ file.write('[email_addr_parsing]\n')
+ file.write('PYTHON_EMAIL_DISABLE_STRICT_ADDR_PARSING = true')
+ utils._EMAIL_CONFIG_FILE = filename
+ yield
+
+ def test_parsing_errors_strict_disabled_via_config_file(self):
+ address = 'alice@example.org )Alice('
+ empty = ('', '')
+
+ # Reset cached default value to make the function
+ # reload the config file provided below.
+ utils._cached_strict_addr_parsing = None
+
+ # Strict disabled via config file, old results expected
+ with self._email_strict_parsing_conf():
+ self.assertEqual(utils.getaddresses([address]),
+ [('', 'alice@example.org'), ('', ''), ('', 'Alice')])
+ self.assertEqual(utils.parseaddr([address]), ('', address))
+
+ # Clear cache again
+ utils._cached_strict_addr_parsing = None
+
+ # Default strict=True, empty result expected
+ self.assertEqual(utils.getaddresses([address]), [empty])
+ self.assertEqual(utils.parseaddr([address]), empty)
+
def test_getaddresses_nasty(self):
for addresses, expected in (
(['"Sürname, Firstname" <to@example.com>'],
--
2.43.0

View File

@ -1,16 +0,0 @@
-----BEGIN PGP SIGNATURE-----
iQIzBAABCAAdFiEEz9yiRbEEPPKl+Xhl/+h0BBaL2EcFAmVuFigACgkQ/+h0BBaL
2EeHPg/+LU5xs2ZDrQogDcH+A1v8RyursiggypdM5hXTrsFsTCIk4iekcI9xkhG1
ltNX4UuCe5PUEbTgtaWP0ncXARrUnPCoQaQ1sHVDTYoHegancsk+sXZc1JM7qr0p
Y4Ig6mKjuHFMXCInQSI2GaH4t5r4Z1jGk/PGrecIHOPJgqfA/6Z3TBF5N+y3jEvS
2QazMB298q4RDhh9m3REe8LwFPHDlfw9eRohv0MB8xygg9KtxhLZrN7gLBQZvKGD
ihNw6EgJj5OZ0dvwKCCXnlZuwknuJW7vAOPHhYeenPdVdYCGoRSyN7JdD07L+5AG
O14l2rqZrz5Eu28by+kAUrcPYAfAXekw1PmtT3HSd9U/nqnUiTkkJcjyGG/e3cjJ
sUDKMNCSBq0G7j5DB3bB6VHkZjVuz+T+iR5QdfJ4kI2pYSuE/rUj1rhkUXApYsHl
7Wff0QbOW6QT1wCtQcMpJSzkTDVJVYxiqrko/ihlOhphDHYLdOIGOrxWAUwc06x/
BhJD6tM1kEVZvifoJp1OsNwDzZ/Ku6CUs05E1vWxdeNVeANyKAgCZ5hOVmhnv866
11zfgo/znRsMzMIyJuy0bhO0C6omVLzzfhipAbZM2jDorn37xxV0v/I0pceNtLrp
YR7Tjs7+Ihe6/oItjW53j9T7ANdgQ1RVDg98lKlPFNL+hxfctwY=
=0Pkd
-----END PGP SIGNATURE-----

7
gating.yaml Normal file
View File

@ -0,0 +1,7 @@
--- !Policy
product_versions:
- rhel-8
decision_context: osci_compose_gate
rules:
- !PassingTestCaseRule {test_case_name: osci.brew-build.tier0.functional}

16
plan.fmf Normal file
View File

@ -0,0 +1,16 @@
execute:
how: tmt
environment:
VERSION: 3.11
# the X variable is what is passed as arguments to python -m test
# test_check_probes https://github.com/python/cpython/issues/104280#issuecomment-1669249980
X: '-i test_check_probes'
discover:
how: fmf
url: https://src.fedoraproject.org/tests/python.git
test:
- /smoke/venv
- /selftest/parallel

View File

@ -16,7 +16,7 @@ URL: https://www.python.org/
# WARNING When rebasing to a new Python version,
# remember to update the python3-docs package as well
%global general_version %{pybasever}.7
%global general_version %{pybasever}.10
#global prerel ...
%global upstream_version %{general_version}%{?prerel}
Version: %{general_version}%{?prerel:~%{prerel}}
@ -63,7 +63,7 @@ License: Python
# If the rpmwheels condition is disabled, we use the bundled wheel packages
# from Python with the versions below.
# This needs to be manually updated when we update Python.
%global pip_version 23.2.1
%global pip_version 24.0
%global setuptools_version 65.5.0
# Expensive optimizations (mainly, profile-guided optimizations)
@ -145,6 +145,13 @@ License: Python
%global py_INSTSONAME_optimized libpython%{LDVERSION_optimized}.so.%{py_SOVERSION}
%global py_INSTSONAME_debug libpython%{LDVERSION_debug}.so.%{py_SOVERSION}
# The -O flag for the compiler, optimized builds
# https://fedoraproject.org/wiki/Changes/Python_built_with_gcc_O3
%global optflags_optimized -O3
# The -O flag for the compiler, debug builds
# -Wno-cpp avoids some warnings with -O0
%global optflags_debug -O0 -Wno-cpp
# Disable automatic bytecompilation. The python3 binary is not yet be
# available in /usr/bin when Python is built. Also, the bytecompilation fails
# on files that test invalid syntax.
@ -381,10 +388,16 @@ Patch397: 00397-tarfile-filter.patch
#
# Upstream PR: https://github.com/python/cpython/pull/111116
#
# Second patch implmenets the possibility to restore the old behavior via
# The patch implements the possibility to restore the old behavior via
# config file or environment variable.
Patch415: 00415-cve-2023-27043-gh-102988-reject-malformed-addresses-in-email-parseaddr-111116.patch
# 00422 #
# Fix the test suite for releases of expat < 2.6.0
# which backport the CVE-2023-52425 fix.
# Downstream only.
Patch422: 00422-fix-expat-tests.patch
# (New patches go here ^^^)
#
# When adding new patches to "python" and "python3" in Fedora, EL, etc.,
@ -798,6 +811,7 @@ BuildPython() {
ConfName=$1
ExtraConfigArgs=$2
MoreCFlags=$3
MoreCFlagsNodist=$4
# Each build is done in its own directory
ConfDir=build/$ConfName
@ -837,7 +851,7 @@ BuildPython() {
$ExtraConfigArgs \
%{nil}
%global flags_override EXTRA_CFLAGS="$MoreCFlags" CFLAGS_NODIST="$CFLAGS_NODIST $MoreCFlags"
%global flags_override EXTRA_CFLAGS="$MoreCFlags" CFLAGS_NODIST="$CFLAGS_NODIST $MoreCFlags $MoreCFlagsNodist"
%if %{without bootstrap}
# Regenerate generated files (needs python3)
@ -860,12 +874,14 @@ BuildPython() {
# See also: https://bugzilla.redhat.com/show_bug.cgi?id=1818857
BuildPython debug \
"--without-ensurepip --with-pydebug" \
"-O0 -Wno-cpp"
"%{optflags_debug}" \
""
%endif # with debug_build
BuildPython optimized \
"--without-ensurepip %{optimizations_flag}" \
""
"" \
"%{optflags_optimized}"
# ======================================================
# Installing the built code:
@ -964,7 +980,7 @@ EOF
%if %{with debug_build}
InstallPython debug \
%{py_INSTSONAME_debug} \
-O0 \
"%{optflags_debug}" \
%{LDVERSION_debug}
%endif # with debug_build
@ -1839,6 +1855,39 @@ fi
# ======================================================
%changelog
* Mon Sep 09 2024 Tomáš Hrnčiar <thrnciar@redhat.com> - 3.11.10-1
- Update to 3.11.10
Resolves: RHEL-57400
* Fri Aug 23 2024 Charalampos Stratakis <cstratak@redhat.com> - 3.11.9-7
- Security fix for CVE-2024-8088
Resolves: RHEL-55934
* Thu Aug 15 2024 Charalampos Stratakis <cstratak@redhat.com> - 3.11.9-6
- Security fix for CVE-2024-6923
Resolves: RHEL-53089
* Thu Jul 25 2024 Charalampos Stratakis <cstratak@redhat.com> - 3.11.9-5
- Properly propagate the optimization flags to C extensions
* Thu Jul 18 2024 Charalampos Stratakis <cstratak@redhat.com> - 3.11.9-4
- Build Python with -O3
- https://fedoraproject.org/wiki/Changes/Python_built_with_gcc_O3
* Thu Jul 18 2024 Charalampos Stratakis <cstratak@redhat.com> - 3.11.9-3
- Security fix for CVE-2024-4032
Resolves: RHEL-44067
* Tue Jun 11 2024 Charalampos Stratakis <cstratak@redhat.com> - 3.11.9-2
- Enable importing of hash-based .pyc files under FIPS mode
Resolves: RHEL-40783
* Mon Apr 22 2024 Charalampos Stratakis <cstratak@redhat.com> - 3.11.9-1
- Rebase to 3.11.9
- Security fixes for CVE-2023-6597 and CVE-2024-0450
- Fix expat tests for the latest expat security release
Resolves: RHEL-33672, RHEL-33684
* Mon Jan 22 2024 Charalampos Stratakis <cstratak@redhat.com> - 3.11.7-1
- Rebase to 3.11.7
Resolves: RHEL-21915

37
rpminspect.yaml Normal file
View File

@ -0,0 +1,37 @@
# exclude test XML data (not always valid) from XML validity check:
xml:
ignore:
- /usr/lib*/python*/test/xmltestdata/*
- /usr/lib*/python*/test/xmltestdata/*/*
# exclude _socket from ipv4 only functions check, it has both ipv4 and ipv6 only
badfuncs:
allowed:
/usr/lib*/python*/lib-dynload/_socket.*:
- inet_aton
- inet_ntoa
# exclude the debug build from annocheck entirely
annocheck:
ignore:
- /usr/bin/python*d
- /usr/lib*/libpython*d.so.1.0
- /usr/lib*/python*/lib-dynload/*.cpython-*d-*-*-*.so
# don't report changed content of compiled files
# that is expected with every toolchain update and not reproducible yet
changedfiles:
# note that this is a posix regex, so no \d
exclude_path: (\.so(\.[0-9]+(\.[0-9]+)?)?$|^/usr/bin/python[0-9]+\.[0-9]+d?m?$)
# files change size all the time, we don't need to VERIFY it
# however, the INFO is useful, so we don't disable the check entirely
filesize:
# artificially large number, TODO a better way
size_threshold: 100000
# completely disabled inspections:
inspections:
# we know about our patches, no need to report anything
patches: off

2
sources Normal file
View File

@ -0,0 +1,2 @@
SHA512 (Python-3.11.10.tar.xz) = 6ce77cced1ce90bb5eea38504dfc1bc19c872149a5a63fdd8353ac8c772c54ab7a42176e141c7f6f898d31761bf93e1739b238920fbeefbedd6016ad033c1de0
SHA512 (Python-3.11.10.tar.xz.asc) = 3a9efe1dd39fd6883ae3fc8dd7f6e25af7e06c6e6049cf9a31a1a82e6d7c84f85ac838d2a71127977d93ce77233a8bbff86a10a80d24fcee85a4e70fcbd1db19

1
tests/.gitignore vendored Normal file
View File

@ -0,0 +1 @@
__*__/

87
tests/pythontest.spec Normal file
View File

@ -0,0 +1,87 @@
%global __python3 /usr/bin/python3.11
%global basedir /opt/test/byte_compilation
# We have 3 different ways of bytecompiling: for 3.9+, 3.4-3.8, and 2.7
# Test with a representative of each.
%global python36_sitelib /usr/lib/python3.6/site-packages
%global python27_sitelib /usr/lib/python2.7/site-packages
Name: pythontest
Version: 0
Release: 0%{?dist}
Summary: ...
License: MIT
BuildRequires: python3-devel
BuildRequires: python3
BuildRequires: python2
BuildRequires: python3.11-devel
BuildRequires: python3.11-rpm-macros
%description
...
%install
mkdir -p %{buildroot}%{basedir}/directory/to/test/recursion
echo "print()" > %{buildroot}%{basedir}/file.py
echo "print()" > %{buildroot}%{basedir}/directory/to/test/recursion/file_in_dir.py
%py_byte_compile %{python3} %{buildroot}%{basedir}/file.py
%py_byte_compile %{python3} %{buildroot}%{basedir}/directory
# Files in sitelib are compiled automatically by brp-python-bytecompile
mkdir -p %{buildroot}%{python3_sitelib}/directory/
echo "print()" > %{buildroot}%{python3_sitelib}/directory/file.py
mkdir -p %{buildroot}%{python36_sitelib}/directory/
echo "print()" > %{buildroot}%{python36_sitelib}/directory/file.py
mkdir -p %{buildroot}%{python27_sitelib}/directory/
echo "print()" > %{buildroot}%{python27_sitelib}/directory/file.py
%check
LOCATIONS="
%{buildroot}%{basedir}
%{buildroot}%{python3_sitelib}/directory/
%{buildroot}%{python36_sitelib}/directory/
%{buildroot}%{python27_sitelib}/directory/
"
echo "============== Print .py files found in LOCATIONS =================="
find $LOCATIONS -name "*.py"
echo "============== Print .pyc files found in LOCATIONS =================="
find $LOCATIONS -name "*.py[co]"
echo "============== End of print =================="
# Count .py and .pyc files
PY=$(find $LOCATIONS -name "*.py" | wc -l)
PYC=$(find $LOCATIONS -name "*.py[co]" | wc -l)
# We should have 5 .py files (3 for python3, one each for 3.6 & 2.7)
test $PY -eq 5
# Every .py file should be byte-compiled to two .pyc files (optimization level 0 and 1)
# so we should have two times more .pyc files than .py files
test $(expr $PY \* 2) -eq $PYC
# In this case the .pyc files should be identical across omtimization levels
# (they don't use docstrings and assert staements)
# So they should be hardlinked; the number of distinct inodes should match the
# number of source files. (Or be smaller, if the dupe detection is done
# across all files.)
INODES=$(stat --format %i $(find $LOCATIONS -name "*.py[co]") | sort -u | wc -l)
test $PY -ge $INODES
%files
%pycached %{basedir}/file.py
%pycached %{basedir}/directory/to/test/recursion/file_in_dir.py
%pycached %{python3_sitelib}/directory/file.py
%pycached %{python36_sitelib}/directory/file.py
%{python27_sitelib}/directory/file.py*
%changelog
* Thu Jan 01 2015 Fedora Packager <nobody@fedoraproject.org> - 0-0
- This changelog entry exists and is deliberately set in the past

922
tests/test_evals.py Normal file
View File

@ -0,0 +1,922 @@
import os
import subprocess
import platform
import re
import sys
import textwrap
import pytest
X_Y = f'{sys.version_info[0]}.{sys.version_info[1]}'
XY = f'{sys.version_info[0]}{sys.version_info[1]}'
# Handy environment variable you can use to run the tests
# with modified macros files. Multiple files should be
# separated by colon.
# You can use * if you escape it from your Shell:
# TESTED_FILES='macros.*' pytest -v
# Remember that some tests might need more macros files than just
# the local ones. You might need to use:
# TESTED_FILES='/usr/lib/rpm/macros:/usr/lib/rpm/platform/x86_64-linux/macros:macros.*'
TESTED_FILES = os.getenv("TESTED_FILES", None)
def rpm_eval(expression, fails=False, **kwargs):
cmd = ['rpmbuild']
if TESTED_FILES:
cmd += ['--macros', TESTED_FILES]
for var, value in kwargs.items():
if value is None:
cmd += ['--undefine', var]
else:
cmd += ['--define', f'{var} {value}']
cmd += ['--eval', expression]
cp = subprocess.run(cmd, text=True, env={**os.environ, 'LANG': 'C.utf-8'},
stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
if fails:
assert cp.returncode != 0, cp.stdout
elif fails is not None:
assert cp.returncode == 0, cp.stdout
return cp.stdout.strip().splitlines()
@pytest.fixture(scope="session")
def lib():
lib_eval = rpm_eval("%_lib")[0]
if lib_eval == "%_lib" and TESTED_FILES:
raise ValueError(
"%_lib is not resolved to an actual value. "
"You may want to include /usr/lib/rpm/platform/x86_64-linux/macros to TESTED_FILES."
)
return lib_eval
def get_alt_x_y():
"""
Some tests require alternate Python version to be installed.
In order to allow any Python version (or none at all),
this function/fixture exists.
You can control the behavior by setting the $ALTERNATE_PYTHON_VERSION
environment variable to X.Y (e.g. 3.6) or SKIP.
The environment variable must be set.
"""
env_name = "ALTERNATE_PYTHON_VERSION"
alternate_python_version = os.getenv(env_name, "")
if alternate_python_version.upper() == "SKIP":
pytest.skip(f"${env_name} set to SKIP")
if not alternate_python_version:
raise ValueError(f"${env_name} must be set, "
f"set it to SKIP if you want to skip tests that "
f"require alternate Python version.")
if not re.match(r"^\d+\.\d+$", alternate_python_version):
raise ValueError(f"${env_name} must be X.Y")
return alternate_python_version
def get_alt_xy():
"""
Same as get_alt_x_y() but without a dot
"""
return get_alt_x_y().replace(".", "")
# We don't use decorators, to be able to call the functions directly
alt_x_y = pytest.fixture(scope="session")(get_alt_x_y)
alt_xy = pytest.fixture(scope="session")(get_alt_xy)
# https://fedoraproject.org/wiki/Changes/PythonSafePath
def safe_path_flag(x_y):
return 'P' if tuple(int(i) for i in x_y.split('.')) >= (3, 11) else ''
def shell_stdout(script):
return subprocess.check_output(script,
env={**os.environ, 'LANG': 'C.utf-8'},
text=True,
shell=True).rstrip()
@pytest.mark.parametrize('macro', ['%__python3', '%python3'])
def test_python3(macro):
assert rpm_eval(macro) == ['/usr/bin/python3.11']
@pytest.mark.skip("Overriden in python3.11-rpm-macros")
@pytest.mark.parametrize('macro', ['%__python3', '%python3'])
@pytest.mark.parametrize('pkgversion', ['3', '3.9', '3.12'])
def test_python3_with_pkgversion(macro, pkgversion):
assert rpm_eval(macro, python3_pkgversion=pkgversion) == [f'/usr/bin/python{pkgversion}']
@pytest.mark.skip("py_dist_name uses the old canonicalization on RHEL 8")
@pytest.mark.parametrize('argument, result', [
('a', 'a'),
('a-a', 'a-a'),
('a_a', 'a-a'),
('a.a', 'a-a'),
('a---a', 'a-a'),
('a-_-a', 'a-a'),
('a-_-a', 'a-a'),
('a[b]', 'a[b]'),
('Aha[Boom]', 'aha[boom]'),
('a.a[b.b]', 'a-a[b-b]'),
])
def test_pydist_name(argument, result):
assert rpm_eval(f'%py_dist_name {argument}') == [result]
@pytest.mark.skip("py2_dist uses the old canonicalization on RHEL 8")
def test_py2_dist():
assert rpm_eval(f'%py2_dist Aha[Boom] a') == ['python2dist(aha[boom]) python2dist(a)']
@pytest.mark.skip("py3_dist uses the old canonicalization on RHEL 8")
def test_py3_dist():
assert rpm_eval(f'%py3_dist Aha[Boom] a') == ['python3dist(aha[boom]) python3dist(a)']
def test_py3_dist_with_python3_pkgversion_redefined(alt_x_y):
assert rpm_eval(f'%py3_dist Aha[Boom] a', python3_pkgversion=alt_x_y) == [f'python{alt_x_y}dist(aha[boom]) python{alt_x_y}dist(a)']
@pytest.mark.skip("python_provide is still enabled on RHEL 8")
def test_python_provide_python():
assert rpm_eval('%python_provide python-foo') == []
@pytest.mark.skip("python_provide does not work with this test when __python3 is overridden")
def test_python_provide_python3():
lines = rpm_eval('%python_provide python3-foo', version='6', release='1.fc66')
assert 'Obsoletes: python-foo < 6-1.fc66' in lines
assert 'Provides: python-foo = 6-1.fc66' in lines
assert f'Provides: python{X_Y}-foo = 6-1.fc66' in lines
assert len(lines) == 3
@pytest.mark.skip("python_provide does not work with this test when __python3 is overridden")
def test_python_provide_python3_epoched():
lines = rpm_eval('%python_provide python3-foo', epoch='1', version='6', release='1.fc66')
assert 'Obsoletes: python-foo < 1:6-1.fc66' in lines
assert 'Provides: python-foo = 1:6-1.fc66' in lines
assert f'Provides: python{X_Y}-foo = 1:6-1.fc66' in lines
assert len(lines) == 3
@pytest.mark.skip("python_provide does not work with this test when __python3 is overridden")
def test_python_provide_python3X():
lines = rpm_eval(f'%python_provide python{X_Y}-foo', version='6', release='1.fc66')
assert 'Obsoletes: python-foo < 6-1.fc66' in lines
assert 'Provides: python-foo = 6-1.fc66' in lines
assert 'Provides: python3-foo = 6-1.fc66' in lines
assert len(lines) == 3
@pytest.mark.skip("python_provide does not work with this test when __python3 is overridden")
def test_python_provide_python3X_epoched():
lines = rpm_eval(f'%python_provide python{X_Y}-foo', epoch='1', version='6', release='1.fc66')
assert 'Obsoletes: python-foo < 1:6-1.fc66' in lines
assert 'Provides: python-foo = 1:6-1.fc66' in lines
assert 'Provides: python3-foo = 1:6-1.fc66' in lines
assert len(lines) == 3
@pytest.mark.skip("python_provide does not work with this test when __python3 is overridden")
def test_python_provide_doubleuse():
lines = rpm_eval('%{python_provide python3-foo}%{python_provide python3-foo}',
version='6', release='1.fc66')
assert 'Obsoletes: python-foo < 6-1.fc66' in lines
assert 'Provides: python-foo = 6-1.fc66' in lines
assert f'Provides: python{X_Y}-foo = 6-1.fc66' in lines
assert len(lines) == 6
assert len(set(lines)) == 3
@pytest.mark.parametrize('rhel', [None, 10])
def test_py_provides_python(rhel):
lines = rpm_eval('%py_provides python-foo', version='6', release='1.fc66', rhel=rhel)
assert 'Provides: python-foo = 6-1.fc66' in lines
assert len(lines) == 1
@pytest.mark.parametrize('rhel', [None, 12])
def test_py_provides_whatever(rhel):
lines = rpm_eval('%py_provides whatever', version='6', release='1.fc66', rhel=rhel)
assert 'Provides: whatever = 6-1.fc66' in lines
assert len(lines) == 1
@pytest.mark.skip("py_provides behaves differently for alternative Python stacks")
@pytest.mark.parametrize('rhel', [None, 9])
def test_py_provides_python3(rhel):
lines = rpm_eval('%py_provides python3-foo', version='6', release='1.fc66', rhel=rhel)
assert 'Provides: python3-foo = 6-1.fc66' in lines
assert 'Provides: python-foo = 6-1.fc66' in lines
assert f'Provides: python{X_Y}-foo = 6-1.fc66' in lines
if rhel:
assert f'Obsoletes: python{X_Y}-foo < 6-1.fc66' in lines
assert len(lines) == 4
else:
assert len(lines) == 3
@pytest.mark.skip("py_provides behaves differently for alternative Python stacks")
@pytest.mark.parametrize('rhel', [None, 9])
def test_py_provides_python3_with_isa(rhel):
lines = rpm_eval('%py_provides python3-foo(x86_64)', version='6', release='1.fc66', rhel=rhel)
assert 'Provides: python3-foo(x86_64) = 6-1.fc66' in lines
assert 'Provides: python-foo(x86_64) = 6-1.fc66' in lines
assert f'Provides: python{X_Y}-foo(x86_64) = 6-1.fc66' in lines
assert f'Obsoletes: python{X_Y}-foo(x86_64) < 6-1.fc66' not in lines
assert len(lines) == 3
@pytest.mark.skip("py_provides behaves differently for alternative Python stacks")
@pytest.mark.parametrize('rhel', [None, 13])
def test_py_provides_python3_epoched(rhel):
lines = rpm_eval('%py_provides python3-foo', epoch='1', version='6', release='1.fc66', rhel=rhel)
assert 'Provides: python3-foo = 1:6-1.fc66' in lines
assert 'Provides: python-foo = 1:6-1.fc66' in lines
assert f'Provides: python{X_Y}-foo = 1:6-1.fc66' in lines
if rhel:
assert f'Obsoletes: python{X_Y}-foo < 1:6-1.fc66' in lines
assert len(lines) == 4
else:
assert len(lines) == 3
@pytest.mark.skip("py_provides behaves differently for alternative Python stacks")
@pytest.mark.parametrize('rhel', [None, 13])
def test_py_provides_python3X(rhel):
lines = rpm_eval(f'%py_provides python{X_Y}-foo', version='6', release='1.fc66', rhel=rhel)
assert f'Provides: python{X_Y}-foo = 6-1.fc66' in lines
assert 'Provides: python-foo = 6-1.fc66' in lines
assert 'Provides: python3-foo = 6-1.fc66' in lines
assert len(lines) == 3
@pytest.mark.skip("py_provides behaves differently for alternative Python stacks")
@pytest.mark.parametrize('rhel', [None, 27])
def test_py_provides_python3X_epoched(rhel):
lines = rpm_eval(f'%py_provides python{X_Y}-foo', epoch='1', version='6', release='1.fc66', rhel=rhel)
assert f'Provides: python{X_Y}-foo = 1:6-1.fc66' in lines
assert 'Provides: python-foo = 1:6-1.fc66' in lines
assert 'Provides: python3-foo = 1:6-1.fc66' in lines
assert len(lines) == 3
@pytest.mark.skip("py_provides behaves differently for alternative Python stacks")
@pytest.mark.parametrize('rhel', [None, 2])
def test_py_provides_doubleuse(rhel):
lines = rpm_eval('%{py_provides python3-foo}%{py_provides python3-foo}',
version='6', release='1.fc66', rhel=rhel)
assert 'Provides: python3-foo = 6-1.fc66' in lines
assert 'Provides: python-foo = 6-1.fc66' in lines
assert f'Provides: python{X_Y}-foo = 6-1.fc66' in lines
if rhel:
assert f'Obsoletes: python{X_Y}-foo < 6-1.fc66' in lines
assert len(lines) == 8
assert len(set(lines)) == 4
else:
assert len(lines) == 6
assert len(set(lines)) == 3
@pytest.mark.skip("py_provides behaves differently for alternative Python stacks")
@pytest.mark.parametrize('rhel', [None, 2])
def test_py_provides_with_evr(rhel):
lines = rpm_eval('%py_provides python3-foo 123',
version='6', release='1.fc66', rhel=rhel)
assert 'Provides: python3-foo = 123' in lines
assert 'Provides: python-foo = 123' in lines
assert f'Provides: python{X_Y}-foo = 123' in lines
if rhel:
assert f'Obsoletes: python{X_Y}-foo < 123' in lines
assert len(lines) == 4
else:
assert len(lines) == 3
def test_python_wheel_pkg_prefix():
assert rpm_eval('%python_wheel_pkg_prefix', fedora='44', rhel=None, eln=None) == ['python']
assert rpm_eval('%python_wheel_pkg_prefix', fedora='44', rhel=None, eln=None, python3_pkgversion='3.9') == ['python']
assert rpm_eval('%python_wheel_pkg_prefix', fedora=None, rhel='1', eln='1') == ['python']
assert rpm_eval('%python_wheel_pkg_prefix', fedora=None, rhel='1', eln=None) == ['python3.11']
assert rpm_eval('%python_wheel_pkg_prefix', fedora=None, rhel='1', eln=None, python3_pkgversion='3.10') == ['python3.10']
assert rpm_eval('%python_wheel_pkg_prefix', fedora=None, rhel='1', eln=None, python3_pkgversion='3.11') == ['python3.11']
def test_python_wheel_dir():
assert rpm_eval('%python_wheel_dir', fedora='44', rhel=None, eln=None) == ['/usr/share/python-wheels']
assert rpm_eval('%python_wheel_dir', fedora='44', rhel=None, eln=None, python3_pkgversion='3.9') == ['/usr/share/python-wheels']
assert rpm_eval('%python_wheel_dir', fedora=None, rhel='1', eln='1') == ['/usr/share/python-wheels']
assert rpm_eval('%python_wheel_dir', fedora=None, rhel='1', eln=None) == ['/usr/share/python3.11-wheels']
assert rpm_eval('%python_wheel_dir', fedora=None, rhel='1', eln=None, python3_pkgversion='3.10') == ['/usr/share/python3.10-wheels']
assert rpm_eval('%python_wheel_dir', fedora=None, rhel='1', eln=None, python3_pkgversion='3.11') == ['/usr/share/python3.11-wheels']
def test_pytest_passes_options_naturally():
lines = rpm_eval('%pytest -k foo')
assert '/usr/bin/pytest-3.11 -k foo' in lines[-1]
def test_pytest_different_command():
lines = rpm_eval('%pytest', __pytest='pytest-3')
assert 'pytest-3' in lines[-1]
def test_pytest_command_suffix():
lines = rpm_eval('%pytest -v')
assert '/usr/bin/pytest-3.11 -v' in lines[-1]
# this test does not require alternate Pythons to be installed
@pytest.mark.parametrize('version', ['3.6', '3.7', '3.12'])
def test_pytest_command_suffix_alternate_pkgversion(version):
lines = rpm_eval('%pytest -v', python3_pkgversion=version, python3_version=version)
assert f'/usr/bin/pytest-{version} -v' in lines[-1]
@pytest.mark.skip("This functionality is not present in RHEL 8")
def test_pytest_sets_pytest_xdist_auto_num_workers():
lines = rpm_eval('%pytest', _smp_build_ncpus=2)
assert 'PYTEST_XDIST_AUTO_NUM_WORKERS=2' in '\n'.join(lines)
def test_pytest_undefined_addopts_are_not_set():
lines = rpm_eval('%pytest', __pytest_addopts=None)
assert 'PYTEST_ADDOPTS' not in '\n'.join(lines)
def test_pytest_defined_addopts_are_set():
lines = rpm_eval('%pytest', __pytest_addopts="--ignore=stuff")
assert 'PYTEST_ADDOPTS="${PYTEST_ADDOPTS:-} --ignore=stuff"' in '\n'.join(lines)
@pytest.mark.parametrize('__pytest_addopts', ['--macronized-option', 'x y z', None])
def test_pytest_addopts_preserves_envvar(__pytest_addopts):
# this is the line a packager might put in the spec file before running %pytest:
spec_line = 'export PYTEST_ADDOPTS="--exported-option1 --exported-option2"'
# instead of actually running /usr/bin/pytest,
# we run a small shell script that echoes the tested value for inspection
lines = rpm_eval('%pytest', __pytest_addopts=__pytest_addopts,
__pytest="sh -c 'echo $PYTEST_ADDOPTS'")
echoed = shell_stdout('\n'.join([spec_line] + lines))
# assert all values were echoed
assert '--exported-option1' in echoed
assert '--exported-option2' in echoed
if __pytest_addopts is not None:
assert __pytest_addopts in echoed
# assert the options are separated
assert 'option--' not in echoed
assert 'z--' not in echoed
@pytest.mark.skip("RHEL 8 predates py3_test_envvars")
@pytest.mark.parametrize('__pytest_addopts', ['-X', None])
def test_py3_test_envvars(lib, __pytest_addopts):
lines = rpm_eval('%{py3_test_envvars}\\\n%{python3} -m unittest',
buildroot='BUILDROOT',
_smp_build_ncpus='3',
__pytest_addopts=__pytest_addopts)
assert all(l.endswith('\\') for l in lines[:-1])
stripped_lines = [l.strip(' \\') for l in lines]
sitearch = f'BUILDROOT/usr/{lib}/python{X_Y}/site-packages'
sitelib = f'BUILDROOT/usr/lib/python{X_Y}/site-packages'
assert f'PYTHONPATH="${{PYTHONPATH:-{sitearch}:{sitelib}}}"' in stripped_lines
assert 'PATH="BUILDROOT/usr/bin:$PATH"' in stripped_lines
assert 'CFLAGS="${CFLAGS:-${RPM_OPT_FLAGS}}" LDFLAGS="${LDFLAGS:-${RPM_LD_FLAGS}}"' in stripped_lines
assert 'PYTHONDONTWRITEBYTECODE=1' in stripped_lines
assert 'PYTEST_XDIST_AUTO_NUM_WORKERS=3' in stripped_lines
if __pytest_addopts:
assert f'PYTEST_ADDOPTS="${{PYTEST_ADDOPTS:-}} {__pytest_addopts}"' in stripped_lines
else:
assert 'PYTEST_ADDOPTS' not in ''.join(lines)
assert stripped_lines[-1] == '/usr/bin/python3.11 -m unittest'
def test_pypi_source_default_name():
urls = rpm_eval('%pypi_source',
name='foo', version='6')
assert urls == ['https://files.pythonhosted.org/packages/source/f/foo/foo-6.tar.gz']
def test_pypi_source_default_srcname():
urls = rpm_eval('%pypi_source',
name='python-foo', srcname='foo', version='6')
assert urls == ['https://files.pythonhosted.org/packages/source/f/foo/foo-6.tar.gz']
def test_pypi_source_default_pypi_name():
urls = rpm_eval('%pypi_source',
name='python-foo', pypi_name='foo', version='6')
assert urls == ['https://files.pythonhosted.org/packages/source/f/foo/foo-6.tar.gz']
def test_pypi_source_default_name_uppercase():
urls = rpm_eval('%pypi_source',
name='Foo', version='6')
assert urls == ['https://files.pythonhosted.org/packages/source/F/Foo/Foo-6.tar.gz']
def test_pypi_source_provided_name():
urls = rpm_eval('%pypi_source foo',
name='python-bar', pypi_name='bar', version='6')
assert urls == ['https://files.pythonhosted.org/packages/source/f/foo/foo-6.tar.gz']
def test_pypi_source_provided_name_version():
urls = rpm_eval('%pypi_source foo 6',
name='python-bar', pypi_name='bar', version='3')
assert urls == ['https://files.pythonhosted.org/packages/source/f/foo/foo-6.tar.gz']
def test_pypi_source_provided_name_version_ext():
url = rpm_eval('%pypi_source foo 6 zip',
name='python-bar', pypi_name='bar', version='3')
assert url == ['https://files.pythonhosted.org/packages/source/f/foo/foo-6.zip']
def test_pypi_source_prerelease():
urls = rpm_eval('%pypi_source',
name='python-foo', pypi_name='foo', version='6~b2')
assert urls == ['https://files.pythonhosted.org/packages/source/f/foo/foo-6b2.tar.gz']
def test_pypi_source_explicit_tilde():
urls = rpm_eval('%pypi_source foo 6~6',
name='python-foo', pypi_name='foo', version='6')
assert urls == ['https://files.pythonhosted.org/packages/source/f/foo/foo-6~6.tar.gz']
@pytest.mark.skip("%py3_shebang_fix has a complicated evaluation in RHEL 8 due to differences between Python stacks")
def test_py3_shebang_fix():
cmd = rpm_eval('%py3_shebang_fix arg1 arg2 arg3')[-1].strip()
assert cmd == '/usr/bin/python3.11 -B /usr/lib/rpm/redhat/pathfix.py -pni /usr/bin/python3.11 $shebang_flags arg1 arg2 arg3'
def test_py3_shebang_fix_default_shebang_flags():
lines = rpm_eval('%py3_shebang_fix arg1 arg2')
lines[-1] = 'echo $shebang_flags'
assert shell_stdout('\n'.join(lines)) == f'-kas{safe_path_flag(X_Y)}'
def test_py3_shebang_fix_custom_shebang_flags():
lines = rpm_eval('%py3_shebang_fix arg1 arg2', py3_shebang_flags='Es')
lines[-1] = 'echo $shebang_flags'
assert shell_stdout('\n'.join(lines)) == '-kaEs'
@pytest.mark.parametrize('_py3_shebang_s', [None, '%{nil}'])
def test_py3_shebang_fix_undefined_py3_shebang_s(_py3_shebang_s):
lines = rpm_eval('%py3_shebang_fix arg1 arg2', _py3_shebang_s=_py3_shebang_s)
lines[-1] = 'echo $shebang_flags'
expected = f'-ka{safe_path_flag(X_Y)}' if safe_path_flag(X_Y) else '-k'
assert shell_stdout('\n'.join(lines)) == expected
@pytest.mark.parametrize('_py3_shebang_P', [None, '%{nil}'])
def test_py3_shebang_fix_undefined_py3_shebang_P(_py3_shebang_P):
lines = rpm_eval('%py3_shebang_fix arg1 arg2', _py3_shebang_P=_py3_shebang_P)
lines[-1] = 'echo $shebang_flags'
assert shell_stdout('\n'.join(lines)) == '-kas'
@pytest.mark.parametrize('_py3_shebang_s', [None, '%{nil}'])
@pytest.mark.parametrize('_py3_shebang_P', [None, '%{nil}'])
def test_py3_shebang_fix_undefined_py3_shebang_sP(_py3_shebang_s, _py3_shebang_P):
lines = rpm_eval('%py3_shebang_fix arg1 arg2',
_py3_shebang_s=_py3_shebang_s,
_py3_shebang_P=_py3_shebang_P)
lines[-1] = 'echo $shebang_flags'
assert shell_stdout('\n'.join(lines)) == '-k'
@pytest.mark.parametrize('flags', [None, '%{nil}'])
def test_py3_shebang_fix_no_shebang_flags(flags):
lines = rpm_eval('%py3_shebang_fix arg1 arg2', py3_shebang_flags=flags)
lines[-1] = 'echo $shebang_flags'
assert shell_stdout('\n'.join(lines)) == '-k'
@pytest.mark.skip("%py_shebang_fix has a complicated evaluation in RHEL 8 due to differences between Python stacks")
def test_py_shebang_fix_custom_python():
cmd = rpm_eval('%py_shebang_fix arg1 arg2 arg3', __python='/usr/bin/pypy')[-1].strip()
assert cmd == '/usr/bin/pypy -B /usr/lib/rpm/redhat/pathfix.py -pni /usr/bin/pypy $shebang_flags arg1 arg2 arg3'
def test_pycached_in_sitelib():
lines = rpm_eval('%pycached %{python3_sitelib}/foo*.py')
assert lines == [
f'/usr/lib/python{X_Y}/site-packages/foo*.py',
f'/usr/lib/python{X_Y}/site-packages/__pycache__/foo*.cpython-{XY}{{,.opt-?}}.pyc'
]
def test_pycached_in_sitearch(lib):
lines = rpm_eval('%pycached %{python3_sitearch}/foo*.py')
assert lines == [
f'/usr/{lib}/python{X_Y}/site-packages/foo*.py',
f'/usr/{lib}/python{X_Y}/site-packages/__pycache__/foo*.cpython-{XY}{{,.opt-?}}.pyc'
]
# this test does not require alternate Pythons to be installed
@pytest.mark.parametrize('version', ['3.6', '3.7', '3.12'])
def test_pycached_with_alternate_version(version):
version_nodot = version.replace('.', '')
lines = rpm_eval(f'%pycached /usr/lib/python{version}/site-packages/foo*.py')
assert lines == [
f'/usr/lib/python{version}/site-packages/foo*.py',
f'/usr/lib/python{version}/site-packages/__pycache__/foo*.cpython-{version_nodot}{{,.opt-?}}.pyc'
]
def test_pycached_in_custom_dir():
lines = rpm_eval('%pycached /bar/foo*.py')
assert lines == [
'/bar/foo*.py',
'/bar/__pycache__/foo*.cpython-3*{,.opt-?}.pyc'
]
def test_pycached_with_exclude():
lines = rpm_eval('%pycached %exclude %{python3_sitelib}/foo*.py')
assert lines == [
f'%exclude /usr/lib/python{X_Y}/site-packages/foo*.py',
f'%exclude /usr/lib/python{X_Y}/site-packages/__pycache__/foo*.cpython-{XY}{{,.opt-?}}.pyc'
]
def test_pycached_fails_with_extension_glob():
lines = rpm_eval('%pycached %{python3_sitelib}/foo.py*', fails=True)
assert lines[0] == 'error: %pycached can only be used with paths explicitly ending with .py'
@pytest.mark.skip("Python extras subpackages are not handled in RHEL 8")
def test_python_extras_subpkg_i():
lines = rpm_eval('%python_extras_subpkg -n python3-setuptools_scm -i %{python3_sitelib}/*.egg-info toml yaml',
version='6', release='7')
expected = textwrap.dedent(f"""
%package -n python3-setuptools_scm+toml
Summary: Metapackage for python3-setuptools_scm: toml extras
Requires: python3-setuptools_scm = 6-7
%description -n python3-setuptools_scm+toml
This is a metapackage bringing in toml extras requires for
python3-setuptools_scm.
It makes sure the dependencies are installed.
%files -n python3-setuptools_scm+toml
%ghost /usr/lib/python{X_Y}/site-packages/*.egg-info
%package -n python3-setuptools_scm+yaml
Summary: Metapackage for python3-setuptools_scm: yaml extras
Requires: python3-setuptools_scm = 6-7
%description -n python3-setuptools_scm+yaml
This is a metapackage bringing in yaml extras requires for
python3-setuptools_scm.
It makes sure the dependencies are installed.
%files -n python3-setuptools_scm+yaml
%ghost /usr/lib/python{X_Y}/site-packages/*.egg-info
""").lstrip().splitlines()
assert lines == expected
@pytest.mark.skip("Python extras subpackages are not handled in RHEL 8")
def test_python_extras_subpkg_f():
lines = rpm_eval('%python_extras_subpkg -n python3-setuptools_scm -f ghost_filelist toml yaml',
version='6', release='7')
expected = textwrap.dedent(f"""
%package -n python3-setuptools_scm+toml
Summary: Metapackage for python3-setuptools_scm: toml extras
Requires: python3-setuptools_scm = 6-7
%description -n python3-setuptools_scm+toml
This is a metapackage bringing in toml extras requires for
python3-setuptools_scm.
It makes sure the dependencies are installed.
%files -n python3-setuptools_scm+toml -f ghost_filelist
%package -n python3-setuptools_scm+yaml
Summary: Metapackage for python3-setuptools_scm: yaml extras
Requires: python3-setuptools_scm = 6-7
%description -n python3-setuptools_scm+yaml
This is a metapackage bringing in yaml extras requires for
python3-setuptools_scm.
It makes sure the dependencies are installed.
%files -n python3-setuptools_scm+yaml -f ghost_filelist
""").lstrip().splitlines()
assert lines == expected
@pytest.mark.skip("Python extras subpackages are not handled in RHEL 8")
def test_python_extras_subpkg_F():
lines = rpm_eval('%python_extras_subpkg -n python3-setuptools_scm -F toml yaml',
version='6', release='7')
expected = textwrap.dedent(f"""
%package -n python3-setuptools_scm+toml
Summary: Metapackage for python3-setuptools_scm: toml extras
Requires: python3-setuptools_scm = 6-7
%description -n python3-setuptools_scm+toml
This is a metapackage bringing in toml extras requires for
python3-setuptools_scm.
It makes sure the dependencies are installed.
%package -n python3-setuptools_scm+yaml
Summary: Metapackage for python3-setuptools_scm: yaml extras
Requires: python3-setuptools_scm = 6-7
%description -n python3-setuptools_scm+yaml
This is a metapackage bringing in yaml extras requires for
python3-setuptools_scm.
It makes sure the dependencies are installed.
""").lstrip().splitlines()
assert lines == expected
@pytest.mark.skip("Python extras subpackages are not handled in RHEL 8")
def test_python_extras_subpkg_underscores():
lines = rpm_eval('%python_extras_subpkg -n python3-webscrapbook -F adhoc_ssl',
version='0.33.3', release='1.fc33')
expected = textwrap.dedent(f"""
%package -n python3-webscrapbook+adhoc_ssl
Summary: Metapackage for python3-webscrapbook: adhoc_ssl extras
Requires: python3-webscrapbook = 0.33.3-1.fc33
%description -n python3-webscrapbook+adhoc_ssl
This is a metapackage bringing in adhoc_ssl extras requires for
python3-webscrapbook.
It makes sure the dependencies are installed.
""").lstrip().splitlines()
assert lines == expected
@pytest.mark.skip("Python extras subpackages are not handled in RHEL 8")
@pytest.mark.parametrize('sep', [pytest.param(('', ' ', ' ', ''), id='spaces'),
pytest.param(('', ',', ',', ''), id='commas'),
pytest.param(('', ',', ',', ','), id='commas-trailing'),
pytest.param((',', ',', ',', ''), id='commas-leading'),
pytest.param((',', ',', ',', ','), id='commas-trailing-leading'),
pytest.param(('', ',', ' ', ''), id='mixture'),
pytest.param((' ', ' ', '\t\t, ', '\t'), id='chaotic-good'),
pytest.param(('', '\t ,, \t\r ', ',,\t , ', ',,'), id='chaotic-evil')])
def test_python_extras_subpkg_arg_separators(sep):
lines = rpm_eval('%python_extras_subpkg -n python3-hypothesis -F {}cli{}ghostwriter{}pytz{}'.format(*sep),
version='6.6.0', release='1.fc35')
expected = textwrap.dedent(f"""
%package -n python3-hypothesis+cli
Summary: Metapackage for python3-hypothesis: cli extras
Requires: python3-hypothesis = 6.6.0-1.fc35
%description -n python3-hypothesis+cli
This is a metapackage bringing in cli extras requires for python3-hypothesis.
It makes sure the dependencies are installed.
%package -n python3-hypothesis+ghostwriter
Summary: Metapackage for python3-hypothesis: ghostwriter extras
Requires: python3-hypothesis = 6.6.0-1.fc35
%description -n python3-hypothesis+ghostwriter
This is a metapackage bringing in ghostwriter extras requires for
python3-hypothesis.
It makes sure the dependencies are installed.
%package -n python3-hypothesis+pytz
Summary: Metapackage for python3-hypothesis: pytz extras
Requires: python3-hypothesis = 6.6.0-1.fc35
%description -n python3-hypothesis+pytz
This is a metapackage bringing in pytz extras requires for python3-hypothesis.
It makes sure the dependencies are installed.
""").lstrip().splitlines()
assert lines == expected
@pytest.mark.skip("Python extras subpackages are not handled in RHEL 8")
@pytest.mark.parametrize('basename_len', [1, 10, 30, 45, 78])
@pytest.mark.parametrize('extra_len', [1, 13, 28, 52, 78])
def test_python_extras_subpkg_description_wrapping(basename_len, extra_len):
basename = 'x' * basename_len
extra = 'y' * extra_len
lines = rpm_eval(f'%python_extras_subpkg -n {basename} -F {extra}',
version='6', release='7')
for idx, line in enumerate(lines):
if line.startswith('%description'):
start = idx + 1
lines = lines[start:]
assert all(len(l) < 80 for l in lines)
assert len(lines) < 6
if len(" ".join(lines[:-1])) < 80:
assert len(lines) == 2
expected_singleline = (f"This is a metapackage bringing in {extra} extras "
f"requires for {basename}. "
f"It makes sure the dependencies are installed.")
description_singleline = " ".join(lines)
assert description_singleline == expected_singleline
unversioned_macros = pytest.mark.parametrize('macro', [
'%__python',
'%python',
'%python_version',
'%python_version_nodots',
'%python_sitelib',
'%python_sitearch',
'%python_platform',
'%python_platform_triplet',
'%python_ext_suffix',
'%python_cache_tag',
'%py_shebang_fix',
'%py_build',
'%py_build_egg',
'%py_build_wheel',
'%py_install',
'%py_install_egg',
'%py_install_wheel',
'%py_check_import',
'%py_test_envvars',
])
@pytest.mark.skip("Not applicable for python3.11-rpm-macros")
@unversioned_macros
def test_unversioned_python_errors(macro):
lines = rpm_eval(macro, fails=True)
assert lines[0] == (
'error: attempt to use unversioned python, '
'define %__python to /usr/bin/python2 or /usr/bin/python3 explicitly'
)
# when the macros are %global, the error is longer
# we deliberately allow this extra line to be optional
if len(lines) > 1:
# the failed macro is not unnecessarily our tested macro
pattern = r'error: Macro %\S+ failed to expand'
assert re.match(pattern, lines[1])
# but there should be no more lines
assert len(lines) < 3
@pytest.mark.skip("Not applicable for python3.11-rpm-macros")
@unversioned_macros
def test_unversioned_python_works_when_defined(macro):
macro3 = macro.replace('python', 'python3').replace('py_', 'py3_')
assert rpm_eval(macro, __python='/usr/bin/python3.11') == rpm_eval(macro3)
# we could rework the test for multiple architectures, but the Fedora CI currently only runs on x86_64
x86_64_only = pytest.mark.skipif(platform.machine() != "x86_64", reason="works on x86_64 only")
@x86_64_only
def test_platform_triplet():
assert rpm_eval("%python3_platform_triplet") == ["x86_64-linux-gnu"]
@x86_64_only
def test_ext_suffix():
assert rpm_eval("%python3_ext_suffix") == [f".cpython-{XY}-x86_64-linux-gnu.so"]
def test_cache_tag():
assert rpm_eval("%python3_cache_tag") == [f"cpython-{XY}"]
def test_cache_tag_alternate_python(alt_x_y, alt_xy):
assert rpm_eval("%python_cache_tag", __python=f"/usr/bin/python{alt_x_y}") == [f"cpython-{alt_xy}"]
def test_cache_tag_alternate_python3(alt_x_y, alt_xy):
assert rpm_eval("%python3_cache_tag", __python3=f"/usr/bin/python{alt_x_y}") == [f"cpython-{alt_xy}"]
@pytest.mark.skip("Throws DeprecationWarning when run with Python 3.11, expected")
def test_python_sitelib_value_python3():
macro = '%python_sitelib'
assert rpm_eval(macro, __python='%__python3') == [f'/usr/lib/python{X_Y}/site-packages']
def test_python_sitelib_value_alternate_python(alt_x_y):
macro = '%python_sitelib'
assert rpm_eval(macro, __python=f'/usr/bin/python{alt_x_y}') == [f'/usr/lib/python{alt_x_y}/site-packages']
def test_python3_sitelib_value_default():
macro = '%python3_sitelib'
assert rpm_eval(macro) == [f'/usr/lib/python{X_Y}/site-packages']
def test_python3_sitelib_value_alternate_python(alt_x_y):
macro = '%python3_sitelib'
assert (rpm_eval(macro, __python3=f'/usr/bin/python{alt_x_y}') ==
rpm_eval(macro, python3_pkgversion=alt_x_y) ==
[f'/usr/lib/python{alt_x_y}/site-packages'])
def test_python3_sitelib_value_alternate_prefix():
macro = '%python3_sitelib'
assert rpm_eval(macro, _prefix='/app') == [f'/app/lib/python{X_Y}/site-packages']
@pytest.mark.skip("Throws DeprecationWarning when run with Python 3.11, expected")
def test_python_sitearch_value_python3(lib):
macro = '%python_sitearch'
assert rpm_eval(macro, __python='%__python3') == [f'/usr/{lib}/python{X_Y}/site-packages']
def test_python_sitearch_value_alternate_python(lib, alt_x_y):
macro = '%python_sitearch'
assert rpm_eval(macro, __python=f'/usr/bin/python{alt_x_y}') == [f'/usr/{lib}/python{alt_x_y}/site-packages']
def test_python3_sitearch_value_default(lib):
macro = '%python3_sitearch'
assert rpm_eval(macro) == [f'/usr/{lib}/python{X_Y}/site-packages']
def test_python3_sitearch_value_alternate_python(lib, alt_x_y):
macro = '%python3_sitearch'
assert (rpm_eval(macro, __python3=f'/usr/bin/python{alt_x_y}') ==
rpm_eval(macro, python3_pkgversion=alt_x_y) ==
[f'/usr/{lib}/python{alt_x_y}/site-packages'])
def test_python3_sitearch_value_alternate_prefix(lib):
macro = '%python3_sitearch'
assert rpm_eval(macro, _prefix='/app') == [f'/app/{lib}/python{X_Y}/site-packages']
@pytest.mark.parametrize(
'args, expected_args',
[
('six', 'six'),
('-f foo.txt', '-f foo.txt'),
('-t -f foo.txt six, seven', '-t -f foo.txt six, seven'),
('-e "foo*" -f foo.txt six, seven', '-e "foo*" -f foo.txt six, seven'),
('six.quarter six.half,, SIX', 'six.quarter six.half,, SIX'),
('-f foo.txt six\nsix.half\nSIX', '-f foo.txt six six.half SIX'),
('six \\ -e six.half', 'six -e six.half'),
]
)
@pytest.mark.parametrize('__python3',
[None,
f'/usr/bin/python{X_Y}',
'/usr/bin/pythonX.Y'])
def test_py3_check_import(args, expected_args, __python3, lib):
x_y = X_Y
macros = {
'buildroot': 'BUILDROOT',
'_rpmconfigdir': 'RPMCONFIGDIR',
}
if __python3 is not None:
if 'X.Y' in __python3:
__python3 = __python3.replace('X.Y', get_alt_x_y())
macros['__python3'] = __python3
# If the __python3 command has version at the end, parse it and expect it.
# Note that the command is used to determine %python3_sitelib and %python3_sitearch,
# so we only test known CPython schemes here and not PyPy for simplicity.
if (match := re.match(r'.+python(\d+\.\d+)$', __python3)):
x_y = match.group(1)
invocation = '%{py3_check_import ' + args +'}'
lines = rpm_eval(invocation, **macros)
# An equality check is a bit inflexible here,
# every time we change the macro we need to change this test.
# However actually executing it and verifying the result is much harder :/
# At least, let's make the lines saner to check:
lines = [line.rstrip('\\').strip() for line in lines]
expected = textwrap.dedent(fr"""
PATH="BUILDROOT/usr/bin:$PATH"
PYTHONPATH="${{PYTHONPATH:-BUILDROOT/usr/{lib}/python{x_y}/site-packages:BUILDROOT/usr/lib/python{x_y}/site-packages}}"
_PYTHONSITE="BUILDROOT/usr/{lib}/python{x_y}/site-packages:BUILDROOT/usr/lib/python{x_y}/site-packages"
PYTHONDONTWRITEBYTECODE=1
{__python3 or '/usr/bin/python3.11'} -s{safe_path_flag(x_y)} RPMCONFIGDIR/redhat/import_all_modules_py3_11.py {expected_args}
""")
assert lines == expected.splitlines()
@pytest.mark.parametrize(
'shebang_flags_value, expected_shebang_flags',
[
('sP', '-sP'),
('s', '-s'),
('%{nil}', ''),
(None, ''),
('Es', '-Es'),
]
)
def test_py3_check_import_respects_shebang_flags(shebang_flags_value, expected_shebang_flags, lib):
macros = {
'_rpmconfigdir': 'RPMCONFIGDIR',
'__python3': '/usr/bin/python3',
'py3_shebang_flags': shebang_flags_value,
}
lines = rpm_eval('%py3_check_import sys', **macros)
# Compare the last line of the command, that's where lua part is evaluated
expected = f'/usr/bin/python3 {expected_shebang_flags} RPMCONFIGDIR/redhat/import_all_modules_py3_11.py sys'
assert lines[-1].strip() == expected

View File

@ -0,0 +1,426 @@
from import_all_modules_py3_11 import argparser, exclude_unwanted_module_globs
from import_all_modules_py3_11 import main as modules_main
from import_all_modules_py3_11 import read_modules_from_cli, filter_top_level_modules_only
from pathlib import Path
import pytest
import shlex
import sys
@pytest.fixture(autouse=True)
def preserve_sys_path():
original_sys_path = list(sys.path)
yield
sys.path = original_sys_path
@pytest.fixture(autouse=True)
def preserve_sys_modules():
original_sys_modules = dict(sys.modules)
yield
sys.modules = original_sys_modules
@pytest.mark.parametrize(
'args, imports',
[
('six', ['six']),
('five six seven', ['five', 'six', 'seven']),
('six,seven, eight', ['six', 'seven', 'eight']),
('six.quarter six.half,, SIX', ['six.quarter', 'six.half', 'SIX']),
('six.quarter six.half,, SIX \\ ', ['six.quarter', 'six.half', 'SIX']),
]
)
def test_read_modules_from_cli(args, imports):
argv = shlex.split(args)
cli_args = argparser().parse_args(argv)
assert read_modules_from_cli(cli_args.modules) == imports
@pytest.mark.parametrize(
'all_mods, imports',
[
(['six'], ['six']),
(['five', 'six', 'seven'], ['five', 'six', 'seven']),
(['six.seven', 'eight'], ['eight']),
(['SIX', 'six.quarter', 'six.half.and.sth', 'seven'], ['SIX', 'seven']),
],
)
def test_filter_top_level_modules_only(all_mods, imports):
assert filter_top_level_modules_only(all_mods) == imports
@pytest.mark.parametrize(
'globs, expected',
[
(['*.*'], ['foo', 'boo']),
(['?oo'], ['foo.bar', 'foo.bar.baz', 'foo.baz']),
(['*.baz'], ['foo', 'foo.bar', 'boo']),
(['foo'], ['foo.bar', 'foo.bar.baz', 'foo.baz', 'boo']),
(['foo*'], ['boo']),
(['foo*', '*bar'], ['boo']),
(['foo', 'bar'], ['foo.bar', 'foo.bar.baz', 'foo.baz', 'boo']),
(['*'], []),
]
)
def test_exclude_unwanted_module_globs(globs, expected):
my_modules = ['foo', 'foo.bar', 'foo.bar.baz', 'foo.baz', 'boo']
tested = exclude_unwanted_module_globs(globs, my_modules)
assert tested == expected
def test_cli_with_all_args():
'''A smoke test, all args must be parsed correctly.'''
mods = ['foo', 'foo.bar', 'baz']
files = ['-f', './foo']
top = ['-t']
exclude = ['-e', 'foo*']
cli_args = argparser().parse_args([*mods, *files, *top, *exclude])
assert cli_args.filename == [Path('foo')]
assert cli_args.top_level is True
assert cli_args.modules == ['foo', 'foo.bar', 'baz']
assert cli_args.exclude == ['foo*']
def test_cli_without_filename_toplevel():
'''Modules provided on command line (without files) must be parsed correctly.'''
mods = ['foo', 'foo.bar', 'baz']
cli_args = argparser().parse_args(mods)
assert cli_args.filename is None
assert cli_args.top_level is False
assert cli_args.modules == ['foo', 'foo.bar', 'baz']
def test_cli_with_filename_no_cli_mods():
'''Files (without any modules provided on command line) must be parsed correctly.'''
files = ['-f', './foo', '-f', './bar', '-f', './baz']
cli_args = argparser().parse_args(files)
assert cli_args.filename == [Path('foo'), Path('./bar'), Path('./baz')]
assert not cli_args.top_level
def test_main_raises_error_when_no_modules_provided():
'''If no filename nor modules were provided, ValueError is raised.'''
with pytest.raises(ValueError):
modules_main([])
def test_import_all_modules_does_not_import():
'''Ensure the files from /usr/lib/rpm/redhat cannot be imported and
checked for import'''
# We already imported it in this file once, make sure it's not imported
# from the cache
sys.modules.pop('import_all_modules_py3_11')
with pytest.raises(ModuleNotFoundError):
modules_main(['import_all_modules_py3_11'])
def test_modules_from_cwd_not_found(tmp_path, monkeypatch):
test_module = tmp_path / 'this_is_a_module_in_cwd.py'
test_module.write_text('')
monkeypatch.chdir(tmp_path)
with pytest.raises(ModuleNotFoundError):
modules_main(['this_is_a_module_in_cwd'])
def test_modules_from_sys_path_found(tmp_path):
test_module = tmp_path / 'this_is_a_module_in_sys_path.py'
test_module.write_text('')
sys.path.append(str(tmp_path))
modules_main(['this_is_a_module_in_sys_path'])
assert 'this_is_a_module_in_sys_path' in sys.modules
def test_modules_from_file_are_found(tmp_path):
test_file = tmp_path / 'this_is_a_file_in_tmp_path.txt'
test_file.write_text('math\nwave\ncsv\n')
# Make sure the tested modules are not already in sys.modules
for m in ('math', 'wave', 'csv'):
sys.modules.pop(m, None)
modules_main(['-f', str(test_file)])
assert 'csv' in sys.modules
assert 'math' in sys.modules
assert 'wave' in sys.modules
def test_modules_from_files_are_found(tmp_path):
test_file_1 = tmp_path / 'this_is_a_file_in_tmp_path_1.txt'
test_file_2 = tmp_path / 'this_is_a_file_in_tmp_path_2.txt'
test_file_3 = tmp_path / 'this_is_a_file_in_tmp_path_3.txt'
test_file_1.write_text('math\nwave\n')
test_file_2.write_text('csv\npathlib\n')
test_file_3.write_text('logging\ncsv\n')
# Make sure the tested modules are not already in sys.modules
for m in ('math', 'wave', 'csv', 'pathlib', 'logging'):
sys.modules.pop(m, None)
modules_main(['-f', str(test_file_1), '-f', str(test_file_2), '-f', str(test_file_3), ])
for module in ('csv', 'math', 'wave', 'pathlib', 'logging'):
assert module in sys.modules
def test_nonexisting_modules_raise_exception_on_import(tmp_path):
test_file = tmp_path / 'this_is_a_file_in_tmp_path.txt'
test_file.write_text('nonexisting_module\nanother\n')
with pytest.raises(ModuleNotFoundError):
modules_main(['-f', str(test_file)])
def test_nested_modules_found_when_expected(tmp_path, monkeypatch, capsys):
# This one is supposed to raise an error
cwd_path = tmp_path / 'test_cwd'
Path.mkdir(cwd_path)
test_module_1 = cwd_path / 'this_is_a_module_in_cwd.py'
# Nested structure that is supposed to be importable
nested_path_1 = tmp_path / 'nested'
nested_path_2 = nested_path_1 / 'more_nested'
for path in (nested_path_1, nested_path_2):
Path.mkdir(path)
test_module_2 = tmp_path / 'this_is_a_module_in_level_0.py'
test_module_3 = nested_path_1 / 'this_is_a_module_in_level_1.py'
test_module_4 = nested_path_2 / 'this_is_a_module_in_level_2.py'
for module in (test_module_1, test_module_2, test_module_3, test_module_4):
module.write_text('')
sys.path.append(str(tmp_path))
monkeypatch.chdir(cwd_path)
with pytest.raises(ModuleNotFoundError):
modules_main([
'this_is_a_module_in_level_0',
'nested.this_is_a_module_in_level_1',
'nested.more_nested.this_is_a_module_in_level_2',
'this_is_a_module_in_cwd'])
_, err = capsys.readouterr()
assert 'Check import: this_is_a_module_in_level_0' in err
assert 'Check import: nested.this_is_a_module_in_level_1' in err
assert 'Check import: nested.more_nested.this_is_a_module_in_level_2' in err
assert 'Check import: this_is_a_module_in_cwd' in err
def test_modules_both_from_files_and_cli_are_imported(tmp_path):
test_file_1 = tmp_path / 'this_is_a_file_in_tmp_path_1.txt'
test_file_1.write_text('this_is_a_module_in_tmp_path_1')
test_file_2 = tmp_path / 'this_is_a_file_in_tmp_path_2.txt'
test_file_2.write_text('this_is_a_module_in_tmp_path_2')
test_module_1 = tmp_path / 'this_is_a_module_in_tmp_path_1.py'
test_module_2 = tmp_path / 'this_is_a_module_in_tmp_path_2.py'
test_module_3 = tmp_path / 'this_is_a_module_in_tmp_path_3.py'
for module in (test_module_1, test_module_2, test_module_3):
module.write_text('')
sys.path.append(str(tmp_path))
modules_main([
'-f', str(test_file_1),
'this_is_a_module_in_tmp_path_3',
'-f', str(test_file_2),
])
expected = (
'this_is_a_module_in_tmp_path_1',
'this_is_a_module_in_tmp_path_2',
'this_is_a_module_in_tmp_path_3',
)
for module in expected:
assert module in sys.modules
def test_non_existing_module_raises_exception(tmp_path):
test_module_1 = tmp_path / 'this_is_a_module_in_tmp_path_1.py'
test_module_1.write_text('')
sys.path.append(str(tmp_path))
with pytest.raises(ModuleNotFoundError):
modules_main([
'this_is_a_module_in_tmp_path_1',
'this_is_a_module_in_tmp_path_2',
])
def test_module_with_error_propagates_exception(tmp_path):
test_module_1 = tmp_path / 'this_is_a_module_in_tmp_path_1.py'
test_module_1.write_text('0/0')
sys.path.append(str(tmp_path))
# The correct exception must be raised
with pytest.raises(ZeroDivisionError):
modules_main([
'this_is_a_module_in_tmp_path_1',
])
def test_correct_modules_are_excluded(tmp_path):
test_module_1 = tmp_path / 'module_in_tmp_path_1.py'
test_module_2 = tmp_path / 'module_in_tmp_path_2.py'
test_module_3 = tmp_path / 'module_in_tmp_path_3.py'
for module in (test_module_1, test_module_2, test_module_3):
module.write_text('')
sys.path.append(str(tmp_path))
test_file_1 = tmp_path / 'a_file_in_tmp_path_1.txt'
test_file_1.write_text('module_in_tmp_path_1\nmodule_in_tmp_path_2\nmodule_in_tmp_path_3\n')
modules_main([
'-e', 'module_in_tmp_path_2',
'-f', str(test_file_1),
'-e', 'module_in_tmp_path_3',
])
assert 'module_in_tmp_path_1' in sys.modules
assert 'module_in_tmp_path_2' not in sys.modules
assert 'module_in_tmp_path_3' not in sys.modules
def test_excluding_all_modules_raises_error(tmp_path):
test_module_1 = tmp_path / 'module_in_tmp_path_1.py'
test_module_2 = tmp_path / 'module_in_tmp_path_2.py'
test_module_3 = tmp_path / 'module_in_tmp_path_3.py'
for module in (test_module_1, test_module_2, test_module_3):
module.write_text('')
sys.path.append(str(tmp_path))
test_file_1 = tmp_path / 'a_file_in_tmp_path_1.txt'
test_file_1.write_text('module_in_tmp_path_1\nmodule_in_tmp_path_2\nmodule_in_tmp_path_3\n')
with pytest.raises(ValueError):
modules_main([
'-e', 'module_in_tmp_path*',
'-f', str(test_file_1),
])
def test_only_toplevel_modules_found(tmp_path):
# Nested structure that is supposed to be importable
nested_path_1 = tmp_path / 'nested'
nested_path_2 = nested_path_1 / 'more_nested'
for path in (nested_path_1, nested_path_2):
Path.mkdir(path)
test_module_1 = tmp_path / 'this_is_a_module_in_level_0.py'
test_module_2 = nested_path_1 / 'this_is_a_module_in_level_1.py'
test_module_3 = nested_path_2 / 'this_is_a_module_in_level_2.py'
for module in (test_module_1, test_module_2, test_module_3):
module.write_text('')
sys.path.append(str(tmp_path))
modules_main([
'this_is_a_module_in_level_0',
'nested.this_is_a_module_in_level_1',
'nested.more_nested.this_is_a_module_in_level_2',
'-t'])
assert 'nested.this_is_a_module_in_level_1' not in sys.modules
assert 'nested.more_nested.this_is_a_module_in_level_2' not in sys.modules
def test_only_toplevel_included_modules_found(tmp_path):
# Nested structure that is supposed to be importable
nested_path_1 = tmp_path / 'nested'
nested_path_2 = nested_path_1 / 'more_nested'
for path in (nested_path_1, nested_path_2):
Path.mkdir(path)
test_module_1 = tmp_path / 'this_is_a_module_in_level_0.py'
test_module_4 = tmp_path / 'this_is_another_module_in_level_0.py'
test_module_2 = nested_path_1 / 'this_is_a_module_in_level_1.py'
test_module_3 = nested_path_2 / 'this_is_a_module_in_level_2.py'
for module in (test_module_1, test_module_2, test_module_3, test_module_4):
module.write_text('')
sys.path.append(str(tmp_path))
modules_main([
'this_is_a_module_in_level_0',
'this_is_another_module_in_level_0',
'nested.this_is_a_module_in_level_1',
'nested.more_nested.this_is_a_module_in_level_2',
'-t',
'-e', '*another*'
])
assert 'nested.this_is_a_module_in_level_1' not in sys.modules
assert 'nested.more_nested.this_is_a_module_in_level_2' not in sys.modules
assert 'this_is_another_module_in_level_0' not in sys.modules
assert 'this_is_a_module_in_level_0' in sys.modules
def test_module_list_from_relative_path(tmp_path, monkeypatch):
monkeypatch.chdir(tmp_path)
test_file_1 = Path('this_is_a_file_in_cwd.txt')
test_file_1.write_text('wave')
sys.modules.pop('wave', None)
modules_main([
'-f', 'this_is_a_file_in_cwd.txt'
])
assert 'wave' in sys.modules
@pytest.mark.parametrize('arch_in_path', [True, False])
def test_pth_files_are_read_from__PYTHONSITE(arch_in_path, tmp_path, monkeypatch, capsys):
sitearch = tmp_path / 'lib64'
sitearch.mkdir()
sitelib = tmp_path / 'lib'
sitelib.mkdir()
for where, word in (sitearch, "ARCH"), (sitelib, "LIB"), (sitelib, "MOD"):
module = where / f'print{word}.py'
module.write_text(f'print("{word}")')
pth_sitearch = sitearch / 'ARCH.pth'
pth_sitearch.write_text('import printARCH\n')
pth_sitelib = sitelib / 'LIB.pth'
pth_sitelib.write_text('import printLIB\n')
if arch_in_path:
sys.path.append(str(sitearch))
sys.path.append(str(sitelib))
# we always add sitearch to _PYTHONSITE
# but when not in sys.path, it should not be processed for .pth files
monkeypatch.setenv('_PYTHONSITE', f'{sitearch}:{sitelib}')
modules_main(['printMOD'])
out, err = capsys.readouterr()
if arch_in_path:
assert out == 'ARCH\nLIB\nMOD\n'
else:
assert out == 'LIB\nMOD\n'

42
tests/tests.yml Normal file
View File

@ -0,0 +1,42 @@
---
- hosts: localhost
tags:
- classic
tasks:
- dnf:
name: "*"
state: latest
- hosts: localhost
roles:
- role: standard-test-basic
tags:
- classic
tests:
- pytest:
dir: .
run: PYTHONPATH=/usr/lib/rpm/redhat ALTERNATE_PYTHON_VERSION=skip pytest-3.11 -v
- manual_byte_compilation_clamp_mtime_off:
dir: .
run: rpmbuild --define 'dist .clamp0' --define 'clamp_mtime_to_source_date_epoch 0' -ba pythontest.spec
- manual_byte_compilation_clamp_mtime_on:
dir: .
run: rpmbuild --define 'dist .clamp1' --define 'clamp_mtime_to_source_date_epoch 1' -ba pythontest.spec
- rpmlint_clamp_mtime_off:
dir: .
run: rpmlint ~/rpmbuild/RPMS/x86_64/pythontest-0-0.clamp0.x86_64.rpm | grep python-bytecode-inconsistent-mtime || exit 0 && exit 1
- rpmlint_clamp_mtime_on:
dir: .
run: rpmlint ~/rpmbuild/RPMS/x86_64/pythontest-0-0.clamp1.x86_64.rpm | grep python-bytecode-inconsistent-mtime || exit 0 && exit 1
required_packages:
- rpm-build
- rpmlint
- python-rpm-macros
- python3-rpm-macros
- python3.11-rpm-macros
- python3.11-devel
- python3.11-pytest
- python3
- python3-devel
- python2