rpms/python39
7
0
Fork 0
Code Issues Pull Requests Releases Activity

import UBI python39-3.9.19-7.module+el8.10.0+22237+51382d7a

Browse Source
This commit is contained in:
eabdullin 2024-08-28 18:21:59 +00:00
parent 615e5a80f2
commit e8ba7c6250
5 changed files with 1015 additions and 311 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

365
SOURCES/00329-fips.patch
View File

@ -1,4 +1,4 @@
From ccb2659fa0ec259d4161ed84345553bf3f216531 Mon Sep 17 00:00:00 2001
From 4cef6c756055a15dc33a475c1f405676fa69410c Mon Sep 17 00:00:00 2001
From: Petr Viktorin <encukou@gmail.com>
Date: Wed, 11 Aug 2021 16:51:03 +0200
Subject: [PATCH 01/10] Backport PyModule_AddObjectRef as
@ -71,10 +71,10 @@ index 13482c6..fca1083 100644
PyModule_AddIntConstant(PyObject *m, const char *name, long value)
{
--
2.37.2
2.45.0
From 794c37495d91823bd820b96382b999d84dcad58d Mon Sep 17 00:00:00 2001
From a115773e979f968edaed8a3419f3ccc34eef8320 Mon Sep 17 00:00:00 2001
From: Petr Viktorin <encukou@gmail.com>
Date: Fri, 13 Aug 2021 13:16:43 +0200
Subject: [PATCH 02/10] _hashopenssl: Uncomment and use initialization function
@ -144,10 +144,10 @@ index 4db058c..56dfff9 100644
return m;
--
2.37.2
2.45.0
From 94b56c82b459474c3e0f9e5421fa7becbf5a1c70 Mon Sep 17 00:00:00 2001
From 28506af69b2b005bb7a8931624f97273269458a1 Mon Sep 17 00:00:00 2001
From: Christian Heimes <christian@python.org>
Date: Sat, 27 Mar 2021 14:55:03 +0100
Subject: [PATCH 03/10] bpo-40645: use C implementation of HMAC (GH-24920,
@ -927,10 +927,10 @@ index 68aa765..4466ec4 100644
-/*[clinic end generated code: output=b6b280e46bf0b139 input=a9049054013a1b77]*/
+/*[clinic end generated code: output=7ff9aad0bd53e7ce input=a9049054013a1b77]*/
--
2.37.2
2.45.0
From b63e3fbd7c0506b5a6c00c1bb0d255054e38bbe8 Mon Sep 17 00:00:00 2001
From 1f5f7d3892febb68cf96ef151beab06eae1792ce Mon Sep 17 00:00:00 2001
From: Charalampos Stratakis <cstratak@redhat.com>
Date: Thu, 12 Dec 2019 16:58:31 +0100
Subject: [PATCH 04/10] Expose blake2b and blake2s hashes from OpenSSL
@ -944,7 +944,7 @@ used under FIPS.
3 files changed, 148 insertions(+), 1 deletion(-)
diff --git a/Lib/test/test_hashlib.py b/Lib/test/test_hashlib.py
index f845c7a..7aaeb76 100644
index bc11a8d..9a07499 100644
--- a/Lib/test/test_hashlib.py
+++ b/Lib/test/test_hashlib.py
@@ -363,6 +363,12 @@ class HashLibTestCase(unittest.TestCase):
@ -1137,10 +1137,10 @@ index 4466ec4..54c22b2 100644
-/*[clinic end generated code: output=7ff9aad0bd53e7ce input=a9049054013a1b77]*/
+/*[clinic end generated code: output=fab05055e982f112 input=a9049054013a1b77]*/
--
2.37.2
2.45.0
From dc8ad7b98d6d9bf14cae439acb3a99fa8f4f5020 Mon Sep 17 00:00:00 2001
From 842bd1c8c8a62c342fed848e5a5f0f1d97daeaba Mon Sep 17 00:00:00 2001
From: Petr Viktorin <pviktori@redhat.com>
Date: Thu, 1 Aug 2019 17:57:05 +0200
Subject: [PATCH 05/10] Use a stronger hash in multiprocessing handshake
@ -1152,7 +1152,7 @@ https://bugs.python.org/issue17258
1 file changed, 6 insertions(+), 2 deletions(-)
diff --git a/Lib/multiprocessing/connection.py b/Lib/multiprocessing/connection.py
index 510e4b5..b68f2fb 100644
index 8e2facf..bb4acb6 100644
--- a/Lib/multiprocessing/connection.py
+++ b/Lib/multiprocessing/connection.py
@@ -42,6 +42,10 @@ BUFSIZE = 8192
@ -1166,7 +1166,7 @@ index 510e4b5..b68f2fb 100644
_mmap_counter = itertools.count()
default_family = 'AF_INET'
@@ -741,7 +745,7 @@ def deliver_challenge(connection, authkey):
@@ -736,7 +740,7 @@ def deliver_challenge(connection, authkey):
"Authkey must be bytes, not {0!s}".format(type(authkey)))
message = os.urandom(MESSAGE_LENGTH)
connection.send_bytes(CHALLENGE + message)
@ -1175,7 +1175,7 @@ index 510e4b5..b68f2fb 100644
response = connection.recv_bytes(256) # reject large message
if response == digest:
connection.send_bytes(WELCOME)
@@ -757,7 +761,7 @@ def answer_challenge(connection, authkey):
@@ -752,7 +756,7 @@ def answer_challenge(connection, authkey):
message = connection.recv_bytes(256) # reject large message
assert message[:len(CHALLENGE)] == CHALLENGE, 'message = %r' % message
message = message[len(CHALLENGE):]
@ -1185,10 +1185,10 @@ index 510e4b5..b68f2fb 100644
response = connection.recv_bytes(256) # reject large message
if response != WELCOME:
--
2.37.2
2.45.0
From af0c88c9d5bc4f9c127e49ed80d14e25d18813f2 Mon Sep 17 00:00:00 2001
From 0390f1ea33f8a604467829733541791a29cee4da Mon Sep 17 00:00:00 2001
From: Petr Viktorin <pviktori@redhat.com>
Date: Thu, 25 Jul 2019 17:19:06 +0200
Subject: [PATCH 06/10] Disable Python's hash implementations in FIPS mode,
@ -1231,7 +1231,7 @@ index ffa3be0..3e3f4dd 100644
def __get_builtin_constructor(name):
cache = __builtin_constructor_cache
diff --git a/Lib/test/test_hashlib.py b/Lib/test/test_hashlib.py
index 7aaeb76..fa4a8d7 100644
index 9a07499..56dfbaa 100644
--- a/Lib/test/test_hashlib.py
+++ b/Lib/test/test_hashlib.py
@@ -35,14 +35,15 @@ else:
@ -1446,10 +1446,10 @@ index 0bec170..479f4b5 100644
))
--
2.37.2
2.45.0
From 9bc3d493a3508fb82df7d24cc62315c072d9eca8 Mon Sep 17 00:00:00 2001
From 667781e01425308ae95f062f8596866a5af76f77 Mon Sep 17 00:00:00 2001
From: Charalampos Stratakis <cstratak@redhat.com>
Date: Fri, 29 Jan 2021 14:16:21 +0100
Subject: [PATCH 07/10] Use python's fall back crypto implementations only if
@ -1565,7 +1565,7 @@ index 3e3f4dd..b842f5f 100644
for __func_name in __always_supported:
diff --git a/Lib/test/test_hashlib.py b/Lib/test/test_hashlib.py
index fa4a8d7..ec6c883 100644
index 56dfbaa..05f4a54 100644
--- a/Lib/test/test_hashlib.py
+++ b/Lib/test/test_hashlib.py
@@ -171,7 +171,13 @@ class HashLibTestCase(unittest.TestCase):
@ -1604,7 +1604,7 @@ index fa4a8d7..ec6c883 100644
def test_get_builtin_constructor(self):
get_builtin_constructor = getattr(hashlib,
'__get_builtin_constructor')
@@ -1061,6 +1081,7 @@ class KDFTests(unittest.TestCase):
@@ -1070,6 +1090,7 @@ class KDFTests(unittest.TestCase):
iterations=1, dklen=None)
self.assertEqual(out, self.pbkdf2_results['sha1'][0][0])
@ -1613,10 +1613,10 @@ index fa4a8d7..ec6c883 100644
def test_pbkdf2_hmac_py(self):
self._test_pbkdf2_hmac(builtin_hashlib.pbkdf2_hmac, builtin_hashes)
--
2.37.2
2.45.0
From 331c0d39cbc9c4df266c375bae8c1a0d27dd78d9 Mon Sep 17 00:00:00 2001
From e36df0297eca09ca1826163f39a812b45558f690 Mon Sep 17 00:00:00 2001
From: Charalampos Stratakis <cstratak@redhat.com>
Date: Wed, 31 Jul 2019 15:43:43 +0200
Subject: [PATCH 08/10] Test equivalence of hashes for the various digests with
@ -1659,7 +1659,7 @@ index 0000000..1f99dd7
+if __name__ == "__main__":
+ unittest.main()
diff --git a/Lib/test/test_hashlib.py b/Lib/test/test_hashlib.py
index ec6c883..0fd036f 100644
index 05f4a54..980c773 100644
--- a/Lib/test/test_hashlib.py
+++ b/Lib/test/test_hashlib.py
@@ -20,6 +20,7 @@ import warnings
@ -1755,7 +1755,7 @@ index ec6c883..0fd036f 100644
return
m = hash_object_constructor(data, **kwargs)
@@ -974,6 +989,15 @@ class HashLibTestCase(unittest.TestCase):
@@ -983,6 +998,15 @@ class HashLibTestCase(unittest.TestCase):
):
HASHXOF()
@ -1772,10 +1772,10 @@ index ec6c883..0fd036f 100644
class KDFTests(unittest.TestCase):
--
2.37.2
2.45.0
From 1a3df28f95710925bc80018bcf22b7f37bbb1e17 Mon Sep 17 00:00:00 2001
From 8fc0216da1d6e148bf086c8c137ddc19a33ab642 Mon Sep 17 00:00:00 2001
From: Petr Viktorin <pviktori@redhat.com>
Date: Mon, 26 Aug 2019 19:39:48 +0200
Subject: [PATCH 09/10] Guard against Python HMAC in FIPS mode
@ -1889,290 +1889,43 @@ index adf52ad..41e6a14 100644
def test_realcopy_old(self):
# Testing if the copy method created a real copy.
--
2.37.2
2.45.0
From dded0e09dd3e51998a2aa54d2ae8464d73987e51 Mon Sep 17 00:00:00 2001
From: Petr Viktorin <encukou@gmail.com>
Date: Wed, 25 Aug 2021 16:44:43 +0200
Subject: [PATCH 10/10] Disable hash-based PYCs in FIPS mode
From 4271e404c9aa918368d397654d60e4e845dfc844 Mon Sep 17 00:00:00 2001
From: Nikita Sobolev <mail@sobolevn.me>
Date: Thu, 24 Nov 2022 01:47:31 +0300
Subject: [PATCH 10/10] closes gh-99508: fix `TypeError` in
`Lib/importlib/_bootstrap_external.py` (GH-99635)
If FIPS mode is on, we can't use siphash-based HMAC
(_Py_KeyedHash), so:
- Unchecked hash PYCs can be imported, but not created
- Checked hash PYCs can not be imported nor created
- The default mode is timestamp-based PYCs, even if
SOURCE_DATE_EPOCH is set.
If FIPS mode is off, there are no changes in behavior.
Resolves: https://bugzilla.redhat.com/show_bug.cgi?id=1835169
---
Lib/py_compile.py | 2 ++
Lib/test/support/__init__.py | 14 +++++++++++++
Lib/test/test_cmd_line_script.py | 2 ++
Lib/test/test_compileall.py | 11 +++++++++-
Lib/test/test_imp.py | 2 ++
.../test_importlib/source/test_file_loader.py | 6 ++++++
Lib/test/test_py_compile.py | 11 ++++++++--
Lib/test/test_zipimport.py | 2 ++
Python/import.c | 20 +++++++++++++++++++
9 files changed, 67 insertions(+), 3 deletions(-)
Lib/importlib/_bootstrap_external.py | 3 ++-
.../next/Library/2022-11-21-10-45-54.gh-issue-99508.QqVbby.rst | 2 ++
2 files changed, 4 insertions(+), 1 deletion(-)
create mode 100644 Misc/NEWS.d/next/Library/2022-11-21-10-45-54.gh-issue-99508.QqVbby.rst
diff --git a/Lib/py_compile.py b/Lib/py_compile.py
index bba3642..02db901 100644
--- a/Lib/py_compile.py
+++ b/Lib/py_compile.py
@@ -70,7 +70,9 @@ class PycInvalidationMode(enum.Enum):
def _get_default_invalidation_mode():
+ import _hashlib
if (os.environ.get('SOURCE_DATE_EPOCH') and not
+ _hashlib.get_fips_mode() and not
os.environ.get('RPM_BUILD_ROOT')):
return PycInvalidationMode.CHECKED_HASH
else:
diff --git a/Lib/test/support/__init__.py b/Lib/test/support/__init__.py
index 6dc0813..b9d5f9a 100644
--- a/Lib/test/support/__init__.py
+++ b/Lib/test/support/__init__.py
@@ -3296,6 +3296,20 @@ def clear_ignored_deprecations(*tokens: object) -> None:
warnings._filters_mutated()
+def fails_in_fips_mode(expected_error):
+ import _hashlib
+ if _hashlib.get_fips_mode():
+ def _decorator(func):
+ def _wrapper(self, *args, **kwargs):
+ with self.assertRaises(expected_error):
+ func(self, *args, **kwargs)
+ return _wrapper
+ else:
+ def _decorator(func):
+ return func
+ return _decorator
+
+
@contextlib.contextmanager
def adjust_int_max_str_digits(max_digits):
"""Temporarily change the integer string conversion length limit."""
diff --git a/Lib/test/test_cmd_line_script.py b/Lib/test/test_cmd_line_script.py
index 7cb1370..61df232 100644
--- a/Lib/test/test_cmd_line_script.py
+++ b/Lib/test/test_cmd_line_script.py
@@ -282,6 +282,7 @@ class CmdLineTest(unittest.TestCase):
self._check_script(zip_name, run_name, zip_name, zip_name, '',
zipimport.zipimporter)
+ @support.fails_in_fips_mode(ImportError)
def test_zipfile_compiled_checked_hash(self):
with support.temp_dir() as script_dir:
script_name = _make_test_script(script_dir, '__main__')
@@ -292,6 +293,7 @@ class CmdLineTest(unittest.TestCase):
self._check_script(zip_name, run_name, zip_name, zip_name, '',
zipimport.zipimporter)
+ @support.fails_in_fips_mode(ImportError)
def test_zipfile_compiled_unchecked_hash(self):
with support.temp_dir() as script_dir:
script_name = _make_test_script(script_dir, '__main__')
diff --git a/Lib/test/test_compileall.py b/Lib/test/test_compileall.py
index ab647d6..7d50f07 100644
--- a/Lib/test/test_compileall.py
+++ b/Lib/test/test_compileall.py
@@ -758,14 +758,23 @@ class CommandLineTestsBase:
out = self.assertRunOK('badfilename')
self.assertRegex(out, b"Can't list 'badfilename'")
- def test_pyc_invalidation_mode(self):
+ @support.fails_in_fips_mode(AssertionError)
+ def test_pyc_invalidation_mode_checked(self):
script_helper.make_script(self.pkgdir, 'f1', '')
pyc = importlib.util.cache_from_source(
os.path.join(self.pkgdir, 'f1.py'))
+
self.assertRunOK('--invalidation-mode=checked-hash', self.pkgdir)
with open(pyc, 'rb') as fp:
data = fp.read()
self.assertEqual(int.from_bytes(data[4:8], 'little'), 0b11)
+
+ @support.fails_in_fips_mode(AssertionError)
+ def test_pyc_invalidation_mode_unchecked(self):
+ script_helper.make_script(self.pkgdir, 'f1', '')
+ pyc = importlib.util.cache_from_source(
+ os.path.join(self.pkgdir, 'f1.py'))
+
self.assertRunOK('--invalidation-mode=unchecked-hash', self.pkgdir)
with open(pyc, 'rb') as fp:
data = fp.read()
diff --git a/Lib/test/test_imp.py b/Lib/test/test_imp.py
index fe394dc..802f0e8 100644
--- a/Lib/test/test_imp.py
+++ b/Lib/test/test_imp.py
@@ -343,6 +343,7 @@ class ImportTests(unittest.TestCase):
import _frozen_importlib
self.assertEqual(_frozen_importlib.__spec__.origin, "frozen")
+ @support.fails_in_fips_mode(ImportError)
def test_source_hash(self):
self.assertEqual(_imp.source_hash(42, b'hi'), b'\xc6\xe7Z\r\x03:}\xab')
self.assertEqual(_imp.source_hash(43, b'hi'), b'\x85\x9765\xf8\x9a\x8b9')
@@ -362,6 +363,7 @@ class ImportTests(unittest.TestCase):
res = script_helper.assert_python_ok(*args)
self.assertEqual(res.out.strip().decode('utf-8'), expected)
+ @support.fails_in_fips_mode(ImportError)
def test_find_and_load_checked_pyc(self):
# issue 34056
with support.temp_cwd():
diff --git a/Lib/test/test_importlib/source/test_file_loader.py b/Lib/test/test_importlib/source/test_file_loader.py
index ab44722..480cc81 100644
--- a/Lib/test/test_importlib/source/test_file_loader.py
+++ b/Lib/test/test_importlib/source/test_file_loader.py
@@ -17,6 +17,7 @@ import types
import unittest
import warnings
+from test import support
from test.support import make_legacy_pyc, unload
from test.test_py_compile import without_source_date_epoch
@@ -239,6 +240,7 @@ class SimpleTest(abc.LoaderTests):
loader.load_module('bad name')
@util.writes_bytecode_files
+ @support.fails_in_fips_mode(ImportError)
def test_checked_hash_based_pyc(self):
with util.create_modules('_temp') as mapping:
source = mapping['_temp']
@@ -270,6 +272,7 @@ class SimpleTest(abc.LoaderTests):
)
@util.writes_bytecode_files
+ @support.fails_in_fips_mode(ImportError)
def test_overridden_checked_hash_based_pyc(self):
with util.create_modules('_temp') as mapping, \
unittest.mock.patch('_imp.check_hash_based_pycs', 'never'):
@@ -295,6 +298,7 @@ class SimpleTest(abc.LoaderTests):
self.assertEqual(mod.state, 'old')
@util.writes_bytecode_files
+ @support.fails_in_fips_mode(ImportError)
def test_unchecked_hash_based_pyc(self):
with util.create_modules('_temp') as mapping:
source = mapping['_temp']
@@ -325,6 +329,7 @@ class SimpleTest(abc.LoaderTests):
)
@util.writes_bytecode_files
+ @support.fails_in_fips_mode(ImportError)
def test_overridden_unchecked_hash_based_pyc(self):
with util.create_modules('_temp') as mapping, \
unittest.mock.patch('_imp.check_hash_based_pycs', 'always'):
@@ -434,6 +439,7 @@ class BadBytecodeTest:
del_source=del_source)
test('_temp', mapping, bc_path)
+ @support.fails_in_fips_mode(ImportError)
def _test_partial_hash(self, test, *, del_source=False):
with util.create_modules('_temp') as mapping:
bc_path = self.manipulate_bytecode(
diff --git a/Lib/test/test_py_compile.py b/Lib/test/test_py_compile.py
index b2d3dcf..7e4b0c5 100644
--- a/Lib/test/test_py_compile.py
+++ b/Lib/test/test_py_compile.py
@@ -141,13 +141,16 @@ class PyCompileTestsBase:
importlib.util.cache_from_source(bad_coding)))
def test_source_date_epoch(self):
+ import _hashlib
py_compile.compile(self.source_path, self.pyc_path)
self.assertTrue(os.path.exists(self.pyc_path))
self.assertFalse(os.path.exists(self.cache_path))
with open(self.pyc_path, 'rb') as fp:
flags = importlib._bootstrap_external._classify_pyc(
fp.read(), 'test', {})
- if os.environ.get('SOURCE_DATE_EPOCH'):
+ if _hashlib.get_fips_mode():
+ expected_flags = 0b00
+ elif os.environ.get('SOURCE_DATE_EPOCH'):
expected_flags = 0b11
else:
expected_flags = 0b00
@@ -178,7 +181,8 @@ class PyCompileTestsBase:
# Specifying optimized bytecode should lead to a path reflecting that.
self.assertIn('opt-2', py_compile.compile(self.source_path, optimize=2))
- def test_invalidation_mode(self):
+ @support.fails_in_fips_mode(ImportError)
+ def test_invalidation_mode_checked(self):
py_compile.compile(
self.source_path,
invalidation_mode=py_compile.PycInvalidationMode.CHECKED_HASH,
@@ -187,6 +191,9 @@ class PyCompileTestsBase:
flags = importlib._bootstrap_external._classify_pyc(
fp.read(), 'test', {})
self.assertEqual(flags, 0b11)
+
+ @support.fails_in_fips_mode(ImportError)
+ def test_invalidation_mode_unchecked(self):
py_compile.compile(
self.source_path,
invalidation_mode=py_compile.PycInvalidationMode.UNCHECKED_HASH,
diff --git a/Lib/test/test_zipimport.py b/Lib/test/test_zipimport.py
index b7347a3..09ea990 100644
--- a/Lib/test/test_zipimport.py
+++ b/Lib/test/test_zipimport.py
@@ -186,6 +186,7 @@ class UncompressedZipImportTestCase(ImportHooksBaseTestCase):
TESTMOD + pyc_ext: (NOW, test_pyc)}
self.doTest(pyc_ext, files, TESTMOD)
+ @support.fails_in_fips_mode(ImportError)
def testUncheckedHashBasedPyc(self):
source = b"state = 'old'"
source_hash = importlib.util.source_hash(source)
@@ -200,6 +201,7 @@ class UncompressedZipImportTestCase(ImportHooksBaseTestCase):
self.assertEqual(mod.state, 'old')
self.doTest(None, files, TESTMOD, call=check)
+ @support.fails_in_fips_mode(ImportError)
@unittest.mock.patch('_imp.check_hash_based_pycs', 'always')
def test_checked_hash_based_change_pyc(self):
source = b"state = 'old'"
diff --git a/Python/import.c b/Python/import.c
index 8358d70..1b7fb85 100644
--- a/Python/import.c
+++ b/Python/import.c
@@ -2354,6 +2354,26 @@ static PyObject *
_imp_source_hash_impl(PyObject *module, long key, Py_buffer *source)
/*[clinic end generated code: output=edb292448cf399ea input=9aaad1e590089789]*/
{
+ PyObject *_hashlib = PyImport_ImportModule("_hashlib");
+ if (_hashlib == NULL) {
+ return NULL;
+ }
+ PyObject *fips_mode_obj = PyObject_CallMethod(_hashlib, "get_fips_mode", NULL);
+ Py_DECREF(_hashlib);
+ if (fips_mode_obj == NULL) {
+ return NULL;
+ }
+ int fips_mode = PyObject_IsTrue(fips_mode_obj);
+ Py_DECREF(fips_mode_obj);
+ if (fips_mode < 0) {
+ return NULL;
+ }
+ if (fips_mode) {
+ PyErr_SetString(
+ PyExc_ImportError,
+ "hash-based PYC validation (siphash24) not available in FIPS mode");
+ return NULL;
+ };
union {
uint64_t x;
char data[sizeof(uint64_t)];
diff --git a/Lib/importlib/_bootstrap_external.py b/Lib/importlib/_bootstrap_external.py
index f0c9f8e..cccf6b2 100644
--- a/Lib/importlib/_bootstrap_external.py
+++ b/Lib/importlib/_bootstrap_external.py
@@ -986,7 +986,8 @@ class SourceLoader(_LoaderBasics):
source_mtime is not None):
if hash_based:
if source_hash is None:
- source_hash = _imp.source_hash(source_bytes)
+ source_hash = _imp.source_hash(_RAW_MAGIC_NUMBER,
+ source_bytes)
data = _code_to_hash_pyc(code_object, source_hash, check_source)
else:
data = _code_to_timestamp_pyc(code_object, source_mtime,
diff --git a/Misc/NEWS.d/next/Library/2022-11-21-10-45-54.gh-issue-99508.QqVbby.rst b/Misc/NEWS.d/next/Library/2022-11-21-10-45-54.gh-issue-99508.QqVbby.rst
new file mode 100644
index 0000000..82720d1
--- /dev/null
+++ b/Misc/NEWS.d/next/Library/2022-11-21-10-45-54.gh-issue-99508.QqVbby.rst
@@ -0,0 +1,2 @@
+Fix ``TypeError`` in ``Lib/importlib/_bootstrap_external.py`` while calling
+``_imp.source_hash()``.
--
2.37.2
2.45.0

402
SOURCES/00431-CVE-2024-4032.patch Normal file
View File

@ -0,0 +1,402 @@
From f647bd8884bc89767914a5e0dea9ae099a8b50b5 Mon Sep 17 00:00:00 2001
From: Petr Viktorin <encukou@gmail.com>
Date: Tue, 7 May 2024 11:57:58 +0200
Subject: [PATCH] gh-113171: gh-65056: Fix "private" (non-global) IP address
ranges (GH-113179) (GH-113186) (GH-118177) (GH-118472)
The _private_networks variables, used by various is_private
implementations, were missing some ranges and at the same time had
overly strict ranges (where there are more specific ranges considered
globally reachable by the IANA registries).
This patch updates the ranges with what was missing or otherwise
incorrect.
100.64.0.0/10 is left alone, for now, as it's been made special in [1].
The _address_exclude_many() call returns 8 networks for IPv4, 121
networks for IPv6.
[1] https://github.com/python/cpython/issues/61602
In 3.10 and below, is_private checks whether the network and broadcast
address are both private.
In later versions (where the test wss backported from), it checks
whether they both are in the same private network.
For 0.0.0.0/0, both 0.0.0.0 and 255.225.255.255 are private,
but one is in 0.0.0.0/8 ("This network") and the other in
255.255.255.255/32 ("Limited broadcast").
---------
Co-authored-by: Jakub Stasiak <jakub@stasiak.at>
---
Doc/library/ipaddress.rst | 43 ++++++++-
Doc/tools/susp-ignored.csv | 8 ++
Doc/whatsnew/3.9.rst | 9 ++
Lib/ipaddress.py | 95 +++++++++++++++----
Lib/test/test_ipaddress.py | 52 ++++++++++
...-03-14-01-38-44.gh-issue-113171.VFnObz.rst | 9 ++
6 files changed, 195 insertions(+), 21 deletions(-)
create mode 100644 Misc/NEWS.d/next/Library/2024-03-14-01-38-44.gh-issue-113171.VFnObz.rst
diff --git a/Doc/library/ipaddress.rst b/Doc/library/ipaddress.rst
index 9c2dff5..f9c1ebf 100644
--- a/Doc/library/ipaddress.rst
+++ b/Doc/library/ipaddress.rst
@@ -188,18 +188,53 @@ write code that handles both IP versions correctly. Address objects are
.. attribute:: is_private
- ``True`` if the address is allocated for private networks. See
+ ``True`` if the address is defined as not globally reachable by
iana-ipv4-special-registry_ (for IPv4) or iana-ipv6-special-registry_
- (for IPv6).
+ (for IPv6) with the following exceptions:
+
+ * ``is_private`` is ``False`` for the shared address space (``100.64.0.0/10``)
+ * For IPv4-mapped IPv6-addresses the ``is_private`` value is determined by the
+ semantics of the underlying IPv4 addresses and the following condition holds
+ (see :attr:`IPv6Address.ipv4_mapped`)::
+
+ address.is_private == address.ipv4_mapped.is_private
+
+ ``is_private`` has value opposite to :attr:`is_global`, except for the shared address space
+ (``100.64.0.0/10`` range) where they are both ``False``.
+
+ .. versionchanged:: 3.9.20
+
+ Fixed some false positives and false negatives.
+
+ * ``192.0.0.0/24`` is considered private with the exception of ``192.0.0.9/32`` and
+ ``192.0.0.10/32`` (previously: only the ``192.0.0.0/29`` sub-range was considered private).
+ * ``64:ff9b:1::/48`` is considered private.
+ * ``2002::/16`` is considered private.
+ * There are exceptions within ``2001::/23`` (otherwise considered private): ``2001:1::1/128``,
+ ``2001:1::2/128``, ``2001:3::/32``, ``2001:4:112::/48``, ``2001:20::/28``, ``2001:30::/28``.
+ The exceptions are not considered private.
.. attribute:: is_global
- ``True`` if the address is allocated for public networks. See
+ ``True`` if the address is defined as globally reachable by
iana-ipv4-special-registry_ (for IPv4) or iana-ipv6-special-registry_
- (for IPv6).
+ (for IPv6) with the following exception:
+
+ For IPv4-mapped IPv6-addresses the ``is_private`` value is determined by the
+ semantics of the underlying IPv4 addresses and the following condition holds
+ (see :attr:`IPv6Address.ipv4_mapped`)::
+
+ address.is_global == address.ipv4_mapped.is_global
+
+ ``is_global`` has value opposite to :attr:`is_private`, except for the shared address space
+ (``100.64.0.0/10`` range) where they are both ``False``.
.. versionadded:: 3.4
+ .. versionchanged:: 3.9.20
+
+ Fixed some false positives and false negatives, see :attr:`is_private` for details.
+
.. attribute:: is_unspecified
``True`` if the address is unspecified. See :RFC:`5735` (for IPv4)
diff --git a/Doc/tools/susp-ignored.csv b/Doc/tools/susp-ignored.csv
index 3eb3d79..de91a50 100644
--- a/Doc/tools/susp-ignored.csv
+++ b/Doc/tools/susp-ignored.csv
@@ -169,6 +169,14 @@ library/ipaddress,,:db00,2001:db00::0/24
library/ipaddress,,::,2001:db00::0/24
library/ipaddress,,:db00,2001:db00::0/ffff:ff00::
library/ipaddress,,::,2001:db00::0/ffff:ff00::
+library/ipaddress,,:ff9b,64:ff9b:1::/48
+library/ipaddress,,::,64:ff9b:1::/48
+library/ipaddress,,::,2001::
+library/ipaddress,,::,2001:1::
+library/ipaddress,,::,2001:3::
+library/ipaddress,,::,2001:4:112::
+library/ipaddress,,::,2001:20::
+library/ipaddress,,::,2001:30::
library/itertools,,:step,elements from seq[start:stop:step]
library/itertools,,:stop,elements from seq[start:stop:step]
library/itertools,,::,kernel = tuple(kernel)[::-1]
diff --git a/Doc/whatsnew/3.9.rst b/Doc/whatsnew/3.9.rst
index 0064e07..1756a37 100644
--- a/Doc/whatsnew/3.9.rst
+++ b/Doc/whatsnew/3.9.rst
@@ -1616,3 +1616,12 @@ tarfile
:exc:`DeprecationWarning`.
In Python 3.14, the default will switch to ``'data'``.
(Contributed by Petr Viktorin in :pep:`706`.)
+
+Notable changes in 3.9.20
+=========================
+
+ipaddress
+---------
+
+* Fixed ``is_global`` and ``is_private`` behavior in ``IPv4Address``,
+ ``IPv6Address``, ``IPv4Network`` and ``IPv6Network``.
diff --git a/Lib/ipaddress.py b/Lib/ipaddress.py
index 25f373a..9b35340 100644
--- a/Lib/ipaddress.py
+++ b/Lib/ipaddress.py
@@ -1322,18 +1322,41 @@ class IPv4Address(_BaseV4, _BaseAddress):
@property
@functools.lru_cache()
def is_private(self):
- """Test if this address is allocated for private networks.
+ """``True`` if the address is defined as not globally reachable by
+ iana-ipv4-special-registry_ (for IPv4) or iana-ipv6-special-registry_
+ (for IPv6) with the following exceptions:
- Returns:
- A boolean, True if the address is reserved per
- iana-ipv4-special-registry.
+ * ``is_private`` is ``False`` for ``100.64.0.0/10``
+ * For IPv4-mapped IPv6-addresses the ``is_private`` value is determined by the
+ semantics of the underlying IPv4 addresses and the following condition holds
+ (see :attr:`IPv6Address.ipv4_mapped`)::
+
+ address.is_private == address.ipv4_mapped.is_private
+ ``is_private`` has value opposite to :attr:`is_global`, except for the ``100.64.0.0/10``
+ IPv4 range where they are both ``False``.
"""
- return any(self in net for net in self._constants._private_networks)
+ return (
+ any(self in net for net in self._constants._private_networks)
+ and all(self not in net for net in self._constants._private_networks_exceptions)
+ )
@property
@functools.lru_cache()
def is_global(self):
+ """``True`` if the address is defined as globally reachable by
+ iana-ipv4-special-registry_ (for IPv4) or iana-ipv6-special-registry_
+ (for IPv6) with the following exception:
+
+ For IPv4-mapped IPv6-addresses the ``is_private`` value is determined by the
+ semantics of the underlying IPv4 addresses and the following condition holds
+ (see :attr:`IPv6Address.ipv4_mapped`)::
+
+ address.is_global == address.ipv4_mapped.is_global
+
+ ``is_global`` has value opposite to :attr:`is_private`, except for the ``100.64.0.0/10``
+ IPv4 range where they are both ``False``.
+ """
return self not in self._constants._public_network and not self.is_private
@property
@@ -1537,13 +1560,15 @@ class _IPv4Constants:
_public_network = IPv4Network('100.64.0.0/10')
+ # Not globally reachable address blocks listed on
+ # https://www.iana.org/assignments/iana-ipv4-special-registry/iana-ipv4-special-registry.xhtml
_private_networks = [
IPv4Network('0.0.0.0/8'),
IPv4Network('10.0.0.0/8'),
IPv4Network('127.0.0.0/8'),
IPv4Network('169.254.0.0/16'),
IPv4Network('172.16.0.0/12'),
- IPv4Network('192.0.0.0/29'),
+ IPv4Network('192.0.0.0/24'),
IPv4Network('192.0.0.170/31'),
IPv4Network('192.0.2.0/24'),
IPv4Network('192.168.0.0/16'),
@@ -1554,6 +1579,11 @@ class _IPv4Constants:
IPv4Network('255.255.255.255/32'),
]
+ _private_networks_exceptions = [
+ IPv4Network('192.0.0.9/32'),
+ IPv4Network('192.0.0.10/32'),
+ ]
+
_reserved_network = IPv4Network('240.0.0.0/4')
_unspecified_address = IPv4Address('0.0.0.0')
@@ -1995,23 +2025,42 @@ class IPv6Address(_BaseV6, _BaseAddress):
@property
@functools.lru_cache()
def is_private(self):
- """Test if this address is allocated for private networks.
+ """``True`` if the address is defined as not globally reachable by
+ iana-ipv4-special-registry_ (for IPv4) or iana-ipv6-special-registry_
+ (for IPv6) with the following exceptions:
- Returns:
- A boolean, True if the address is reserved per
- iana-ipv6-special-registry.
+ * ``is_private`` is ``False`` for ``100.64.0.0/10``
+ * For IPv4-mapped IPv6-addresses the ``is_private`` value is determined by the
+ semantics of the underlying IPv4 addresses and the following condition holds
+ (see :attr:`IPv6Address.ipv4_mapped`)::
+
+ address.is_private == address.ipv4_mapped.is_private
+ ``is_private`` has value opposite to :attr:`is_global`, except for the ``100.64.0.0/10``
+ IPv4 range where they are both ``False``.
"""
- return any(self in net for net in self._constants._private_networks)
+ ipv4_mapped = self.ipv4_mapped
+ if ipv4_mapped is not None:
+ return ipv4_mapped.is_private
+ return (
+ any(self in net for net in self._constants._private_networks)
+ and all(self not in net for net in self._constants._private_networks_exceptions)
+ )
@property
def is_global(self):
- """Test if this address is allocated for public networks.
+ """``True`` if the address is defined as globally reachable by
+ iana-ipv4-special-registry_ (for IPv4) or iana-ipv6-special-registry_
+ (for IPv6) with the following exception:
- Returns:
- A boolean, true if the address is not reserved per
- iana-ipv6-special-registry.
+ For IPv4-mapped IPv6-addresses the ``is_private`` value is determined by the
+ semantics of the underlying IPv4 addresses and the following condition holds
+ (see :attr:`IPv6Address.ipv4_mapped`)::
+
+ address.is_global == address.ipv4_mapped.is_global
+ ``is_global`` has value opposite to :attr:`is_private`, except for the ``100.64.0.0/10``
+ IPv4 range where they are both ``False``.
"""
return not self.is_private
@@ -2252,19 +2301,31 @@ class _IPv6Constants:
_multicast_network = IPv6Network('ff00::/8')
+ # Not globally reachable address blocks listed on
+ # https://www.iana.org/assignments/iana-ipv6-special-registry/iana-ipv6-special-registry.xhtml
_private_networks = [
IPv6Network('::1/128'),
IPv6Network('::/128'),
IPv6Network('::ffff:0:0/96'),
+ IPv6Network('64:ff9b:1::/48'),
IPv6Network('100::/64'),
IPv6Network('2001::/23'),
- IPv6Network('2001:2::/48'),
IPv6Network('2001:db8::/32'),
- IPv6Network('2001:10::/28'),
+ # IANA says N/A, let's consider it not globally reachable to be safe
+ IPv6Network('2002::/16'),
IPv6Network('fc00::/7'),
IPv6Network('fe80::/10'),
]
+ _private_networks_exceptions = [
+ IPv6Network('2001:1::1/128'),
+ IPv6Network('2001:1::2/128'),
+ IPv6Network('2001:3::/32'),
+ IPv6Network('2001:4:112::/48'),
+ IPv6Network('2001:20::/28'),
+ IPv6Network('2001:30::/28'),
+ ]
+
_reserved_networks = [
IPv6Network('::/8'), IPv6Network('100::/8'),
IPv6Network('200::/7'), IPv6Network('400::/6'),
diff --git a/Lib/test/test_ipaddress.py b/Lib/test/test_ipaddress.py
index 90897f6..bd14f04 100644
--- a/Lib/test/test_ipaddress.py
+++ b/Lib/test/test_ipaddress.py
@@ -2263,6 +2263,10 @@ class IpaddrUnitTest(unittest.TestCase):
self.assertEqual(True, ipaddress.ip_address(
'172.31.255.255').is_private)
self.assertEqual(False, ipaddress.ip_address('172.32.0.0').is_private)
+ self.assertFalse(ipaddress.ip_address('192.0.0.0').is_global)
+ self.assertTrue(ipaddress.ip_address('192.0.0.9').is_global)
+ self.assertTrue(ipaddress.ip_address('192.0.0.10').is_global)
+ self.assertFalse(ipaddress.ip_address('192.0.0.255').is_global)
self.assertEqual(True,
ipaddress.ip_address('169.254.100.200').is_link_local)
@@ -2278,6 +2282,40 @@ class IpaddrUnitTest(unittest.TestCase):
self.assertEqual(False, ipaddress.ip_address('128.0.0.0').is_loopback)
self.assertEqual(True, ipaddress.ip_network('0.0.0.0').is_unspecified)
+ def testPrivateNetworks(self):
+ self.assertEqual(True, ipaddress.ip_network("0.0.0.0/0").is_private)
+ self.assertEqual(False, ipaddress.ip_network("1.0.0.0/8").is_private)
+
+ self.assertEqual(True, ipaddress.ip_network("0.0.0.0/8").is_private)
+ self.assertEqual(True, ipaddress.ip_network("10.0.0.0/8").is_private)
+ self.assertEqual(True, ipaddress.ip_network("127.0.0.0/8").is_private)
+ self.assertEqual(True, ipaddress.ip_network("169.254.0.0/16").is_private)
+ self.assertEqual(True, ipaddress.ip_network("172.16.0.0/12").is_private)
+ self.assertEqual(True, ipaddress.ip_network("192.0.0.0/29").is_private)
+ self.assertEqual(False, ipaddress.ip_network("192.0.0.9/32").is_private)
+ self.assertEqual(True, ipaddress.ip_network("192.0.0.170/31").is_private)
+ self.assertEqual(True, ipaddress.ip_network("192.0.2.0/24").is_private)
+ self.assertEqual(True, ipaddress.ip_network("192.168.0.0/16").is_private)
+ self.assertEqual(True, ipaddress.ip_network("198.18.0.0/15").is_private)
+ self.assertEqual(True, ipaddress.ip_network("198.51.100.0/24").is_private)
+ self.assertEqual(True, ipaddress.ip_network("203.0.113.0/24").is_private)
+ self.assertEqual(True, ipaddress.ip_network("240.0.0.0/4").is_private)
+ self.assertEqual(True, ipaddress.ip_network("255.255.255.255/32").is_private)
+
+ self.assertEqual(False, ipaddress.ip_network("::/0").is_private)
+ self.assertEqual(False, ipaddress.ip_network("::ff/128").is_private)
+
+ self.assertEqual(True, ipaddress.ip_network("::1/128").is_private)
+ self.assertEqual(True, ipaddress.ip_network("::/128").is_private)
+ self.assertEqual(True, ipaddress.ip_network("::ffff:0:0/96").is_private)
+ self.assertEqual(True, ipaddress.ip_network("100::/64").is_private)
+ self.assertEqual(True, ipaddress.ip_network("2001:2::/48").is_private)
+ self.assertEqual(False, ipaddress.ip_network("2001:3::/48").is_private)
+ self.assertEqual(True, ipaddress.ip_network("2001:db8::/32").is_private)
+ self.assertEqual(True, ipaddress.ip_network("2001:10::/28").is_private)
+ self.assertEqual(True, ipaddress.ip_network("fc00::/7").is_private)
+ self.assertEqual(True, ipaddress.ip_network("fe80::/10").is_private)
+
def testReservedIpv6(self):
self.assertEqual(True, ipaddress.ip_network('ffff::').is_multicast)
@@ -2351,6 +2389,20 @@ class IpaddrUnitTest(unittest.TestCase):
self.assertEqual(True, ipaddress.ip_address('0::0').is_unspecified)
self.assertEqual(False, ipaddress.ip_address('::1').is_unspecified)
+ self.assertFalse(ipaddress.ip_address('64:ff9b:1::').is_global)
+ self.assertFalse(ipaddress.ip_address('2001::').is_global)
+ self.assertTrue(ipaddress.ip_address('2001:1::1').is_global)
+ self.assertTrue(ipaddress.ip_address('2001:1::2').is_global)
+ self.assertFalse(ipaddress.ip_address('2001:2::').is_global)
+ self.assertTrue(ipaddress.ip_address('2001:3::').is_global)
+ self.assertFalse(ipaddress.ip_address('2001:4::').is_global)
+ self.assertTrue(ipaddress.ip_address('2001:4:112::').is_global)
+ self.assertFalse(ipaddress.ip_address('2001:10::').is_global)
+ self.assertTrue(ipaddress.ip_address('2001:20::').is_global)
+ self.assertTrue(ipaddress.ip_address('2001:30::').is_global)
+ self.assertFalse(ipaddress.ip_address('2001:40::').is_global)
+ self.assertFalse(ipaddress.ip_address('2002::').is_global)
+
# some generic IETF reserved addresses
self.assertEqual(True, ipaddress.ip_address('100::').is_reserved)
self.assertEqual(True, ipaddress.ip_network('4000::1/128').is_reserved)
diff --git a/Misc/NEWS.d/next/Library/2024-03-14-01-38-44.gh-issue-113171.VFnObz.rst b/Misc/NEWS.d/next/Library/2024-03-14-01-38-44.gh-issue-113171.VFnObz.rst
new file mode 100644
index 0000000..f9a7247
--- /dev/null
+++ b/Misc/NEWS.d/next/Library/2024-03-14-01-38-44.gh-issue-113171.VFnObz.rst
@@ -0,0 +1,9 @@
+Fixed various false positives and false negatives in
+
+* :attr:`ipaddress.IPv4Address.is_private` (see these docs for details)
+* :attr:`ipaddress.IPv4Address.is_global`
+* :attr:`ipaddress.IPv6Address.is_private`
+* :attr:`ipaddress.IPv6Address.is_global`
+
+Also in the corresponding :class:`ipaddress.IPv4Network` and :class:`ipaddress.IPv6Network`
+attributes.
--
2.45.2

356
SOURCES/00435-gh-121650-encode-newlines-in-headers-and-verify-headers-are-sound-gh-122233.patch Normal file
View File

@ -0,0 +1,356 @@
From 0000000000000000000000000000000000000000 Mon Sep 17 00:00:00 2001
From: Petr Viktorin <encukou@gmail.com>
Date: Wed, 31 Jul 2024 00:19:48 +0200
Subject: [PATCH] 00435: gh-121650: Encode newlines in headers, and verify
headers are sound (GH-122233)
Per RFC 2047:
> [...] these encoding schemes allow the
> encoding of arbitrary octet values, mail readers that implement this
> decoding should also ensure that display of the decoded data on the
> recipient's terminal will not cause unwanted side-effects
It seems that the "quoted-word" scheme is a valid way to include
a newline character in a header value, just like we already allow
undecodable bytes or control characters.
They do need to be properly quoted when serialized to text, though.
This should fail for custom fold() implementations that aren't careful
about newlines.
(cherry picked from commit 097633981879b3c9de9a1dd120d3aa585ecc2384)
Co-authored-by: Petr Viktorin <encukou@gmail.com>
Co-authored-by: Bas Bloemsaat <bas@bloemsaat.org>
Co-authored-by: Serhiy Storchaka <storchaka@gmail.com>
---
Doc/library/email.errors.rst | 6 ++
Doc/library/email.policy.rst | 18 ++++++
Doc/whatsnew/3.9.rst | 12 ++++
Lib/email/_header_value_parser.py | 12 +++-
Lib/email/_policybase.py | 8 +++
Lib/email/errors.py | 4 ++
Lib/email/generator.py | 13 +++-
Lib/test/test_email/test_generator.py | 62 +++++++++++++++++++
Lib/test/test_email/test_policy.py | 26 ++++++++
...-07-27-16-10-41.gh-issue-121650.nf6oc9.rst | 5 ++
10 files changed, 162 insertions(+), 4 deletions(-)
create mode 100644 Misc/NEWS.d/next/Library/2024-07-27-16-10-41.gh-issue-121650.nf6oc9.rst
diff --git a/Doc/library/email.errors.rst b/Doc/library/email.errors.rst
index f4b9f52509..878c09bb04 100644
--- a/Doc/library/email.errors.rst
+++ b/Doc/library/email.errors.rst
@@ -59,6 +59,12 @@ The following exception classes are defined in the :mod:`email.errors` module:
:class:`~email.mime.image.MIMEImage`).
+.. exception:: HeaderWriteError()
+
+ Raised when an error occurs when the :mod:`~email.generator` outputs
+ headers.
+
+
Here is the list of the defects that the :class:`~email.parser.FeedParser`
can find while parsing messages. Note that the defects are added to the message
where the problem was found, so for example, if a message nested inside a
diff --git a/Doc/library/email.policy.rst b/Doc/library/email.policy.rst
index bf53b9520f..57a75ce452 100644
--- a/Doc/library/email.policy.rst
+++ b/Doc/library/email.policy.rst
@@ -229,6 +229,24 @@ added matters. To illustrate::
.. versionadded:: 3.6
+
+ .. attribute:: verify_generated_headers
+
+ If ``True`` (the default), the generator will raise
+ :exc:`~email.errors.HeaderWriteError` instead of writing a header
+ that is improperly folded or delimited, such that it would
+ be parsed as multiple headers or joined with adjacent data.
+ Such headers can be generated by custom header classes or bugs
+ in the ``email`` module.
+
+ As it's a security feature, this defaults to ``True`` even in the
+ :class:`~email.policy.Compat32` policy.
+ For backwards compatible, but unsafe, behavior, it must be set to
+ ``False`` explicitly.
+
+ .. versionadded:: 3.9.20
+
+
The following :class:`Policy` method is intended to be called by code using
the email library to create policy instances with custom settings:
diff --git a/Doc/whatsnew/3.9.rst b/Doc/whatsnew/3.9.rst
index 1756a37338..eeda4e6955 100644
--- a/Doc/whatsnew/3.9.rst
+++ b/Doc/whatsnew/3.9.rst
@@ -1625,3 +1625,15 @@ ipaddress
* Fixed ``is_global`` and ``is_private`` behavior in ``IPv4Address``,
``IPv6Address``, ``IPv4Network`` and ``IPv6Network``.
+
+email
+-----
+
+* Headers with embedded newlines are now quoted on output.
+
+ The :mod:`~email.generator` will now refuse to serialize (write) headers
+ that are improperly folded or delimited, such that they would be parsed as
+ multiple headers or joined with adjacent data.
+ If you need to turn this safety feature off,
+ set :attr:`~email.policy.Policy.verify_generated_headers`.
+ (Contributed by Bas Bloemsaat and Petr Viktorin in :gh:`121650`.)
diff --git a/Lib/email/_header_value_parser.py b/Lib/email/_header_value_parser.py
index 8a8fb8bc42..e394cfd2e1 100644
--- a/Lib/email/_header_value_parser.py
+++ b/Lib/email/_header_value_parser.py
@@ -92,6 +92,8 @@ TOKEN_ENDS = TSPECIALS | WSP
ASPECIALS = TSPECIALS | set("*'%")
ATTRIBUTE_ENDS = ASPECIALS | WSP
EXTENDED_ATTRIBUTE_ENDS = ATTRIBUTE_ENDS - set('%')
+NLSET = {'\n', '\r'}
+SPECIALSNL = SPECIALS | NLSET
def quote_string(value):
return '"'+str(value).replace('\\', '\\\\').replace('"', r'\"')+'"'
@@ -2778,9 +2780,13 @@ def _refold_parse_tree(parse_tree, *, policy):
wrap_as_ew_blocked -= 1
continue
tstr = str(part)
- if part.token_type == 'ptext' and set(tstr) & SPECIALS:
- # Encode if tstr contains special characters.
- want_encoding = True
+ if not want_encoding:
+ if part.token_type == 'ptext':
+ # Encode if tstr contains special characters.
+ want_encoding = not SPECIALSNL.isdisjoint(tstr)
+ else:
+ # Encode if tstr contains newlines.
+ want_encoding = not NLSET.isdisjoint(tstr)
try:
tstr.encode(encoding)
charset = encoding
diff --git a/Lib/email/_policybase.py b/Lib/email/_policybase.py
index c9cbadd2a8..d1f48211f9 100644
--- a/Lib/email/_policybase.py
+++ b/Lib/email/_policybase.py
@@ -157,6 +157,13 @@ class Policy(_PolicyBase, metaclass=abc.ABCMeta):
message_factory -- the class to use to create new message objects.
If the value is None, the default is Message.
+ verify_generated_headers
+ -- if true, the generator verifies that each header
+ they are properly folded, so that a parser won't
+ treat it as multiple headers, start-of-body, or
+ part of another header.
+ This is a check against custom Header & fold()
+ implementations.
"""
raise_on_defect = False
@@ -165,6 +172,7 @@ class Policy(_PolicyBase, metaclass=abc.ABCMeta):
max_line_length = 78
mangle_from_ = False
message_factory = None
+ verify_generated_headers = True
def handle_defect(self, obj, defect):
"""Based on policy, either raise defect or call register_defect.
diff --git a/Lib/email/errors.py b/Lib/email/errors.py
index d28a680010..1a0d5c63e6 100644
--- a/Lib/email/errors.py
+++ b/Lib/email/errors.py
@@ -29,6 +29,10 @@ class CharsetError(MessageError):
"""An illegal charset was given."""
+class HeaderWriteError(MessageError):
+ """Error while writing headers."""
+
+
# These are parsing defects which the parser was able to work around.
class MessageDefect(ValueError):
"""Base class for a message defect."""
diff --git a/Lib/email/generator.py b/Lib/email/generator.py
index c9b121624e..89224ae41c 100644
--- a/Lib/email/generator.py
+++ b/Lib/email/generator.py
@@ -14,12 +14,14 @@ import random
from copy import deepcopy
from io import StringIO, BytesIO
from email.utils import _has_surrogates
+from email.errors import HeaderWriteError
UNDERSCORE = '_'
NL = '\n' # XXX: no longer used by the code below.
NLCRE = re.compile(r'\r\n|\r|\n')
fcre = re.compile(r'^From ', re.MULTILINE)
+NEWLINE_WITHOUT_FWSP = re.compile(r'\r\n[^ \t]|\r[^ \n\t]|\n[^ \t]')
@@ -223,7 +225,16 @@ class Generator:
def _write_headers(self, msg):
for h, v in msg.raw_items():
- self.write(self.policy.fold(h, v))
+ folded = self.policy.fold(h, v)
+ if self.policy.verify_generated_headers:
+ linesep = self.policy.linesep
+ if not folded.endswith(self.policy.linesep):
+ raise HeaderWriteError(
+ f'folded header does not end with {linesep!r}: {folded!r}')
+ if NEWLINE_WITHOUT_FWSP.search(folded.removesuffix(linesep)):
+ raise HeaderWriteError(
+ f'folded header contains newline: {folded!r}')
+ self.write(folded)
# A blank line always separates headers from body
self.write(self._NL)
diff --git a/Lib/test/test_email/test_generator.py b/Lib/test/test_email/test_generator.py
index 89e7edeb63..d29400f0ed 100644
--- a/Lib/test/test_email/test_generator.py
+++ b/Lib/test/test_email/test_generator.py
@@ -6,6 +6,7 @@ from email.message import EmailMessage
from email.generator import Generator, BytesGenerator
from email.headerregistry import Address
from email import policy
+import email.errors
from test.test_email import TestEmailBase, parameterize
@@ -216,6 +217,44 @@ class TestGeneratorBase:
g.flatten(msg)
self.assertEqual(s.getvalue(), self.typ(expected))
+ def test_keep_encoded_newlines(self):
+ msg = self.msgmaker(self.typ(textwrap.dedent("""\
+ To: nobody
+ Subject: Bad subject=?UTF-8?Q?=0A?=Bcc: injection@example.com
+
+ None
+ """)))
+ expected = textwrap.dedent("""\
+ To: nobody
+ Subject: Bad subject=?UTF-8?Q?=0A?=Bcc: injection@example.com
+
+ None
+ """)
+ s = self.ioclass()
+ g = self.genclass(s, policy=self.policy.clone(max_line_length=80))
+ g.flatten(msg)
+ self.assertEqual(s.getvalue(), self.typ(expected))
+
+ def test_keep_long_encoded_newlines(self):
+ msg = self.msgmaker(self.typ(textwrap.dedent("""\
+ To: nobody
+ Subject: Bad subject=?UTF-8?Q?=0A?=Bcc: injection@example.com
+
+ None
+ """)))
+ expected = textwrap.dedent("""\
+ To: nobody
+ Subject: Bad subject
+ =?utf-8?q?=0A?=Bcc:
+ injection@example.com
+
+ None
+ """)
+ s = self.ioclass()
+ g = self.genclass(s, policy=self.policy.clone(max_line_length=30))
+ g.flatten(msg)
+ self.assertEqual(s.getvalue(), self.typ(expected))
+
class TestGenerator(TestGeneratorBase, TestEmailBase):
@@ -224,6 +263,29 @@ class TestGenerator(TestGeneratorBase, TestEmailBase):
ioclass = io.StringIO
typ = str
+ def test_verify_generated_headers(self):
+ """gh-121650: by default the generator prevents header injection"""
+ class LiteralHeader(str):
+ name = 'Header'
+ def fold(self, **kwargs):
+ return self
+
+ for text in (
+ 'Value\r\nBad Injection\r\n',
+ 'NoNewLine'
+ ):
+ with self.subTest(text=text):
+ message = message_from_string(
+ "Header: Value\r\n\r\nBody",
+ policy=self.policy,
+ )
+
+ del message['Header']
+ message['Header'] = LiteralHeader(text)
+
+ with self.assertRaises(email.errors.HeaderWriteError):
+ message.as_string()
+
class TestBytesGenerator(TestGeneratorBase, TestEmailBase):
diff --git a/Lib/test/test_email/test_policy.py b/Lib/test/test_email/test_policy.py
index e87c275549..ff1ddf7d7a 100644
--- a/Lib/test/test_email/test_policy.py
+++ b/Lib/test/test_email/test_policy.py
@@ -26,6 +26,7 @@ class PolicyAPITests(unittest.TestCase):
'raise_on_defect': False,
'mangle_from_': True,
'message_factory': None,
+ 'verify_generated_headers': True,
}
# These default values are the ones set on email.policy.default.
# If any of these defaults change, the docs must be updated.
@@ -277,6 +278,31 @@ class PolicyAPITests(unittest.TestCase):
with self.assertRaises(email.errors.HeaderParseError):
policy.fold("Subject", subject)
+ def test_verify_generated_headers(self):
+ """Turning protection off allows header injection"""
+ policy = email.policy.default.clone(verify_generated_headers=False)
+ for text in (
+ 'Header: Value\r\nBad: Injection\r\n',
+ 'Header: NoNewLine'
+ ):
+ with self.subTest(text=text):
+ message = email.message_from_string(
+ "Header: Value\r\n\r\nBody",
+ policy=policy,
+ )
+ class LiteralHeader(str):
+ name = 'Header'
+ def fold(self, **kwargs):
+ return self
+
+ del message['Header']
+ message['Header'] = LiteralHeader(text)
+
+ self.assertEqual(
+ message.as_string(),
+ f"{text}\nBody",
+ )
+
# XXX: Need subclassing tests.
# For adding subclassed objects, make sure the usual rules apply (subclass
# wins), but that the order still works (right overrides left).
diff --git a/Misc/NEWS.d/next/Library/2024-07-27-16-10-41.gh-issue-121650.nf6oc9.rst b/Misc/NEWS.d/next/Library/2024-07-27-16-10-41.gh-issue-121650.nf6oc9.rst
new file mode 100644
index 0000000000..83dd28d4ac
--- /dev/null
+++ b/Misc/NEWS.d/next/Library/2024-07-27-16-10-41.gh-issue-121650.nf6oc9.rst
@@ -0,0 +1,5 @@
+:mod:`email` headers with embedded newlines are now quoted on output. The
+:mod:`~email.generator` will now refuse to serialize (write) headers that
+are unsafely folded or delimited; see
+:attr:`~email.policy.Policy.verify_generated_headers`. (Contributed by Bas
+Bloemsaat and Petr Viktorin in :gh:`121650`.)

128
SOURCES/00436-cve-2024-8088-gh-122905-sanitize-names-in-zipfile-path.patch Normal file
View File

@ -0,0 +1,128 @@
From 0000000000000000000000000000000000000000 Mon Sep 17 00:00:00 2001
From: "Jason R. Coombs" <jaraco@jaraco.com>
Date: Mon, 19 Aug 2024 19:28:20 -0400
Subject: [PATCH] 00436: [CVE-2024-8088] gh-122905: Sanitize names in
zipfile.Path.
Co-authored-by: Jason R. Coombs <jaraco@jaraco.com>
---
Lib/test/test_zipfile.py | 17 ++++++
Lib/zipfile.py | 61 ++++++++++++++++++-
...-08-11-14-08-04.gh-issue-122905.7tDsxA.rst | 1 +
3 files changed, 78 insertions(+), 1 deletion(-)
create mode 100644 Misc/NEWS.d/next/Library/2024-08-11-14-08-04.gh-issue-122905.7tDsxA.rst
diff --git a/Lib/test/test_zipfile.py b/Lib/test/test_zipfile.py
index 17e95eb862..9a72152357 100644
--- a/Lib/test/test_zipfile.py
+++ b/Lib/test/test_zipfile.py
@@ -3054,6 +3054,23 @@ class TestPath(unittest.TestCase):
data = ['/'.join(string.ascii_lowercase + str(n)) for n in range(10000)]
zipfile.CompleteDirs._implied_dirs(data)
+ def test_malformed_paths(self):
+ """
+ Path should handle malformed paths.
+ """
+ data = io.BytesIO()
+ zf = zipfile.ZipFile(data, "w")
+ zf.writestr("/one-slash.txt", b"content")
+ zf.writestr("//two-slash.txt", b"content")
+ zf.writestr("../parent.txt", b"content")
+ zf.filename = ''
+ root = zipfile.Path(zf)
+ assert list(map(str, root.iterdir())) == [
+ 'one-slash.txt',
+ 'two-slash.txt',
+ 'parent.txt',
+ ]
+
if __name__ == "__main__":
unittest.main()
diff --git a/Lib/zipfile.py b/Lib/zipfile.py
index 95f95ee112..2e9b2868cd 100644
--- a/Lib/zipfile.py
+++ b/Lib/zipfile.py
@@ -9,6 +9,7 @@ import io
import itertools
import os
import posixpath
+import re
import shutil
import stat
import struct
@@ -2177,7 +2178,65 @@ def _difference(minuend, subtrahend):
return itertools.filterfalse(set(subtrahend).__contains__, minuend)
-class CompleteDirs(ZipFile):
+class SanitizedNames:
+ """
+ ZipFile mix-in to ensure names are sanitized.
+ """
+
+ def namelist(self):
+ return list(map(self._sanitize, super().namelist()))
+
+ @staticmethod
+ def _sanitize(name):
+ r"""
+ Ensure a relative path with posix separators and no dot names.
+ Modeled after
+ https://github.com/python/cpython/blob/bcc1be39cb1d04ad9fc0bd1b9193d3972835a57c/Lib/zipfile/__init__.py#L1799-L1813
+ but provides consistent cross-platform behavior.
+ >>> san = SanitizedNames._sanitize
+ >>> san('/foo/bar')
+ 'foo/bar'
+ >>> san('//foo.txt')
+ 'foo.txt'
+ >>> san('foo/.././bar.txt')
+ 'foo/bar.txt'
+ >>> san('foo../.bar.txt')
+ 'foo../.bar.txt'
+ >>> san('\\foo\\bar.txt')
+ 'foo/bar.txt'
+ >>> san('D:\\foo.txt')
+ 'D/foo.txt'
+ >>> san('\\\\server\\share\\file.txt')
+ 'server/share/file.txt'
+ >>> san('\\\\?\\GLOBALROOT\\Volume3')
+ '?/GLOBALROOT/Volume3'
+ >>> san('\\\\.\\PhysicalDrive1\\root')
+ 'PhysicalDrive1/root'
+ Retain any trailing slash.
+ >>> san('abc/')
+ 'abc/'
+ Raises a ValueError if the result is empty.
+ >>> san('../..')
+ Traceback (most recent call last):
+ ...
+ ValueError: Empty filename
+ """
+
+ def allowed(part):
+ return part and part not in {'..', '.'}
+
+ # Remove the drive letter.
+ # Don't use ntpath.splitdrive, because that also strips UNC paths
+ bare = re.sub('^([A-Z]):', r'\1', name, flags=re.IGNORECASE)
+ clean = bare.replace('\\', '/')
+ parts = clean.split('/')
+ joined = '/'.join(filter(allowed, parts))
+ if not joined:
+ raise ValueError("Empty filename")
+ return joined + '/' * name.endswith('/')
+
+
+class CompleteDirs(SanitizedNames, ZipFile):
"""
A ZipFile subclass that ensures that implied directories
are always included in the namelist.
diff --git a/Misc/NEWS.d/next/Library/2024-08-11-14-08-04.gh-issue-122905.7tDsxA.rst b/Misc/NEWS.d/next/Library/2024-08-11-14-08-04.gh-issue-122905.7tDsxA.rst
new file mode 100644
index 0000000000..1be44c906c
--- /dev/null
+++ b/Misc/NEWS.d/next/Library/2024-08-11-14-08-04.gh-issue-122905.7tDsxA.rst
@@ -0,0 +1 @@
+:class:`zipfile.Path` objects now sanitize names from the zipfile.

75
SPECS/python39.spec
View File

@ -17,7 +17,7 @@ URL: https://www.python.org/
#global prerel ...
%global upstream_version %{general_version}%{?prerel}
Version: %{general_version}%{?prerel:~%{prerel}}
Release: 1%{?dist}
Release: 7%{?dist}
License: Python
# Exclude i686 arch. Due to a modularity issue it's being added to the
@ -182,6 +182,13 @@ ExcludeArch: i686
%global py_INSTSONAME_optimized libpython%{LDVERSION_optimized}.so.%{py_SOVERSION}
%global py_INSTSONAME_debug libpython%{LDVERSION_debug}.so.%{py_SOVERSION}
# The -O flag for the compiler, optimized builds
# https://fedoraproject.org/wiki/Changes/Python_built_with_gcc_O3
%global optflags_optimized -O3
# The -O flag for the compiler, debug builds
# -Wno-cpp avoids some warnings with -O0
%global optflags_debug -O0 -Wno-cpp
# Disable automatic bytecompilation. The python3 binary is not yet be
# available in /usr/bin when Python is built. Also, the bytecompilation fails
# on files that test invalid syntax.
@ -452,6 +459,35 @@ Patch415: 00415-cve-2023-27043-gh-102988-reject-malformed-addresses-in-email-par
# CVE-2023-52425. Future versions of Expat may be more reactive.
Patch422: 00422-fix-tests-for-xmlpullparser-with-expat-2-6-0.patch
# 00431 #
# Security fix for CVE-2024-4032: incorrect IPv4 and IPv6 private ranges
# Resolved upstream: https://github.com/python/cpython/issues/113171
# Tracking bugzilla: https://bugzilla.redhat.com/show_bug.cgi?id=2292921
Patch431: 00431-CVE-2024-4032.patch
# 00435 # f2924d30f4dd44804219c10410a57dd96764d297
# gh-121650: Encode newlines in headers, and verify headers are sound (GH-122233)
#
# Per RFC 2047:
#
# > [...] these encoding schemes allow the
# > encoding of arbitrary octet values, mail readers that implement this
# > decoding should also ensure that display of the decoded data on the
# > recipient's terminal will not cause unwanted side-effects
#
# It seems that the "quoted-word" scheme is a valid way to include
# a newline character in a header value, just like we already allow
# undecodable bytes or control characters.
# They do need to be properly quoted when serialized to text, though.
#
# This should fail for custom fold() implementations that aren't careful
# about newlines.
Patch435: 00435-gh-121650-encode-newlines-in-headers-and-verify-headers-are-sound-gh-122233.patch
# 00436 # 506dd77b7132f69ada7185b8bb91eba0e1296aa8
# [CVE-2024-8088] gh-122905: Sanitize names in zipfile.Path.
Patch436: 00436-cve-2024-8088-gh-122905-sanitize-names-in-zipfile-path.patch
# (New patches go here ^^^)
#
# When adding new patches to "python" and "python3" in Fedora, EL, etc.,
@ -867,6 +903,9 @@ rm Lib/ensurepip/_bundled/*.whl
%apply_patch -q %{PATCH414}
%apply_patch -q %{PATCH415}
%apply_patch -q %{PATCH422}
%apply_patch -q %{PATCH431}
%apply_patch -q %{PATCH435}
%apply_patch -q %{PATCH436}
# Remove all exe files to ensure we are not shipping prebuilt binaries
# note that those are only used to create Microsoft Windows installers
@ -943,6 +982,7 @@ BuildPython() {
ConfName=$1
ExtraConfigArgs=$2
MoreCFlags=$3
MoreCFlagsNodist=$4
# Each build is done in its own directory
ConfDir=build/$ConfName
@ -977,7 +1017,7 @@ BuildPython() {
$ExtraConfigArgs \
%{nil}
%global flags_override EXTRA_CFLAGS="$MoreCFlags" CFLAGS_NODIST="$CFLAGS_NODIST $MoreCFlags"
%global flags_override EXTRA_CFLAGS="$MoreCFlags" CFLAGS_NODIST="$CFLAGS_NODIST $MoreCFlags $MoreCFlagsNodist"
%if %{without bootstrap}
# Regenerate generated files (needs python3)
@ -1000,12 +1040,14 @@ BuildPython() {
# See also: https://bugzilla.redhat.com/show_bug.cgi?id=1818857
BuildPython debug \
"--without-ensurepip --with-pydebug" \
"-O0 -Wno-cpp"
"%{optflags_debug}" \
""
%endif # with debug_build
BuildPython optimized \
"--without-ensurepip %{optimizations_flag}" \
""
"" \
"%{optflags_optimized}"
# ======================================================
# Installing the built code:
@ -1104,7 +1146,7 @@ EOF
%if %{with debug_build}
InstallPython debug \
%{py_INSTSONAME_debug} \
-O0 \
"%{optflags_debug}" \
%{LDVERSION_debug}
%endif # with debug_build
@ -2038,6 +2080,29 @@ fi
# ======================================================
%changelog
* Fri Aug 23 2024 Charalampos Stratakis <cstratak@redhat.com> - 3.9.19-7
- Security fix for CVE-2024-8088
Resolves: RHEL-55954
* Tue Aug 13 2024 Lumír Balhar <lbalhar@redhat.com> - 3.9.19-6
- Security fix for CVE-2024-6923
Resolves: RHEL-53102
* Thu Jul 25 2024 Charalampos Stratakis <cstratak@redhat.com> - 3.9.19-5
- Properly propagate the optimization flags to C extensions
* Thu Jul 18 2024 Charalampos Stratakis <cstratak@redhat.com> - 3.9.19-4
- Build Python with -O3
- https://fedoraproject.org/wiki/Changes/Python_built_with_gcc_O3
* Thu Jul 18 2024 Charalampos Stratakis <cstratak@redhat.com> - 3.9.19-3
- Security fix for CVE-2024-4032
Resolves: RHEL-44094
* Tue Jun 11 2024 Charalampos Stratakis <cstratak@redhat.com> - 3.9.19-2
- Enable importing of hash-based .pyc files under FIPS mode
Resolves: RHEL-40786
* Mon Apr 22 2024 Charalampos Stratakis <cstratak@redhat.com> - 3.9.19-1
- Update to 3.9.19
- Security fixes for CVE-2023-6597 and CVE-2024-0450