rpms/python39
7
0
Fork 0
Code Issues Pull Requests Releases Activity

import UBI python39-3.9.20-1.module+el8.10.0+22342+478c159e

Browse Source
This commit is contained in:
eabdullin 2024-10-23 12:18:54 +00:00
parent e8ba7c6250
commit c695db06ca
10 changed files with 35 additions and 1451 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
.gitignore vendored
View File

@ -1 +1 @@
SOURCES/Python-3.9.19.tar.xz
SOURCES/Python-3.9.20.tar.xz

2
.python39.metadata
View File

@ -1 +1 @@
57d08ec0b329a78923b486abae906d4fa12fadb7 SOURCES/Python-3.9.19.tar.xz
52902dd820d2d41c47ef927ecebb24a96a51cc4b SOURCES/Python-3.9.20.tar.xz

16
SOURCES/00414-skip_test_zlib_s390x.patch
View File

@ -1,4 +1,4 @@
From 2513477cd909e838824f1766d81fdeff01c0790c Mon Sep 17 00:00:00 2001
From f253f1e7e8283b876d40af385d5729646f2c18b6 Mon Sep 17 00:00:00 2001
From: Victor Stinner <vstinner@python.org>
Date: Wed, 17 Jan 2024 14:53:23 +0100
Subject: [PATCH] bpo-46623: Skip two test_zlib tests on s390x (GH-31096)
@ -12,7 +12,7 @@ they fail if zlib uses the s390x hardware accelerator.
create mode 100644 Misc/NEWS.d/next/Tests/2022-02-03-09-45-26.bpo-46623.vxzuhV.rst
diff --git a/Lib/test/test_zlib.py b/Lib/test/test_zlib.py
index 02509cd..f3654c9 100644
index aa7943f..8945b10 100644
--- a/Lib/test/test_zlib.py
+++ b/Lib/test/test_zlib.py
@@ -2,6 +2,7 @@ import unittest
@ -56,10 +56,10 @@ index 02509cd..f3654c9 100644
+skip_on_s390x = unittest.skipIf(hasattr(os, 'uname') and os.uname().machine == 's390x',
+ 'skipped on s390x')
+
class VersionTestCase(unittest.TestCase):
@@ -174,6 +204,7 @@ class CompressTestCase(BaseCompressTestCase, unittest.TestCase):
def _zlib_runtime_version_tuple(zlib_version=zlib.ZLIB_RUNTIME_VERSION):
# Register "1.2.3" as "1.2.3.0"
# or "1.2.0-linux","1.2.0.f","1.2.0.f-linux"
@@ -187,6 +217,7 @@ class CompressTestCase(BaseCompressTestCase, unittest.TestCase):
bufsize=zlib.DEF_BUF_SIZE),
HAMLET_SCENE)
@ -67,7 +67,7 @@ index 02509cd..f3654c9 100644
def test_speech128(self):
# compress more data
data = HAMLET_SCENE * 128
@@ -225,6 +256,7 @@ class CompressTestCase(BaseCompressTestCase, unittest.TestCase):
@@ -238,6 +269,7 @@ class CompressTestCase(BaseCompressTestCase, unittest.TestCase):
class CompressObjectTestCase(BaseCompressTestCase, unittest.TestCase):
# Test compression object
@ -84,5 +84,5 @@ index 0000000..be085c0
+Skip test_pair() and test_speech128() of test_zlib on s390x since they fail
+if zlib uses the s390x hardware accelerator. Patch by Victor Stinner.
--
2.43.0
2.46.0

504
SOURCES/00415-cve-2023-27043-gh-102988-reject-malformed-addresses-in-email-parseaddr-111116.patch
View File

@ -1,505 +1,3 @@
From 0000000000000000000000000000000000000000 Mon Sep 17 00:00:00 2001
From: Victor Stinner <vstinner@python.org>
Date: Fri, 15 Dec 2023 16:10:40 +0100
Subject: [PATCH] 00415: [CVE-2023-27043] gh-102988: Reject malformed addresses
in email.parseaddr() (#111116)
Detect email address parsing errors and return empty tuple to
indicate the parsing error (old API). Add an optional 'strict'
parameter to getaddresses() and parseaddr() functions. Patch by
Thomas Dwyer.
Co-Authored-By: Thomas Dwyer <github@tomd.tel>
---
Doc/library/email.utils.rst | 19 +-
Lib/email/utils.py | 151 ++++++++++++-
Lib/test/test_email/test_email.py | 204 +++++++++++++++++-
...-10-20-15-28-08.gh-issue-102988.dStNO7.rst | 8 +
4 files changed, 361 insertions(+), 21 deletions(-)
create mode 100644 Misc/NEWS.d/next/Library/2023-10-20-15-28-08.gh-issue-102988.dStNO7.rst
diff --git a/Doc/library/email.utils.rst b/Doc/library/email.utils.rst
index 4d0e920eb0..104229e9e5 100644
--- a/Doc/library/email.utils.rst
+++ b/Doc/library/email.utils.rst
@@ -60,13 +60,18 @@ of the new API.
begins with angle brackets, they are stripped off.
-.. function:: parseaddr(address)
+.. function:: parseaddr(address, *, strict=True)
Parse address -- which should be the value of some address-containing field such
as :mailheader:`To` or :mailheader:`Cc` -- into its constituent *realname* and
*email address* parts. Returns a tuple of that information, unless the parse
fails, in which case a 2-tuple of ``('', '')`` is returned.
+ If *strict* is true, use a strict parser which rejects malformed inputs.
+
+ .. versionchanged:: 3.13
+ Add *strict* optional parameter and reject malformed inputs by default.
+
.. function:: formataddr(pair, charset='utf-8')
@@ -84,12 +89,15 @@ of the new API.
Added the *charset* option.
-.. function:: getaddresses(fieldvalues)
+.. function:: getaddresses(fieldvalues, *, strict=True)
This method returns a list of 2-tuples of the form returned by ``parseaddr()``.
*fieldvalues* is a sequence of header field values as might be returned by
- :meth:`Message.get_all <email.message.Message.get_all>`. Here's a simple
- example that gets all the recipients of a message::
+ :meth:`Message.get_all <email.message.Message.get_all>`.
+
+ If *strict* is true, use a strict parser which rejects malformed inputs.
+
+ Here's a simple example that gets all the recipients of a message::
from email.utils import getaddresses
@@ -99,6 +107,9 @@ of the new API.
resent_ccs = msg.get_all('resent-cc', [])
all_recipients = getaddresses(tos + ccs + resent_tos + resent_ccs)
+ .. versionchanged:: 3.13
+ Add *strict* optional parameter and reject malformed inputs by default.
+
.. function:: parsedate(date)
diff --git a/Lib/email/utils.py b/Lib/email/utils.py
index 48d30160aa..7ca7a7c886 100644
--- a/Lib/email/utils.py
+++ b/Lib/email/utils.py
@@ -48,6 +48,7 @@ TICK = "'"
specialsre = re.compile(r'[][\\()<>@,:;".]')
escapesre = re.compile(r'[\\"]')
+
def _has_surrogates(s):
"""Return True if s contains surrogate-escaped binary data."""
# This check is based on the fact that unless there are surrogates, utf8
@@ -106,12 +107,127 @@ def formataddr(pair, charset='utf-8'):
return address
+def _iter_escaped_chars(addr):
+ pos = 0
+ escape = False
+ for pos, ch in enumerate(addr):
+ if escape:
+ yield (pos, '\\' + ch)
+ escape = False
+ elif ch == '\\':
+ escape = True
+ else:
+ yield (pos, ch)
+ if escape:
+ yield (pos, '\\')
-def getaddresses(fieldvalues):
- """Return a list of (REALNAME, EMAIL) for each fieldvalue."""
- all = COMMASPACE.join(str(v) for v in fieldvalues)
- a = _AddressList(all)
- return a.addresslist
+
+def _strip_quoted_realnames(addr):
+ """Strip real names between quotes."""
+ if '"' not in addr:
+ # Fast path
+ return addr
+
+ start = 0
+ open_pos = None
+ result = []
+ for pos, ch in _iter_escaped_chars(addr):
+ if ch == '"':
+ if open_pos is None:
+ open_pos = pos
+ else:
+ if start != open_pos:
+ result.append(addr[start:open_pos])
+ start = pos + 1
+ open_pos = None
+
+ if start < len(addr):
+ result.append(addr[start:])
+
+ return ''.join(result)
+
+
+supports_strict_parsing = True
+
+def getaddresses(fieldvalues, *, strict=True):
+ """Return a list of (REALNAME, EMAIL) or ('','') for each fieldvalue.
+
+ When parsing fails for a fieldvalue, a 2-tuple of ('', '') is returned in
+ its place.
+
+ If strict is true, use a strict parser which rejects malformed inputs.
+ """
+
+ # If strict is true, if the resulting list of parsed addresses is greater
+ # than the number of fieldvalues in the input list, a parsing error has
+ # occurred and consequently a list containing a single empty 2-tuple [('',
+ # '')] is returned in its place. This is done to avoid invalid output.
+ #
+ # Malformed input: getaddresses(['alice@example.com <bob@example.com>'])
+ # Invalid output: [('', 'alice@example.com'), ('', 'bob@example.com')]
+ # Safe output: [('', '')]
+
+ if not strict:
+ all = COMMASPACE.join(str(v) for v in fieldvalues)
+ a = _AddressList(all)
+ return a.addresslist
+
+ fieldvalues = [str(v) for v in fieldvalues]
+ fieldvalues = _pre_parse_validation(fieldvalues)
+ addr = COMMASPACE.join(fieldvalues)
+ a = _AddressList(addr)
+ result = _post_parse_validation(a.addresslist)
+
+ # Treat output as invalid if the number of addresses is not equal to the
+ # expected number of addresses.
+ n = 0
+ for v in fieldvalues:
+ # When a comma is used in the Real Name part it is not a deliminator.
+ # So strip those out before counting the commas.
+ v = _strip_quoted_realnames(v)
+ # Expected number of addresses: 1 + number of commas
+ n += 1 + v.count(',')
+ if len(result) != n:
+ return [('', '')]
+
+ return result
+
+
+def _check_parenthesis(addr):
+ # Ignore parenthesis in quoted real names.
+ addr = _strip_quoted_realnames(addr)
+
+ opens = 0
+ for pos, ch in _iter_escaped_chars(addr):
+ if ch == '(':
+ opens += 1
+ elif ch == ')':
+ opens -= 1
+ if opens < 0:
+ return False
+ return (opens == 0)
+
+
+def _pre_parse_validation(email_header_fields):
+ accepted_values = []
+ for v in email_header_fields:
+ if not _check_parenthesis(v):
+ v = "('', '')"
+ accepted_values.append(v)
+
+ return accepted_values
+
+
+def _post_parse_validation(parsed_email_header_tuples):
+ accepted_values = []
+ # The parser would have parsed a correctly formatted domain-literal
+ # The existence of an [ after parsing indicates a parsing failure
+ for v in parsed_email_header_tuples:
+ if '[' in v[1]:
+ v = ('', '')
+ accepted_values.append(v)
+
+ return accepted_values
def _format_timetuple_and_zone(timetuple, zone):
@@ -202,16 +318,33 @@ def parsedate_to_datetime(data):
tzinfo=datetime.timezone(datetime.timedelta(seconds=tz)))
-def parseaddr(addr):
+def parseaddr(addr, *, strict=True):
"""
Parse addr into its constituent realname and email address parts.
Return a tuple of realname and email address, unless the parse fails, in
which case return a 2-tuple of ('', '').
+
+ If strict is True, use a strict parser which rejects malformed inputs.
"""
- addrs = _AddressList(addr).addresslist
- if not addrs:
- return '', ''
+ if not strict:
+ addrs = _AddressList(addr).addresslist
+ if not addrs:
+ return ('', '')
+ return addrs[0]
+
+ if isinstance(addr, list):
+ addr = addr[0]
+
+ if not isinstance(addr, str):
+ return ('', '')
+
+ addr = _pre_parse_validation([addr])[0]
+ addrs = _post_parse_validation(_AddressList(addr).addresslist)
+
+ if not addrs or len(addrs) > 1:
+ return ('', '')
+
return addrs[0]
diff --git a/Lib/test/test_email/test_email.py b/Lib/test/test_email/test_email.py
index 761ea90b78..0c689643de 100644
--- a/Lib/test/test_email/test_email.py
+++ b/Lib/test/test_email/test_email.py
@@ -16,6 +16,7 @@ from unittest.mock import patch
import email
import email.policy
+import email.utils
from email.charset import Charset
from email.header import Header, decode_header, make_header
@@ -3263,15 +3264,154 @@ Foo
[('Al Person', 'aperson@dom.ain'),
('Bud Person', 'bperson@dom.ain')])
+ def test_getaddresses_comma_in_name(self):
+ """GH-106669 regression test."""
+ self.assertEqual(
+ utils.getaddresses(
+ [
+ '"Bud, Person" <bperson@dom.ain>',
+ 'aperson@dom.ain (Al Person)',
+ '"Mariusz Felisiak" <to@example.com>',
+ ]
+ ),
+ [
+ ('Bud, Person', 'bperson@dom.ain'),
+ ('Al Person', 'aperson@dom.ain'),
+ ('Mariusz Felisiak', 'to@example.com'),
+ ],
+ )
+
+ def test_parsing_errors(self):
+ """Test for parsing errors from CVE-2023-27043 and CVE-2019-16056"""
+ alice = 'alice@example.org'
+ bob = 'bob@example.com'
+ empty = ('', '')
+
+ # Test utils.getaddresses() and utils.parseaddr() on malformed email
+ # addresses: default behavior (strict=True) rejects malformed address,
+ # and strict=False which tolerates malformed address.
+ for invalid_separator, expected_non_strict in (
+ ('(', [(f'<{bob}>', alice)]),
+ (')', [('', alice), empty, ('', bob)]),
+ ('<', [('', alice), empty, ('', bob), empty]),
+ ('>', [('', alice), empty, ('', bob)]),
+ ('[', [('', f'{alice}[<{bob}>]')]),
+ (']', [('', alice), empty, ('', bob)]),
+ ('@', [empty, empty, ('', bob)]),
+ (';', [('', alice), empty, ('', bob)]),
+ (':', [('', alice), ('', bob)]),
+ ('.', [('', alice + '.'), ('', bob)]),
+ ('"', [('', alice), ('', f'<{bob}>')]),
+ ):
+ address = f'{alice}{invalid_separator}<{bob}>'
+ with self.subTest(address=address):
+ self.assertEqual(utils.getaddresses([address]),
+ [empty])
+ self.assertEqual(utils.getaddresses([address], strict=False),
+ expected_non_strict)
+
+ self.assertEqual(utils.parseaddr([address]),
+ empty)
+ self.assertEqual(utils.parseaddr([address], strict=False),
+ ('', address))
+
+ # Comma (',') is treated differently depending on strict parameter.
+ # Comma without quotes.
+ address = f'{alice},<{bob}>'
+ self.assertEqual(utils.getaddresses([address]),
+ [('', alice), ('', bob)])
+ self.assertEqual(utils.getaddresses([address], strict=False),
+ [('', alice), ('', bob)])
+ self.assertEqual(utils.parseaddr([address]),
+ empty)
+ self.assertEqual(utils.parseaddr([address], strict=False),
+ ('', address))
+
+ # Real name between quotes containing comma.
+ address = '"Alice, alice@example.org" <bob@example.com>'
+ expected_strict = ('Alice, alice@example.org', 'bob@example.com')
+ self.assertEqual(utils.getaddresses([address]), [expected_strict])
+ self.assertEqual(utils.getaddresses([address], strict=False), [expected_strict])
+ self.assertEqual(utils.parseaddr([address]), expected_strict)
+ self.assertEqual(utils.parseaddr([address], strict=False),
+ ('', address))
+
+ # Valid parenthesis in comments.
+ address = 'alice@example.org (Alice)'
+ expected_strict = ('Alice', 'alice@example.org')
+ self.assertEqual(utils.getaddresses([address]), [expected_strict])
+ self.assertEqual(utils.getaddresses([address], strict=False), [expected_strict])
+ self.assertEqual(utils.parseaddr([address]), expected_strict)
+ self.assertEqual(utils.parseaddr([address], strict=False),
+ ('', address))
+
+ # Invalid parenthesis in comments.
+ address = 'alice@example.org )Alice('
+ self.assertEqual(utils.getaddresses([address]), [empty])
+ self.assertEqual(utils.getaddresses([address], strict=False),
+ [('', 'alice@example.org'), ('', ''), ('', 'Alice')])
+ self.assertEqual(utils.parseaddr([address]), empty)
+ self.assertEqual(utils.parseaddr([address], strict=False),
+ ('', address))
+
+ # Two addresses with quotes separated by comma.
+ address = '"Jane Doe" <jane@example.net>, "John Doe" <john@example.net>'
+ self.assertEqual(utils.getaddresses([address]),
+ [('Jane Doe', 'jane@example.net'),
+ ('John Doe', 'john@example.net')])
+ self.assertEqual(utils.getaddresses([address], strict=False),
+ [('Jane Doe', 'jane@example.net'),
+ ('John Doe', 'john@example.net')])
+ self.assertEqual(utils.parseaddr([address]), empty)
+ self.assertEqual(utils.parseaddr([address], strict=False),
+ ('', address))
+
+ # Test email.utils.supports_strict_parsing attribute
+ self.assertEqual(email.utils.supports_strict_parsing, True)
+
def test_getaddresses_nasty(self):
- eq = self.assertEqual
- eq(utils.getaddresses(['foo: ;']), [('', '')])
- eq(utils.getaddresses(
- ['[]*-- =~$']),
- [('', ''), ('', ''), ('', '*--')])
- eq(utils.getaddresses(
- ['foo: ;', '"Jason R. Mastaler" <jason@dom.ain>']),
- [('', ''), ('Jason R. Mastaler', 'jason@dom.ain')])
+ for addresses, expected in (
+ (['"Sürname, Firstname" <to@example.com>'],
+ [('Sürname, Firstname', 'to@example.com')]),
+
+ (['foo: ;'],
+ [('', '')]),
+
+ (['foo: ;', '"Jason R. Mastaler" <jason@dom.ain>'],
+ [('', ''), ('Jason R. Mastaler', 'jason@dom.ain')]),
+
+ ([r'Pete(A nice \) chap) <pete(his account)@silly.test(his host)>'],
+ [('Pete (A nice ) chap his account his host)', 'pete@silly.test')]),
+
+ (['(Empty list)(start)Undisclosed recipients :(nobody(I know))'],
+ [('', '')]),
+
+ (['Mary <@machine.tld:mary@example.net>, , jdoe@test . example'],
+ [('Mary', 'mary@example.net'), ('', ''), ('', 'jdoe@test.example')]),
+
+ (['John Doe <jdoe@machine(comment). example>'],
+ [('John Doe (comment)', 'jdoe@machine.example')]),
+
+ (['"Mary Smith: Personal Account" <smith@home.example>'],
+ [('Mary Smith: Personal Account', 'smith@home.example')]),
+
+ (['Undisclosed recipients:;'],
+ [('', '')]),
+
+ ([r'<boss@nil.test>, "Giant; \"Big\" Box" <bob@example.net>'],
+ [('', 'boss@nil.test'), ('Giant; "Big" Box', 'bob@example.net')]),
+ ):
+ with self.subTest(addresses=addresses):
+ self.assertEqual(utils.getaddresses(addresses),
+ expected)
+ self.assertEqual(utils.getaddresses(addresses, strict=False),
+ expected)
+
+ addresses = ['[]*-- =~$']
+ self.assertEqual(utils.getaddresses(addresses),
+ [('', '')])
+ self.assertEqual(utils.getaddresses(addresses, strict=False),
+ [('', ''), ('', ''), ('', '*--')])
def test_getaddresses_embedded_comment(self):
"""Test proper handling of a nested comment"""
@@ -3460,6 +3600,54 @@ multipart/report
m = cls(*constructor, policy=email.policy.default)
self.assertIs(m.policy, email.policy.default)
+ def test_iter_escaped_chars(self):
+ self.assertEqual(list(utils._iter_escaped_chars(r'a\\b\"c\\"d')),
+ [(0, 'a'),
+ (2, '\\\\'),
+ (3, 'b'),
+ (5, '\\"'),
+ (6, 'c'),
+ (8, '\\\\'),
+ (9, '"'),
+ (10, 'd')])
+ self.assertEqual(list(utils._iter_escaped_chars('a\\')),
+ [(0, 'a'), (1, '\\')])
+
+ def test_strip_quoted_realnames(self):
+ def check(addr, expected):
+ self.assertEqual(utils._strip_quoted_realnames(addr), expected)
+
+ check('"Jane Doe" <jane@example.net>, "John Doe" <john@example.net>',
+ ' <jane@example.net>, <john@example.net>')
+ check(r'"Jane \"Doe\"." <jane@example.net>',
+ ' <jane@example.net>')
+
+ # special cases
+ check(r'before"name"after', 'beforeafter')
+ check(r'before"name"', 'before')
+ check(r'b"name"', 'b') # single char
+ check(r'"name"after', 'after')
+ check(r'"name"a', 'a') # single char
+ check(r'"name"', '')
+
+ # no change
+ for addr in (
+ 'Jane Doe <jane@example.net>, John Doe <john@example.net>',
+ 'lone " quote',
+ ):
+ self.assertEqual(utils._strip_quoted_realnames(addr), addr)
+
+
+ def test_check_parenthesis(self):
+ addr = 'alice@example.net'
+ self.assertTrue(utils._check_parenthesis(f'{addr} (Alice)'))
+ self.assertFalse(utils._check_parenthesis(f'{addr} )Alice('))
+ self.assertFalse(utils._check_parenthesis(f'{addr} (Alice))'))
+ self.assertFalse(utils._check_parenthesis(f'{addr} ((Alice)'))
+
+ # Ignore real name between quotes
+ self.assertTrue(utils._check_parenthesis(f'")Alice((" {addr}'))
+
# Test the iterator/generators
class TestIterators(TestEmailBase):
diff --git a/Misc/NEWS.d/next/Library/2023-10-20-15-28-08.gh-issue-102988.dStNO7.rst b/Misc/NEWS.d/next/Library/2023-10-20-15-28-08.gh-issue-102988.dStNO7.rst
new file mode 100644
index 0000000000..3d0e9e4078
--- /dev/null
+++ b/Misc/NEWS.d/next/Library/2023-10-20-15-28-08.gh-issue-102988.dStNO7.rst
@@ -0,0 +1,8 @@
+:func:`email.utils.getaddresses` and :func:`email.utils.parseaddr` now
+return ``('', '')`` 2-tuples in more situations where invalid email
+addresses are encountered instead of potentially inaccurate values. Add
+optional *strict* parameter to these two functions: use ``strict=False`` to
+get the old behavior, accept malformed inputs.
+``getattr(email.utils, 'supports_strict_parsing', False)`` can be use to check
+if the *strict* paramater is available. Patch by Thomas Dwyer and Victor
+Stinner to improve the CVE-2023-27043 fix.
From 4df4fad359c280f2328b98ea9b4414f244624a58 Mon Sep 17 00:00:00 2001
From: Lumir Balhar <lbalhar@redhat.com>
Date: Mon, 18 Dec 2023 20:15:33 +0100
@ -532,7 +30,7 @@ index d1e1898591..7aef773b5f 100644
+ [email_addr_parsing]
+ PYTHON_EMAIL_DISABLE_STRICT_ADDR_PARSING = true
+
.. versionchanged:: 3.13
.. versionchanged:: 3.9.20
Add *strict* optional parameter and reject malformed inputs by default.
@@ -97,6 +110,19 @@ of the new API.

402
SOURCES/00431-CVE-2024-4032.patch
View File

@ -1,402 +0,0 @@
From f647bd8884bc89767914a5e0dea9ae099a8b50b5 Mon Sep 17 00:00:00 2001
From: Petr Viktorin <encukou@gmail.com>
Date: Tue, 7 May 2024 11:57:58 +0200
Subject: [PATCH] gh-113171: gh-65056: Fix "private" (non-global) IP address
ranges (GH-113179) (GH-113186) (GH-118177) (GH-118472)
The _private_networks variables, used by various is_private
implementations, were missing some ranges and at the same time had
overly strict ranges (where there are more specific ranges considered
globally reachable by the IANA registries).
This patch updates the ranges with what was missing or otherwise
incorrect.
100.64.0.0/10 is left alone, for now, as it's been made special in [1].
The _address_exclude_many() call returns 8 networks for IPv4, 121
networks for IPv6.
[1] https://github.com/python/cpython/issues/61602
In 3.10 and below, is_private checks whether the network and broadcast
address are both private.
In later versions (where the test wss backported from), it checks
whether they both are in the same private network.
For 0.0.0.0/0, both 0.0.0.0 and 255.225.255.255 are private,
but one is in 0.0.0.0/8 ("This network") and the other in
255.255.255.255/32 ("Limited broadcast").
---------
Co-authored-by: Jakub Stasiak <jakub@stasiak.at>
---
Doc/library/ipaddress.rst | 43 ++++++++-
Doc/tools/susp-ignored.csv | 8 ++
Doc/whatsnew/3.9.rst | 9 ++
Lib/ipaddress.py | 95 +++++++++++++++----
Lib/test/test_ipaddress.py | 52 ++++++++++
...-03-14-01-38-44.gh-issue-113171.VFnObz.rst | 9 ++
6 files changed, 195 insertions(+), 21 deletions(-)
create mode 100644 Misc/NEWS.d/next/Library/2024-03-14-01-38-44.gh-issue-113171.VFnObz.rst
diff --git a/Doc/library/ipaddress.rst b/Doc/library/ipaddress.rst
index 9c2dff5..f9c1ebf 100644
--- a/Doc/library/ipaddress.rst
+++ b/Doc/library/ipaddress.rst
@@ -188,18 +188,53 @@ write code that handles both IP versions correctly. Address objects are
.. attribute:: is_private
- ``True`` if the address is allocated for private networks. See
+ ``True`` if the address is defined as not globally reachable by
iana-ipv4-special-registry_ (for IPv4) or iana-ipv6-special-registry_
- (for IPv6).
+ (for IPv6) with the following exceptions:
+
+ * ``is_private`` is ``False`` for the shared address space (``100.64.0.0/10``)
+ * For IPv4-mapped IPv6-addresses the ``is_private`` value is determined by the
+ semantics of the underlying IPv4 addresses and the following condition holds
+ (see :attr:`IPv6Address.ipv4_mapped`)::
+
+ address.is_private == address.ipv4_mapped.is_private
+
+ ``is_private`` has value opposite to :attr:`is_global`, except for the shared address space
+ (``100.64.0.0/10`` range) where they are both ``False``.
+
+ .. versionchanged:: 3.9.20
+
+ Fixed some false positives and false negatives.
+
+ * ``192.0.0.0/24`` is considered private with the exception of ``192.0.0.9/32`` and
+ ``192.0.0.10/32`` (previously: only the ``192.0.0.0/29`` sub-range was considered private).
+ * ``64:ff9b:1::/48`` is considered private.
+ * ``2002::/16`` is considered private.
+ * There are exceptions within ``2001::/23`` (otherwise considered private): ``2001:1::1/128``,
+ ``2001:1::2/128``, ``2001:3::/32``, ``2001:4:112::/48``, ``2001:20::/28``, ``2001:30::/28``.
+ The exceptions are not considered private.
.. attribute:: is_global
- ``True`` if the address is allocated for public networks. See
+ ``True`` if the address is defined as globally reachable by
iana-ipv4-special-registry_ (for IPv4) or iana-ipv6-special-registry_
- (for IPv6).
+ (for IPv6) with the following exception:
+
+ For IPv4-mapped IPv6-addresses the ``is_private`` value is determined by the
+ semantics of the underlying IPv4 addresses and the following condition holds
+ (see :attr:`IPv6Address.ipv4_mapped`)::
+
+ address.is_global == address.ipv4_mapped.is_global
+
+ ``is_global`` has value opposite to :attr:`is_private`, except for the shared address space
+ (``100.64.0.0/10`` range) where they are both ``False``.
.. versionadded:: 3.4
+ .. versionchanged:: 3.9.20
+
+ Fixed some false positives and false negatives, see :attr:`is_private` for details.
+
.. attribute:: is_unspecified
``True`` if the address is unspecified. See :RFC:`5735` (for IPv4)
diff --git a/Doc/tools/susp-ignored.csv b/Doc/tools/susp-ignored.csv
index 3eb3d79..de91a50 100644
--- a/Doc/tools/susp-ignored.csv
+++ b/Doc/tools/susp-ignored.csv
@@ -169,6 +169,14 @@ library/ipaddress,,:db00,2001:db00::0/24
library/ipaddress,,::,2001:db00::0/24
library/ipaddress,,:db00,2001:db00::0/ffff:ff00::
library/ipaddress,,::,2001:db00::0/ffff:ff00::
+library/ipaddress,,:ff9b,64:ff9b:1::/48
+library/ipaddress,,::,64:ff9b:1::/48
+library/ipaddress,,::,2001::
+library/ipaddress,,::,2001:1::
+library/ipaddress,,::,2001:3::
+library/ipaddress,,::,2001:4:112::
+library/ipaddress,,::,2001:20::
+library/ipaddress,,::,2001:30::
library/itertools,,:step,elements from seq[start:stop:step]
library/itertools,,:stop,elements from seq[start:stop:step]
library/itertools,,::,kernel = tuple(kernel)[::-1]
diff --git a/Doc/whatsnew/3.9.rst b/Doc/whatsnew/3.9.rst
index 0064e07..1756a37 100644
--- a/Doc/whatsnew/3.9.rst
+++ b/Doc/whatsnew/3.9.rst
@@ -1616,3 +1616,12 @@ tarfile
:exc:`DeprecationWarning`.
In Python 3.14, the default will switch to ``'data'``.
(Contributed by Petr Viktorin in :pep:`706`.)
+
+Notable changes in 3.9.20
+=========================
+
+ipaddress
+---------
+
+* Fixed ``is_global`` and ``is_private`` behavior in ``IPv4Address``,
+ ``IPv6Address``, ``IPv4Network`` and ``IPv6Network``.
diff --git a/Lib/ipaddress.py b/Lib/ipaddress.py
index 25f373a..9b35340 100644
--- a/Lib/ipaddress.py
+++ b/Lib/ipaddress.py
@@ -1322,18 +1322,41 @@ class IPv4Address(_BaseV4, _BaseAddress):
@property
@functools.lru_cache()
def is_private(self):
- """Test if this address is allocated for private networks.
+ """``True`` if the address is defined as not globally reachable by
+ iana-ipv4-special-registry_ (for IPv4) or iana-ipv6-special-registry_
+ (for IPv6) with the following exceptions:
- Returns:
- A boolean, True if the address is reserved per
- iana-ipv4-special-registry.
+ * ``is_private`` is ``False`` for ``100.64.0.0/10``
+ * For IPv4-mapped IPv6-addresses the ``is_private`` value is determined by the
+ semantics of the underlying IPv4 addresses and the following condition holds
+ (see :attr:`IPv6Address.ipv4_mapped`)::
+
+ address.is_private == address.ipv4_mapped.is_private
+ ``is_private`` has value opposite to :attr:`is_global`, except for the ``100.64.0.0/10``
+ IPv4 range where they are both ``False``.
"""
- return any(self in net for net in self._constants._private_networks)
+ return (
+ any(self in net for net in self._constants._private_networks)
+ and all(self not in net for net in self._constants._private_networks_exceptions)
+ )
@property
@functools.lru_cache()
def is_global(self):
+ """``True`` if the address is defined as globally reachable by
+ iana-ipv4-special-registry_ (for IPv4) or iana-ipv6-special-registry_
+ (for IPv6) with the following exception:
+
+ For IPv4-mapped IPv6-addresses the ``is_private`` value is determined by the
+ semantics of the underlying IPv4 addresses and the following condition holds
+ (see :attr:`IPv6Address.ipv4_mapped`)::
+
+ address.is_global == address.ipv4_mapped.is_global
+
+ ``is_global`` has value opposite to :attr:`is_private`, except for the ``100.64.0.0/10``
+ IPv4 range where they are both ``False``.
+ """
return self not in self._constants._public_network and not self.is_private
@property
@@ -1537,13 +1560,15 @@ class _IPv4Constants:
_public_network = IPv4Network('100.64.0.0/10')
+ # Not globally reachable address blocks listed on
+ # https://www.iana.org/assignments/iana-ipv4-special-registry/iana-ipv4-special-registry.xhtml
_private_networks = [
IPv4Network('0.0.0.0/8'),
IPv4Network('10.0.0.0/8'),
IPv4Network('127.0.0.0/8'),
IPv4Network('169.254.0.0/16'),
IPv4Network('172.16.0.0/12'),
- IPv4Network('192.0.0.0/29'),
+ IPv4Network('192.0.0.0/24'),
IPv4Network('192.0.0.170/31'),
IPv4Network('192.0.2.0/24'),
IPv4Network('192.168.0.0/16'),
@@ -1554,6 +1579,11 @@ class _IPv4Constants:
IPv4Network('255.255.255.255/32'),
]
+ _private_networks_exceptions = [
+ IPv4Network('192.0.0.9/32'),
+ IPv4Network('192.0.0.10/32'),
+ ]
+
_reserved_network = IPv4Network('240.0.0.0/4')
_unspecified_address = IPv4Address('0.0.0.0')
@@ -1995,23 +2025,42 @@ class IPv6Address(_BaseV6, _BaseAddress):
@property
@functools.lru_cache()
def is_private(self):
- """Test if this address is allocated for private networks.
+ """``True`` if the address is defined as not globally reachable by
+ iana-ipv4-special-registry_ (for IPv4) or iana-ipv6-special-registry_
+ (for IPv6) with the following exceptions:
- Returns:
- A boolean, True if the address is reserved per
- iana-ipv6-special-registry.
+ * ``is_private`` is ``False`` for ``100.64.0.0/10``
+ * For IPv4-mapped IPv6-addresses the ``is_private`` value is determined by the
+ semantics of the underlying IPv4 addresses and the following condition holds
+ (see :attr:`IPv6Address.ipv4_mapped`)::
+
+ address.is_private == address.ipv4_mapped.is_private
+ ``is_private`` has value opposite to :attr:`is_global`, except for the ``100.64.0.0/10``
+ IPv4 range where they are both ``False``.
"""
- return any(self in net for net in self._constants._private_networks)
+ ipv4_mapped = self.ipv4_mapped
+ if ipv4_mapped is not None:
+ return ipv4_mapped.is_private
+ return (
+ any(self in net for net in self._constants._private_networks)
+ and all(self not in net for net in self._constants._private_networks_exceptions)
+ )
@property
def is_global(self):
- """Test if this address is allocated for public networks.
+ """``True`` if the address is defined as globally reachable by
+ iana-ipv4-special-registry_ (for IPv4) or iana-ipv6-special-registry_
+ (for IPv6) with the following exception:
- Returns:
- A boolean, true if the address is not reserved per
- iana-ipv6-special-registry.
+ For IPv4-mapped IPv6-addresses the ``is_private`` value is determined by the
+ semantics of the underlying IPv4 addresses and the following condition holds
+ (see :attr:`IPv6Address.ipv4_mapped`)::
+
+ address.is_global == address.ipv4_mapped.is_global
+ ``is_global`` has value opposite to :attr:`is_private`, except for the ``100.64.0.0/10``
+ IPv4 range where they are both ``False``.
"""
return not self.is_private
@@ -2252,19 +2301,31 @@ class _IPv6Constants:
_multicast_network = IPv6Network('ff00::/8')
+ # Not globally reachable address blocks listed on
+ # https://www.iana.org/assignments/iana-ipv6-special-registry/iana-ipv6-special-registry.xhtml
_private_networks = [
IPv6Network('::1/128'),
IPv6Network('::/128'),
IPv6Network('::ffff:0:0/96'),
+ IPv6Network('64:ff9b:1::/48'),
IPv6Network('100::/64'),
IPv6Network('2001::/23'),
- IPv6Network('2001:2::/48'),
IPv6Network('2001:db8::/32'),
- IPv6Network('2001:10::/28'),
+ # IANA says N/A, let's consider it not globally reachable to be safe
+ IPv6Network('2002::/16'),
IPv6Network('fc00::/7'),
IPv6Network('fe80::/10'),
]
+ _private_networks_exceptions = [
+ IPv6Network('2001:1::1/128'),
+ IPv6Network('2001:1::2/128'),
+ IPv6Network('2001:3::/32'),
+ IPv6Network('2001:4:112::/48'),
+ IPv6Network('2001:20::/28'),
+ IPv6Network('2001:30::/28'),
+ ]
+
_reserved_networks = [
IPv6Network('::/8'), IPv6Network('100::/8'),
IPv6Network('200::/7'), IPv6Network('400::/6'),
diff --git a/Lib/test/test_ipaddress.py b/Lib/test/test_ipaddress.py
index 90897f6..bd14f04 100644
--- a/Lib/test/test_ipaddress.py
+++ b/Lib/test/test_ipaddress.py
@@ -2263,6 +2263,10 @@ class IpaddrUnitTest(unittest.TestCase):
self.assertEqual(True, ipaddress.ip_address(
'172.31.255.255').is_private)
self.assertEqual(False, ipaddress.ip_address('172.32.0.0').is_private)
+ self.assertFalse(ipaddress.ip_address('192.0.0.0').is_global)
+ self.assertTrue(ipaddress.ip_address('192.0.0.9').is_global)
+ self.assertTrue(ipaddress.ip_address('192.0.0.10').is_global)
+ self.assertFalse(ipaddress.ip_address('192.0.0.255').is_global)
self.assertEqual(True,
ipaddress.ip_address('169.254.100.200').is_link_local)
@@ -2278,6 +2282,40 @@ class IpaddrUnitTest(unittest.TestCase):
self.assertEqual(False, ipaddress.ip_address('128.0.0.0').is_loopback)
self.assertEqual(True, ipaddress.ip_network('0.0.0.0').is_unspecified)
+ def testPrivateNetworks(self):
+ self.assertEqual(True, ipaddress.ip_network("0.0.0.0/0").is_private)
+ self.assertEqual(False, ipaddress.ip_network("1.0.0.0/8").is_private)
+
+ self.assertEqual(True, ipaddress.ip_network("0.0.0.0/8").is_private)
+ self.assertEqual(True, ipaddress.ip_network("10.0.0.0/8").is_private)
+ self.assertEqual(True, ipaddress.ip_network("127.0.0.0/8").is_private)
+ self.assertEqual(True, ipaddress.ip_network("169.254.0.0/16").is_private)
+ self.assertEqual(True, ipaddress.ip_network("172.16.0.0/12").is_private)
+ self.assertEqual(True, ipaddress.ip_network("192.0.0.0/29").is_private)
+ self.assertEqual(False, ipaddress.ip_network("192.0.0.9/32").is_private)
+ self.assertEqual(True, ipaddress.ip_network("192.0.0.170/31").is_private)
+ self.assertEqual(True, ipaddress.ip_network("192.0.2.0/24").is_private)
+ self.assertEqual(True, ipaddress.ip_network("192.168.0.0/16").is_private)
+ self.assertEqual(True, ipaddress.ip_network("198.18.0.0/15").is_private)
+ self.assertEqual(True, ipaddress.ip_network("198.51.100.0/24").is_private)
+ self.assertEqual(True, ipaddress.ip_network("203.0.113.0/24").is_private)
+ self.assertEqual(True, ipaddress.ip_network("240.0.0.0/4").is_private)
+ self.assertEqual(True, ipaddress.ip_network("255.255.255.255/32").is_private)
+
+ self.assertEqual(False, ipaddress.ip_network("::/0").is_private)
+ self.assertEqual(False, ipaddress.ip_network("::ff/128").is_private)
+
+ self.assertEqual(True, ipaddress.ip_network("::1/128").is_private)
+ self.assertEqual(True, ipaddress.ip_network("::/128").is_private)
+ self.assertEqual(True, ipaddress.ip_network("::ffff:0:0/96").is_private)
+ self.assertEqual(True, ipaddress.ip_network("100::/64").is_private)
+ self.assertEqual(True, ipaddress.ip_network("2001:2::/48").is_private)
+ self.assertEqual(False, ipaddress.ip_network("2001:3::/48").is_private)
+ self.assertEqual(True, ipaddress.ip_network("2001:db8::/32").is_private)
+ self.assertEqual(True, ipaddress.ip_network("2001:10::/28").is_private)
+ self.assertEqual(True, ipaddress.ip_network("fc00::/7").is_private)
+ self.assertEqual(True, ipaddress.ip_network("fe80::/10").is_private)
+
def testReservedIpv6(self):
self.assertEqual(True, ipaddress.ip_network('ffff::').is_multicast)
@@ -2351,6 +2389,20 @@ class IpaddrUnitTest(unittest.TestCase):
self.assertEqual(True, ipaddress.ip_address('0::0').is_unspecified)
self.assertEqual(False, ipaddress.ip_address('::1').is_unspecified)
+ self.assertFalse(ipaddress.ip_address('64:ff9b:1::').is_global)
+ self.assertFalse(ipaddress.ip_address('2001::').is_global)
+ self.assertTrue(ipaddress.ip_address('2001:1::1').is_global)
+ self.assertTrue(ipaddress.ip_address('2001:1::2').is_global)
+ self.assertFalse(ipaddress.ip_address('2001:2::').is_global)
+ self.assertTrue(ipaddress.ip_address('2001:3::').is_global)
+ self.assertFalse(ipaddress.ip_address('2001:4::').is_global)
+ self.assertTrue(ipaddress.ip_address('2001:4:112::').is_global)
+ self.assertFalse(ipaddress.ip_address('2001:10::').is_global)
+ self.assertTrue(ipaddress.ip_address('2001:20::').is_global)
+ self.assertTrue(ipaddress.ip_address('2001:30::').is_global)
+ self.assertFalse(ipaddress.ip_address('2001:40::').is_global)
+ self.assertFalse(ipaddress.ip_address('2002::').is_global)
+
# some generic IETF reserved addresses
self.assertEqual(True, ipaddress.ip_address('100::').is_reserved)
self.assertEqual(True, ipaddress.ip_network('4000::1/128').is_reserved)
diff --git a/Misc/NEWS.d/next/Library/2024-03-14-01-38-44.gh-issue-113171.VFnObz.rst b/Misc/NEWS.d/next/Library/2024-03-14-01-38-44.gh-issue-113171.VFnObz.rst
new file mode 100644
index 0000000..f9a7247
--- /dev/null
+++ b/Misc/NEWS.d/next/Library/2024-03-14-01-38-44.gh-issue-113171.VFnObz.rst
@@ -0,0 +1,9 @@
+Fixed various false positives and false negatives in
+
+* :attr:`ipaddress.IPv4Address.is_private` (see these docs for details)
+* :attr:`ipaddress.IPv4Address.is_global`
+* :attr:`ipaddress.IPv6Address.is_private`
+* :attr:`ipaddress.IPv6Address.is_global`
+
+Also in the corresponding :class:`ipaddress.IPv4Network` and :class:`ipaddress.IPv6Network`
+attributes.
--
2.45.2

356
SOURCES/00435-gh-121650-encode-newlines-in-headers-and-verify-headers-are-sound-gh-122233.patch
View File

@ -1,356 +0,0 @@
From 0000000000000000000000000000000000000000 Mon Sep 17 00:00:00 2001
From: Petr Viktorin <encukou@gmail.com>
Date: Wed, 31 Jul 2024 00:19:48 +0200
Subject: [PATCH] 00435: gh-121650: Encode newlines in headers, and verify
headers are sound (GH-122233)
Per RFC 2047:
> [...] these encoding schemes allow the
> encoding of arbitrary octet values, mail readers that implement this
> decoding should also ensure that display of the decoded data on the
> recipient's terminal will not cause unwanted side-effects
It seems that the "quoted-word" scheme is a valid way to include
a newline character in a header value, just like we already allow
undecodable bytes or control characters.
They do need to be properly quoted when serialized to text, though.
This should fail for custom fold() implementations that aren't careful
about newlines.
(cherry picked from commit 097633981879b3c9de9a1dd120d3aa585ecc2384)
Co-authored-by: Petr Viktorin <encukou@gmail.com>
Co-authored-by: Bas Bloemsaat <bas@bloemsaat.org>
Co-authored-by: Serhiy Storchaka <storchaka@gmail.com>
---
Doc/library/email.errors.rst | 6 ++
Doc/library/email.policy.rst | 18 ++++++
Doc/whatsnew/3.9.rst | 12 ++++
Lib/email/_header_value_parser.py | 12 +++-
Lib/email/_policybase.py | 8 +++
Lib/email/errors.py | 4 ++
Lib/email/generator.py | 13 +++-
Lib/test/test_email/test_generator.py | 62 +++++++++++++++++++
Lib/test/test_email/test_policy.py | 26 ++++++++
...-07-27-16-10-41.gh-issue-121650.nf6oc9.rst | 5 ++
10 files changed, 162 insertions(+), 4 deletions(-)
create mode 100644 Misc/NEWS.d/next/Library/2024-07-27-16-10-41.gh-issue-121650.nf6oc9.rst
diff --git a/Doc/library/email.errors.rst b/Doc/library/email.errors.rst
index f4b9f52509..878c09bb04 100644
--- a/Doc/library/email.errors.rst
+++ b/Doc/library/email.errors.rst
@@ -59,6 +59,12 @@ The following exception classes are defined in the :mod:`email.errors` module:
:class:`~email.mime.image.MIMEImage`).
+.. exception:: HeaderWriteError()
+
+ Raised when an error occurs when the :mod:`~email.generator` outputs
+ headers.
+
+
Here is the list of the defects that the :class:`~email.parser.FeedParser`
can find while parsing messages. Note that the defects are added to the message
where the problem was found, so for example, if a message nested inside a
diff --git a/Doc/library/email.policy.rst b/Doc/library/email.policy.rst
index bf53b9520f..57a75ce452 100644
--- a/Doc/library/email.policy.rst
+++ b/Doc/library/email.policy.rst
@@ -229,6 +229,24 @@ added matters. To illustrate::
.. versionadded:: 3.6
+
+ .. attribute:: verify_generated_headers
+
+ If ``True`` (the default), the generator will raise
+ :exc:`~email.errors.HeaderWriteError` instead of writing a header
+ that is improperly folded or delimited, such that it would
+ be parsed as multiple headers or joined with adjacent data.
+ Such headers can be generated by custom header classes or bugs
+ in the ``email`` module.
+
+ As it's a security feature, this defaults to ``True`` even in the
+ :class:`~email.policy.Compat32` policy.
+ For backwards compatible, but unsafe, behavior, it must be set to
+ ``False`` explicitly.
+
+ .. versionadded:: 3.9.20
+
+
The following :class:`Policy` method is intended to be called by code using
the email library to create policy instances with custom settings:
diff --git a/Doc/whatsnew/3.9.rst b/Doc/whatsnew/3.9.rst
index 1756a37338..eeda4e6955 100644
--- a/Doc/whatsnew/3.9.rst
+++ b/Doc/whatsnew/3.9.rst
@@ -1625,3 +1625,15 @@ ipaddress
* Fixed ``is_global`` and ``is_private`` behavior in ``IPv4Address``,
``IPv6Address``, ``IPv4Network`` and ``IPv6Network``.
+
+email
+-----
+
+* Headers with embedded newlines are now quoted on output.
+
+ The :mod:`~email.generator` will now refuse to serialize (write) headers
+ that are improperly folded or delimited, such that they would be parsed as
+ multiple headers or joined with adjacent data.
+ If you need to turn this safety feature off,
+ set :attr:`~email.policy.Policy.verify_generated_headers`.
+ (Contributed by Bas Bloemsaat and Petr Viktorin in :gh:`121650`.)
diff --git a/Lib/email/_header_value_parser.py b/Lib/email/_header_value_parser.py
index 8a8fb8bc42..e394cfd2e1 100644
--- a/Lib/email/_header_value_parser.py
+++ b/Lib/email/_header_value_parser.py
@@ -92,6 +92,8 @@ TOKEN_ENDS = TSPECIALS | WSP
ASPECIALS = TSPECIALS | set("*'%")
ATTRIBUTE_ENDS = ASPECIALS | WSP
EXTENDED_ATTRIBUTE_ENDS = ATTRIBUTE_ENDS - set('%')
+NLSET = {'\n', '\r'}
+SPECIALSNL = SPECIALS | NLSET
def quote_string(value):
return '"'+str(value).replace('\\', '\\\\').replace('"', r'\"')+'"'
@@ -2778,9 +2780,13 @@ def _refold_parse_tree(parse_tree, *, policy):
wrap_as_ew_blocked -= 1
continue
tstr = str(part)
- if part.token_type == 'ptext' and set(tstr) & SPECIALS:
- # Encode if tstr contains special characters.
- want_encoding = True
+ if not want_encoding:
+ if part.token_type == 'ptext':
+ # Encode if tstr contains special characters.
+ want_encoding = not SPECIALSNL.isdisjoint(tstr)
+ else:
+ # Encode if tstr contains newlines.
+ want_encoding = not NLSET.isdisjoint(tstr)
try:
tstr.encode(encoding)
charset = encoding
diff --git a/Lib/email/_policybase.py b/Lib/email/_policybase.py
index c9cbadd2a8..d1f48211f9 100644
--- a/Lib/email/_policybase.py
+++ b/Lib/email/_policybase.py
@@ -157,6 +157,13 @@ class Policy(_PolicyBase, metaclass=abc.ABCMeta):
message_factory -- the class to use to create new message objects.
If the value is None, the default is Message.
+ verify_generated_headers
+ -- if true, the generator verifies that each header
+ they are properly folded, so that a parser won't
+ treat it as multiple headers, start-of-body, or
+ part of another header.
+ This is a check against custom Header & fold()
+ implementations.
"""
raise_on_defect = False
@@ -165,6 +172,7 @@ class Policy(_PolicyBase, metaclass=abc.ABCMeta):
max_line_length = 78
mangle_from_ = False
message_factory = None
+ verify_generated_headers = True
def handle_defect(self, obj, defect):
"""Based on policy, either raise defect or call register_defect.
diff --git a/Lib/email/errors.py b/Lib/email/errors.py
index d28a680010..1a0d5c63e6 100644
--- a/Lib/email/errors.py
+++ b/Lib/email/errors.py
@@ -29,6 +29,10 @@ class CharsetError(MessageError):
"""An illegal charset was given."""
+class HeaderWriteError(MessageError):
+ """Error while writing headers."""
+
+
# These are parsing defects which the parser was able to work around.
class MessageDefect(ValueError):
"""Base class for a message defect."""
diff --git a/Lib/email/generator.py b/Lib/email/generator.py
index c9b121624e..89224ae41c 100644
--- a/Lib/email/generator.py
+++ b/Lib/email/generator.py
@@ -14,12 +14,14 @@ import random
from copy import deepcopy
from io import StringIO, BytesIO
from email.utils import _has_surrogates
+from email.errors import HeaderWriteError
UNDERSCORE = '_'
NL = '\n' # XXX: no longer used by the code below.
NLCRE = re.compile(r'\r\n|\r|\n')
fcre = re.compile(r'^From ', re.MULTILINE)
+NEWLINE_WITHOUT_FWSP = re.compile(r'\r\n[^ \t]|\r[^ \n\t]|\n[^ \t]')
@@ -223,7 +225,16 @@ class Generator:
def _write_headers(self, msg):
for h, v in msg.raw_items():
- self.write(self.policy.fold(h, v))
+ folded = self.policy.fold(h, v)
+ if self.policy.verify_generated_headers:
+ linesep = self.policy.linesep
+ if not folded.endswith(self.policy.linesep):
+ raise HeaderWriteError(
+ f'folded header does not end with {linesep!r}: {folded!r}')
+ if NEWLINE_WITHOUT_FWSP.search(folded.removesuffix(linesep)):
+ raise HeaderWriteError(
+ f'folded header contains newline: {folded!r}')
+ self.write(folded)
# A blank line always separates headers from body
self.write(self._NL)
diff --git a/Lib/test/test_email/test_generator.py b/Lib/test/test_email/test_generator.py
index 89e7edeb63..d29400f0ed 100644
--- a/Lib/test/test_email/test_generator.py
+++ b/Lib/test/test_email/test_generator.py
@@ -6,6 +6,7 @@ from email.message import EmailMessage
from email.generator import Generator, BytesGenerator
from email.headerregistry import Address
from email import policy
+import email.errors
from test.test_email import TestEmailBase, parameterize
@@ -216,6 +217,44 @@ class TestGeneratorBase:
g.flatten(msg)
self.assertEqual(s.getvalue(), self.typ(expected))
+ def test_keep_encoded_newlines(self):
+ msg = self.msgmaker(self.typ(textwrap.dedent("""\
+ To: nobody
+ Subject: Bad subject=?UTF-8?Q?=0A?=Bcc: injection@example.com
+
+ None
+ """)))
+ expected = textwrap.dedent("""\
+ To: nobody
+ Subject: Bad subject=?UTF-8?Q?=0A?=Bcc: injection@example.com
+
+ None
+ """)
+ s = self.ioclass()
+ g = self.genclass(s, policy=self.policy.clone(max_line_length=80))
+ g.flatten(msg)
+ self.assertEqual(s.getvalue(), self.typ(expected))
+
+ def test_keep_long_encoded_newlines(self):
+ msg = self.msgmaker(self.typ(textwrap.dedent("""\
+ To: nobody
+ Subject: Bad subject=?UTF-8?Q?=0A?=Bcc: injection@example.com
+
+ None
+ """)))
+ expected = textwrap.dedent("""\
+ To: nobody
+ Subject: Bad subject
+ =?utf-8?q?=0A?=Bcc:
+ injection@example.com
+
+ None
+ """)
+ s = self.ioclass()
+ g = self.genclass(s, policy=self.policy.clone(max_line_length=30))
+ g.flatten(msg)
+ self.assertEqual(s.getvalue(), self.typ(expected))
+
class TestGenerator(TestGeneratorBase, TestEmailBase):
@@ -224,6 +263,29 @@ class TestGenerator(TestGeneratorBase, TestEmailBase):
ioclass = io.StringIO
typ = str
+ def test_verify_generated_headers(self):
+ """gh-121650: by default the generator prevents header injection"""
+ class LiteralHeader(str):
+ name = 'Header'
+ def fold(self, **kwargs):
+ return self
+
+ for text in (
+ 'Value\r\nBad Injection\r\n',
+ 'NoNewLine'
+ ):
+ with self.subTest(text=text):
+ message = message_from_string(
+ "Header: Value\r\n\r\nBody",
+ policy=self.policy,
+ )
+
+ del message['Header']
+ message['Header'] = LiteralHeader(text)
+
+ with self.assertRaises(email.errors.HeaderWriteError):
+ message.as_string()
+
class TestBytesGenerator(TestGeneratorBase, TestEmailBase):
diff --git a/Lib/test/test_email/test_policy.py b/Lib/test/test_email/test_policy.py
index e87c275549..ff1ddf7d7a 100644
--- a/Lib/test/test_email/test_policy.py
+++ b/Lib/test/test_email/test_policy.py
@@ -26,6 +26,7 @@ class PolicyAPITests(unittest.TestCase):
'raise_on_defect': False,
'mangle_from_': True,
'message_factory': None,
+ 'verify_generated_headers': True,
}
# These default values are the ones set on email.policy.default.
# If any of these defaults change, the docs must be updated.
@@ -277,6 +278,31 @@ class PolicyAPITests(unittest.TestCase):
with self.assertRaises(email.errors.HeaderParseError):
policy.fold("Subject", subject)
+ def test_verify_generated_headers(self):
+ """Turning protection off allows header injection"""
+ policy = email.policy.default.clone(verify_generated_headers=False)
+ for text in (
+ 'Header: Value\r\nBad: Injection\r\n',
+ 'Header: NoNewLine'
+ ):
+ with self.subTest(text=text):
+ message = email.message_from_string(
+ "Header: Value\r\n\r\nBody",
+ policy=policy,
+ )
+ class LiteralHeader(str):
+ name = 'Header'
+ def fold(self, **kwargs):
+ return self
+
+ del message['Header']
+ message['Header'] = LiteralHeader(text)
+
+ self.assertEqual(
+ message.as_string(),
+ f"{text}\nBody",
+ )
+
# XXX: Need subclassing tests.
# For adding subclassed objects, make sure the usual rules apply (subclass
# wins), but that the order still works (right overrides left).
diff --git a/Misc/NEWS.d/next/Library/2024-07-27-16-10-41.gh-issue-121650.nf6oc9.rst b/Misc/NEWS.d/next/Library/2024-07-27-16-10-41.gh-issue-121650.nf6oc9.rst
new file mode 100644
index 0000000000..83dd28d4ac
--- /dev/null
+++ b/Misc/NEWS.d/next/Library/2024-07-27-16-10-41.gh-issue-121650.nf6oc9.rst
@@ -0,0 +1,5 @@
+:mod:`email` headers with embedded newlines are now quoted on output. The
+:mod:`~email.generator` will now refuse to serialize (write) headers that
+are unsafely folded or delimited; see
+:attr:`~email.policy.Policy.verify_generated_headers`. (Contributed by Bas
+Bloemsaat and Petr Viktorin in :gh:`121650`.)

128
SOURCES/00436-cve-2024-8088-gh-122905-sanitize-names-in-zipfile-path.patch
View File

@ -1,128 +0,0 @@
From 0000000000000000000000000000000000000000 Mon Sep 17 00:00:00 2001
From: "Jason R. Coombs" <jaraco@jaraco.com>
Date: Mon, 19 Aug 2024 19:28:20 -0400
Subject: [PATCH] 00436: [CVE-2024-8088] gh-122905: Sanitize names in
zipfile.Path.
Co-authored-by: Jason R. Coombs <jaraco@jaraco.com>
---
Lib/test/test_zipfile.py | 17 ++++++
Lib/zipfile.py | 61 ++++++++++++++++++-
...-08-11-14-08-04.gh-issue-122905.7tDsxA.rst | 1 +
3 files changed, 78 insertions(+), 1 deletion(-)
create mode 100644 Misc/NEWS.d/next/Library/2024-08-11-14-08-04.gh-issue-122905.7tDsxA.rst
diff --git a/Lib/test/test_zipfile.py b/Lib/test/test_zipfile.py
index 17e95eb862..9a72152357 100644
--- a/Lib/test/test_zipfile.py
+++ b/Lib/test/test_zipfile.py
@@ -3054,6 +3054,23 @@ class TestPath(unittest.TestCase):
data = ['/'.join(string.ascii_lowercase + str(n)) for n in range(10000)]
zipfile.CompleteDirs._implied_dirs(data)
+ def test_malformed_paths(self):
+ """
+ Path should handle malformed paths.
+ """
+ data = io.BytesIO()
+ zf = zipfile.ZipFile(data, "w")
+ zf.writestr("/one-slash.txt", b"content")
+ zf.writestr("//two-slash.txt", b"content")
+ zf.writestr("../parent.txt", b"content")
+ zf.filename = ''
+ root = zipfile.Path(zf)
+ assert list(map(str, root.iterdir())) == [
+ 'one-slash.txt',
+ 'two-slash.txt',
+ 'parent.txt',
+ ]
+
if __name__ == "__main__":
unittest.main()
diff --git a/Lib/zipfile.py b/Lib/zipfile.py
index 95f95ee112..2e9b2868cd 100644
--- a/Lib/zipfile.py
+++ b/Lib/zipfile.py
@@ -9,6 +9,7 @@ import io
import itertools
import os
import posixpath
+import re
import shutil
import stat
import struct
@@ -2177,7 +2178,65 @@ def _difference(minuend, subtrahend):
return itertools.filterfalse(set(subtrahend).__contains__, minuend)
-class CompleteDirs(ZipFile):
+class SanitizedNames:
+ """
+ ZipFile mix-in to ensure names are sanitized.
+ """
+
+ def namelist(self):
+ return list(map(self._sanitize, super().namelist()))
+
+ @staticmethod
+ def _sanitize(name):
+ r"""
+ Ensure a relative path with posix separators and no dot names.
+ Modeled after
+ https://github.com/python/cpython/blob/bcc1be39cb1d04ad9fc0bd1b9193d3972835a57c/Lib/zipfile/__init__.py#L1799-L1813
+ but provides consistent cross-platform behavior.
+ >>> san = SanitizedNames._sanitize
+ >>> san('/foo/bar')
+ 'foo/bar'
+ >>> san('//foo.txt')
+ 'foo.txt'
+ >>> san('foo/.././bar.txt')
+ 'foo/bar.txt'
+ >>> san('foo../.bar.txt')
+ 'foo../.bar.txt'
+ >>> san('\\foo\\bar.txt')
+ 'foo/bar.txt'
+ >>> san('D:\\foo.txt')
+ 'D/foo.txt'
+ >>> san('\\\\server\\share\\file.txt')
+ 'server/share/file.txt'
+ >>> san('\\\\?\\GLOBALROOT\\Volume3')
+ '?/GLOBALROOT/Volume3'
+ >>> san('\\\\.\\PhysicalDrive1\\root')
+ 'PhysicalDrive1/root'
+ Retain any trailing slash.
+ >>> san('abc/')
+ 'abc/'
+ Raises a ValueError if the result is empty.
+ >>> san('../..')
+ Traceback (most recent call last):
+ ...
+ ValueError: Empty filename
+ """
+
+ def allowed(part):
+ return part and part not in {'..', '.'}
+
+ # Remove the drive letter.
+ # Don't use ntpath.splitdrive, because that also strips UNC paths
+ bare = re.sub('^([A-Z]):', r'\1', name, flags=re.IGNORECASE)
+ clean = bare.replace('\\', '/')
+ parts = clean.split('/')
+ joined = '/'.join(filter(allowed, parts))
+ if not joined:
+ raise ValueError("Empty filename")
+ return joined + '/' * name.endswith('/')
+
+
+class CompleteDirs(SanitizedNames, ZipFile):
"""
A ZipFile subclass that ensures that implied directories
are always included in the namelist.
diff --git a/Misc/NEWS.d/next/Library/2024-08-11-14-08-04.gh-issue-122905.7tDsxA.rst b/Misc/NEWS.d/next/Library/2024-08-11-14-08-04.gh-issue-122905.7tDsxA.rst
new file mode 100644
index 0000000000..1be44c906c
--- /dev/null
+++ b/Misc/NEWS.d/next/Library/2024-08-11-14-08-04.gh-issue-122905.7tDsxA.rst
@@ -0,0 +1 @@
+:class:`zipfile.Path` objects now sanitize names from the zipfile.

16
SOURCES/Python-3.9.19.tar.xz.asc
View File

@ -1,16 +0,0 @@
-----BEGIN PGP SIGNATURE-----
iQIzBAABCgAdFiEE4/8oOcBIslwITevpsmmV4xAlBWgFAmX5uMIACgkQsmmV4xAl
BWj1tQ//T2qX0m08xWGV7az0D1sH3qjoY+4fEYrknw5uAHqZFiQecRsF27jxv6iH
gP/6GAUw+lbH+9UofhCc0NbPOklliS7gFLNqJdKYFB6JXRNxiRYKh3uVx5o2n0ES
kR3kRl77S47rtCbSMrKTh6ZoWowyIUZGFsIonk5KsLv+oELXY1AK/Im9i3/iTJ1Z
jd/e2oHWuseIxbGZAO8AEP8zOsMMIHfsL3ry8H9xhhPyQM6t5DldqLH3UVE6kq95
fs+olGO4FEKif3VDuLaHVlgtGZOUr6aDIYUmWxctPicboSb6RJAq37CCYgWykOyB
WQec0ONbU7lxt5jhemLSDRy0mEio7+nXIKsO9rDN0Wk1QMpHUl77/C5qVlzfHal7
NhPt8Yl0hBnOjzTq+di+xhAKJcdKp+zZH7/ugAbthuqhNfnkqiF68PANHrCm3gbY
myN0eSaQ9yIa/MbHW8Am9NL/nuFbxdJUL/OIKQ9kFHgD7Qid86TZF0G2vbiBH/eF
IVYoMxRZLd7eu5dIcwXSef+Ai97pODbx9y7bOCFyBO9FuFrlhPObgc7KXCeAzP+y
k5eWvZtWTvvQ+2si2iT22EPBO0D0pnhYWZKpGK5EuKuw8nasNS1yLbhDTVpARynd
8buQh3t2wPfILlQr0+JzDY8GSdQ/nIHGgx2IERdSX/v+9Yo2AvU=
=gYAl
-----END PGP SIGNATURE-----

16
SOURCES/Python-3.9.20.tar.xz.asc Normal file
View File

@ -0,0 +1,16 @@
-----BEGIN PGP SIGNATURE-----
iQIzBAABCAAdFiEE4/8oOcBIslwITevpsmmV4xAlBWgFAmbcKf0ACgkQsmmV4xAl
BWh4rg//R5E1EjsifYqhLeIyT+JnrBvbTZeEcdxPXevsgilojYmrxBUKuXXViul0
YZFaoDf6wjbHh6NMNgUpqcOH/5S/LsFZvuEcrw0jyGlMr0AMA4KLmNvQ9Wxf+wp4
mUmhymQx555nVivsdPiziNnDwubZeA870ZllYEMWP5vXw7p2LbnlZvn7A+LSKjqM
S/6xbiKYVexK3vHY/uG0xo4z24FySfvs0/PF11JfRJCxm9+bli7FmHOoFMwpOO6S
caZLok4987YWOcPIPY6h+o2sFhDqHs8POGKd8k+0KQNQs5UbEQ4t/eKgnaoATkGn
nfcAGXSjX5RSv5uXPzBUc0PulYo6EalIn1b5fu96La/FEg9GLMR/n9g75Fgm/j9L
QGYu/DSaastY/c7Ot4QVyB6pxbQKjM438yneQrjhKBILGla4Crh1k6yRCx93j/TH
hF9kiuRf7jtLIGTp0cnquELGnatmL1RhOySn/1Y+asMR+oK8d+XQab//w4VsAt7C
SIfVXg25PUgZoaiYj/qIjLK9vkcj/EZ1IacivP5qBWb3O1E8gzSV8Z9duGT8Ef3P
ch4M/pd6hefVVVfyCoazB3gwDs68O6U2BIRdYLRlet8AuKTBysQKFwOo3EcCMmJV
W20KutPnERCzt8jeJdzFd0z3po9mvxNTKDLYaABtNI6NN00LcsM=
=svjf
-----END PGP SIGNATURE-----

42
SPECS/python39.spec
View File

@ -13,11 +13,11 @@ URL: https://www.python.org/
# WARNING When rebasing to a new Python version,
# remember to update the python3-docs package as well
%global general_version %{pybasever}.19
%global general_version %{pybasever}.20
#global prerel ...
%global upstream_version %{general_version}%{?prerel}
Version: %{general_version}%{?prerel:~%{prerel}}
Release: 7%{?dist}
Release: 1%{?dist}
License: Python
# Exclude i686 arch. Due to a modularity issue it's being added to the
@ -448,7 +448,7 @@ Patch414: 00414-skip_test_zlib_s390x.patch
#
# Upstream PR: https://github.com/python/cpython/pull/111116
#
# Second patch implmenets the possibility to restore the old behavior via
# This patch implements the possibility to restore the old behavior via
# config file or environment variable.
Patch415: 00415-cve-2023-27043-gh-102988-reject-malformed-addresses-in-email-parseaddr-111116.patch
@ -459,35 +459,6 @@ Patch415: 00415-cve-2023-27043-gh-102988-reject-malformed-addresses-in-email-par
# CVE-2023-52425. Future versions of Expat may be more reactive.
Patch422: 00422-fix-tests-for-xmlpullparser-with-expat-2-6-0.patch
# 00431 #
# Security fix for CVE-2024-4032: incorrect IPv4 and IPv6 private ranges
# Resolved upstream: https://github.com/python/cpython/issues/113171
# Tracking bugzilla: https://bugzilla.redhat.com/show_bug.cgi?id=2292921
Patch431: 00431-CVE-2024-4032.patch
# 00435 # f2924d30f4dd44804219c10410a57dd96764d297
# gh-121650: Encode newlines in headers, and verify headers are sound (GH-122233)
#
# Per RFC 2047:
#
# > [...] these encoding schemes allow the
# > encoding of arbitrary octet values, mail readers that implement this
# > decoding should also ensure that display of the decoded data on the
# > recipient's terminal will not cause unwanted side-effects
#
# It seems that the "quoted-word" scheme is a valid way to include
# a newline character in a header value, just like we already allow
# undecodable bytes or control characters.
# They do need to be properly quoted when serialized to text, though.
#
# This should fail for custom fold() implementations that aren't careful
# about newlines.
Patch435: 00435-gh-121650-encode-newlines-in-headers-and-verify-headers-are-sound-gh-122233.patch
# 00436 # 506dd77b7132f69ada7185b8bb91eba0e1296aa8
# [CVE-2024-8088] gh-122905: Sanitize names in zipfile.Path.
Patch436: 00436-cve-2024-8088-gh-122905-sanitize-names-in-zipfile-path.patch
# (New patches go here ^^^)
#
# When adding new patches to "python" and "python3" in Fedora, EL, etc.,
@ -903,9 +874,6 @@ rm Lib/ensurepip/_bundled/*.whl
%apply_patch -q %{PATCH414}
%apply_patch -q %{PATCH415}
%apply_patch -q %{PATCH422}
%apply_patch -q %{PATCH431}
%apply_patch -q %{PATCH435}
%apply_patch -q %{PATCH436}
# Remove all exe files to ensure we are not shipping prebuilt binaries
# note that those are only used to create Microsoft Windows installers
@ -2080,6 +2048,10 @@ fi
# ======================================================
%changelog
* Mon Sep 09 2024 Tomáš Hrnčiar <thrnciar@redhat.com> - 3.9.20-1
- Update to 3.9.20
Resolves: RHEL-60007
* Fri Aug 23 2024 Charalampos Stratakis <cstratak@redhat.com> - 3.9.19-7
- Security fix for CVE-2024-8088
Resolves: RHEL-55954