Compare commits
18 Commits
imports/c8
...
c8
Author | SHA1 | Date |
---|---|---|
eabdullin | d0c83e320b | |
eabdullin | c51dc1346c | |
eabdullin | a37af4d525 | |
eabdullin | 346df28398 | |
eabdullin | 1563e29a86 | |
eabdullin | c3b837042b | |
Andrew Lukoshko | b140357418 | |
Andrew Lukoshko | a4d48ab5dc | |
CentOS Sources | 6ac83c33f3 | |
CentOS Sources | d01f487196 | |
CentOS Sources | 93994c4d72 | |
CentOS Sources | 69319ca074 | |
CentOS Sources | a62df8a19c | |
CentOS Sources | 3a450dae9d | |
CentOS Sources | d72d9cb256 | |
CentOS Sources | b2efcd69f9 | |
CentOS Sources | 7c0aac331f | |
CentOS Sources | 87132fa408 |
|
@ -200,3 +200,19 @@ index 0f2dfc4..da37896 100644
|
|||
] )
|
||||
|
||||
if (ssl_incs is not None and
|
||||
diff --git a/configure.ac b/configure.ac
|
||||
index 01c66fe..1e6d515 100644
|
||||
--- a/configure.ac
|
||||
+++ b/configure.ac
|
||||
@@ -4772,9 +4772,9 @@ AC_MSG_RESULT($LDVERSION)
|
||||
dnl define LIBPL after ABIFLAGS and LDVERSION is defined.
|
||||
AC_SUBST(PY_ENABLE_SHARED)
|
||||
if test x$PLATFORM_TRIPLET = x; then
|
||||
- LIBPL='$(prefix)'"/lib/python${VERSION}/config-${LDVERSION}"
|
||||
+ LIBPL='$(prefix)'"/lib64/python${VERSION}/config-${LDVERSION}"
|
||||
else
|
||||
- LIBPL='$(prefix)'"/lib/python${VERSION}/config-${LDVERSION}-${PLATFORM_TRIPLET}"
|
||||
+ LIBPL='$(prefix)'"/lib64/python${VERSION}/config-${LDVERSION}-${PLATFORM_TRIPLET}"
|
||||
fi
|
||||
AC_SUBST(LIBPL)
|
||||
|
||||
|
|
|
@ -1,30 +0,0 @@
|
|||
diff -r 39b9b05c3085 Lib/distutils/sysconfig.py
|
||||
--- a/Lib/distutils/sysconfig.py Wed Apr 10 00:27:23 2013 +0200
|
||||
+++ b/Lib/distutils/sysconfig.py Wed Apr 10 10:14:18 2013 +0200
|
||||
@@ -362,7 +362,10 @@
|
||||
done[n] = item = ""
|
||||
if found:
|
||||
after = value[m.end():]
|
||||
- value = value[:m.start()] + item + after
|
||||
+ value = value[:m.start()]
|
||||
+ if item.strip() not in value:
|
||||
+ value += item
|
||||
+ value += after
|
||||
if "$" in after:
|
||||
notdone[name] = value
|
||||
else:
|
||||
diff -r 39b9b05c3085 Lib/sysconfig.py
|
||||
--- a/Lib/sysconfig.py Wed Apr 10 00:27:23 2013 +0200
|
||||
+++ b/Lib/sysconfig.py Wed Apr 10 10:14:18 2013 +0200
|
||||
@@ -296,7 +296,10 @@
|
||||
|
||||
if found:
|
||||
after = value[m.end():]
|
||||
- value = value[:m.start()] + item + after
|
||||
+ value = value[:m.start()]
|
||||
+ if item.strip() not in value:
|
||||
+ value += item
|
||||
+ value += after
|
||||
if "$" in after:
|
||||
notdone[name] = value
|
||||
else:
|
|
@ -1,12 +0,0 @@
|
|||
diff -up Python-3.5.0/Makefile.pre.in.lib Python-3.5.0/Makefile.pre.in
|
||||
--- Python-3.5.0/Makefile.pre.in.lib 2015-09-21 15:39:47.928286620 +0200
|
||||
+++ Python-3.5.0/Makefile.pre.in 2015-09-21 15:42:58.004042762 +0200
|
||||
@@ -1340,7 +1340,7 @@ inclinstall:
|
||||
|
||||
# Install the library and miscellaneous stuff needed for extending/embedding
|
||||
# This goes into $(exec_prefix)
|
||||
-LIBPL= @LIBPL@
|
||||
+LIBPL= $(LIBDEST)/config-$(LDVERSION)-$(MULTIARCH)
|
||||
|
||||
# pkgconfig directory
|
||||
LIBPC= $(LIBDIR)/pkgconfig
|
|
@ -0,0 +1,40 @@
|
|||
diff --git a/Lib/threading.py b/Lib/threading.py
|
||||
index 7ab9ad8..dcedd3b 100644
|
||||
--- a/Lib/threading.py
|
||||
+++ b/Lib/threading.py
|
||||
@@ -3,7 +3,7 @@
|
||||
import sys as _sys
|
||||
import _thread
|
||||
|
||||
-from time import monotonic as _time
|
||||
+from time import monotonic as _time, sleep as _sleep
|
||||
from traceback import format_exc as _format_exc
|
||||
from _weakrefset import WeakSet
|
||||
from itertools import islice as _islice, count as _count
|
||||
@@ -296,7 +296,25 @@ class Condition:
|
||||
gotit = True
|
||||
else:
|
||||
if timeout > 0:
|
||||
- gotit = waiter.acquire(True, timeout)
|
||||
+ # rhbz#2003758: Avoid waiter.acquire(True, timeout) since
|
||||
+ # it uses the system clock internally.
|
||||
+ #
|
||||
+ # Balancing act: We can't afford a pure busy loop, so we
|
||||
+ # have to sleep; but if we sleep the whole timeout time,
|
||||
+ # we'll be unresponsive. The scheme here sleeps very
|
||||
+ # little at first, longer as time goes on, but never longer
|
||||
+ # than 20 times per second (or the timeout time remaining).
|
||||
+ endtime = _time() + timeout
|
||||
+ delay = 0.0005 # 500 us -> initial delay of 1 ms
|
||||
+ while True:
|
||||
+ gotit = waiter.acquire(0)
|
||||
+ if gotit:
|
||||
+ break
|
||||
+ remaining = min(endtime - _time(), timeout)
|
||||
+ if remaining <= 0:
|
||||
+ break
|
||||
+ delay = min(delay * 2, remaining, .05)
|
||||
+ _sleep(delay)
|
||||
else:
|
||||
gotit = waiter.acquire(False)
|
||||
return gotit
|
|
@ -1,58 +0,0 @@
|
|||
diff -up Python-3.5.0/configure.ac.than Python-3.5.0/configure.ac
|
||||
--- Python-3.5.0/configure.ac.than 2015-11-13 11:51:32.039560172 -0500
|
||||
+++ Python-3.5.0/configure.ac 2015-11-13 11:52:11.670168157 -0500
|
||||
@@ -788,9 +788,9 @@ cat >> conftest.c <<EOF
|
||||
alpha-linux-gnu
|
||||
# elif defined(__ARM_EABI__) && defined(__ARM_PCS_VFP)
|
||||
# if defined(__ARMEL__)
|
||||
- arm-linux-gnueabihf
|
||||
+ arm-linux-gnueabi
|
||||
# else
|
||||
- armeb-linux-gnueabihf
|
||||
+ armeb-linux-gnueabi
|
||||
# endif
|
||||
# elif defined(__ARM_EABI__) && !defined(__ARM_PCS_VFP)
|
||||
# if defined(__ARMEL__)
|
||||
@@ -810,7 +810,7 @@ cat >> conftest.c <<EOF
|
||||
# elif _MIPS_SIM == _ABIN32
|
||||
mips64el-linux-gnuabin32
|
||||
# elif _MIPS_SIM == _ABI64
|
||||
- mips64el-linux-gnuabi64
|
||||
+ mips64el-linux-gnu
|
||||
# else
|
||||
# error unknown platform triplet
|
||||
# endif
|
||||
@@ -820,7 +820,7 @@ cat >> conftest.c <<EOF
|
||||
# elif _MIPS_SIM == _ABIN32
|
||||
mips64-linux-gnuabin32
|
||||
# elif _MIPS_SIM == _ABI64
|
||||
- mips64-linux-gnuabi64
|
||||
+ mips64-linux-gnu
|
||||
# else
|
||||
# error unknown platform triplet
|
||||
# endif
|
||||
@@ -830,9 +830,9 @@ cat >> conftest.c <<EOF
|
||||
powerpc-linux-gnuspe
|
||||
# elif defined(__powerpc64__)
|
||||
# if defined(__LITTLE_ENDIAN__)
|
||||
- powerpc64le-linux-gnu
|
||||
+ ppc64le-linux-gnu
|
||||
# else
|
||||
- powerpc64-linux-gnu
|
||||
+ ppc64-linux-gnu
|
||||
# endif
|
||||
# elif defined(__powerpc__)
|
||||
powerpc-linux-gnu
|
||||
diff --git a/config.sub b/config.sub
|
||||
index 40ea5df..932128b 100755
|
||||
--- a/config.sub
|
||||
+++ b/config.sub
|
||||
@@ -1045,7 +1045,7 @@ case $basic_machine in
|
||||
;;
|
||||
ppc64) basic_machine=powerpc64-unknown
|
||||
;;
|
||||
- ppc64-*) basic_machine=powerpc64-`echo $basic_machine | sed 's/^[^-]*-//'`
|
||||
+ ppc64-* | ppc64p7-*) basic_machine=powerpc64-`echo $basic_machine | sed 's/^[^-]*-//'`
|
||||
;;
|
||||
ppc64le | powerpc64little)
|
||||
basic_machine=powerpc64le-unknown
|
File diff suppressed because it is too large
Load Diff
|
@ -0,0 +1,22 @@
|
|||
diff --git a/Lib/test/test_gdb.py b/Lib/test/test_gdb.py
|
||||
index 9c15fca..c972409 100644
|
||||
--- a/Lib/test/test_gdb.py
|
||||
+++ b/Lib/test/test_gdb.py
|
||||
@@ -279,8 +279,15 @@ class DebuggerTests(unittest.TestCase):
|
||||
# gdb can insert additional '\n' and space characters in various places
|
||||
# in its output, depending on the width of the terminal it's connected
|
||||
# to (using its "wrap_here" function)
|
||||
- m = re.match(r'.*#0\s+builtin_id\s+\(self\=.*,\s+v=\s*(.*?)\)\s+at\s+\S*Python/bltinmodule.c.*',
|
||||
- gdb_output, re.DOTALL)
|
||||
+ m = re.search(
|
||||
+ # Match '#0 builtin_id(self=..., v=...)'
|
||||
+ r'#0\s+builtin_id\s+\(self\=.*,\s+v=\s*(.*?)?\)'
|
||||
+ # Match ' at Python/bltinmodule.c'.
|
||||
+ # bpo-38239: builtin_id() is defined in Python/bltinmodule.c,
|
||||
+ # but accept any "Directory\file.c" to support Link Time
|
||||
+ # Optimization (LTO).
|
||||
+ r'\s+at\s+\S*[A-Za-z]+/[A-Za-z0-9_-]+\.c',
|
||||
+ gdb_output, re.DOTALL)
|
||||
if not m:
|
||||
self.fail('Unexpected gdb output: %r\n%s' % (gdb_output, gdb_output))
|
||||
return m.group(1), gdb_output
|
|
@ -0,0 +1,54 @@
|
|||
diff --git a/Lib/test/test_docxmlrpc.py b/Lib/test/test_docxmlrpc.py
|
||||
index 0090333..d2adb21 100644
|
||||
--- a/Lib/test/test_docxmlrpc.py
|
||||
+++ b/Lib/test/test_docxmlrpc.py
|
||||
@@ -1,5 +1,6 @@
|
||||
from xmlrpc.server import DocXMLRPCServer
|
||||
import http.client
|
||||
+import re
|
||||
import sys
|
||||
from test import support
|
||||
threading = support.import_module('threading')
|
||||
@@ -193,6 +194,21 @@ class DocXMLRPCHTTPGETServer(unittest.TestCase):
|
||||
b'method_annotation</strong></a>(x: bytes)</dt></dl>'),
|
||||
response.read())
|
||||
|
||||
+ def test_server_title_escape(self):
|
||||
+ # bpo-38243: Ensure that the server title and documentation
|
||||
+ # are escaped for HTML.
|
||||
+ self.serv.set_server_title('test_title<script>')
|
||||
+ self.serv.set_server_documentation('test_documentation<script>')
|
||||
+ self.assertEqual('test_title<script>', self.serv.server_title)
|
||||
+ self.assertEqual('test_documentation<script>',
|
||||
+ self.serv.server_documentation)
|
||||
+
|
||||
+ generated = self.serv.generate_html_documentation()
|
||||
+ title = re.search(r'<title>(.+?)</title>', generated).group()
|
||||
+ documentation = re.search(r'<p><tt>(.+?)</tt></p>', generated).group()
|
||||
+ self.assertEqual('<title>Python: test_title<script></title>', title)
|
||||
+ self.assertEqual('<p><tt>test_documentation<script></tt></p>', documentation)
|
||||
+
|
||||
|
||||
if __name__ == '__main__':
|
||||
unittest.main()
|
||||
diff --git a/Lib/xmlrpc/server.py b/Lib/xmlrpc/server.py
|
||||
index 3e0dca0..efe5937 100644
|
||||
--- a/Lib/xmlrpc/server.py
|
||||
+++ b/Lib/xmlrpc/server.py
|
||||
@@ -106,6 +106,7 @@ server.handle_request()
|
||||
|
||||
from xmlrpc.client import Fault, dumps, loads, gzip_encode, gzip_decode
|
||||
from http.server import BaseHTTPRequestHandler
|
||||
+import html
|
||||
import http.server
|
||||
import socketserver
|
||||
import sys
|
||||
@@ -904,7 +905,7 @@ class XMLRPCDocGenerator:
|
||||
methods
|
||||
)
|
||||
|
||||
- return documenter.page(self.server_title, documentation)
|
||||
+ return documenter.page(html.escape(self.server_title), documentation)
|
||||
|
||||
class DocXMLRPCRequestHandler(SimpleXMLRPCRequestHandler):
|
||||
"""XML-RPC and documentation request handler class.
|
|
@ -0,0 +1,44 @@
|
|||
diff --git a/Lib/test/test_site.py b/Lib/test/test_site.py
|
||||
index d0cd84f..9d2c28c 100644
|
||||
--- a/Lib/test/test_site.py
|
||||
+++ b/Lib/test/test_site.py
|
||||
@@ -10,6 +10,7 @@ from test import support
|
||||
from test.support import (captured_stderr, TESTFN, EnvironmentVarGuard,
|
||||
change_cwd)
|
||||
import builtins
|
||||
+import glob
|
||||
import os
|
||||
import sys
|
||||
import re
|
||||
@@ -500,6 +501,23 @@ class ImportSideEffectTests(unittest.TestCase):
|
||||
class StartupImportTests(unittest.TestCase):
|
||||
|
||||
def test_startup_imports(self):
|
||||
+ # Get sys.path in isolated mode (python3 -I)
|
||||
+ popen = subprocess.Popen([sys.executable, '-I', '-c',
|
||||
+ 'import sys; print(repr(sys.path))'],
|
||||
+ stdout=subprocess.PIPE,
|
||||
+ encoding='utf-8')
|
||||
+ stdout = popen.communicate()[0]
|
||||
+ self.assertEqual(popen.returncode, 0, repr(stdout))
|
||||
+ isolated_paths = eval(stdout)
|
||||
+
|
||||
+ # bpo-27807: Even with -I, the site module executes all .pth files
|
||||
+ # found in sys.path (see site.addpackage()). Skip the test if at least
|
||||
+ # one .pth file is found.
|
||||
+ for path in isolated_paths:
|
||||
+ pth_files = glob.glob(os.path.join(path, "*.pth"))
|
||||
+ if pth_files:
|
||||
+ self.skipTest(f"found {len(pth_files)} .pth files in: {path}")
|
||||
+
|
||||
# This tests checks which modules are loaded by Python when it
|
||||
# initially starts upon startup.
|
||||
popen = subprocess.Popen([sys.executable, '-I', '-v', '-c',
|
||||
@@ -508,6 +526,7 @@ class StartupImportTests(unittest.TestCase):
|
||||
stderr=subprocess.PIPE,
|
||||
encoding='utf-8')
|
||||
stdout, stderr = popen.communicate()
|
||||
+ self.assertEqual(popen.returncode, 0, (stdout, stderr))
|
||||
modules = eval(stdout)
|
||||
|
||||
self.assertIn('site', modules)
|
|
@ -0,0 +1,193 @@
|
|||
diff --git a/Lib/test/test_urllib2.py b/Lib/test/test_urllib2.py
|
||||
index 876fcd4..fe9a32b 100644
|
||||
--- a/Lib/test/test_urllib2.py
|
||||
+++ b/Lib/test/test_urllib2.py
|
||||
@@ -1445,40 +1445,64 @@ class HandlerTests(unittest.TestCase):
|
||||
bypass = {'exclude_simple': True, 'exceptions': []}
|
||||
self.assertTrue(_proxy_bypass_macosx_sysconf('test', bypass))
|
||||
|
||||
- def test_basic_auth(self, quote_char='"'):
|
||||
- opener = OpenerDirector()
|
||||
- password_manager = MockPasswordManager()
|
||||
- auth_handler = urllib.request.HTTPBasicAuthHandler(password_manager)
|
||||
- realm = "ACME Widget Store"
|
||||
- http_handler = MockHTTPHandler(
|
||||
- 401, 'WWW-Authenticate: Basic realm=%s%s%s\r\n\r\n' %
|
||||
- (quote_char, realm, quote_char))
|
||||
- opener.add_handler(auth_handler)
|
||||
- opener.add_handler(http_handler)
|
||||
- self._test_basic_auth(opener, auth_handler, "Authorization",
|
||||
- realm, http_handler, password_manager,
|
||||
- "http://acme.example.com/protected",
|
||||
- "http://acme.example.com/protected",
|
||||
- )
|
||||
-
|
||||
- def test_basic_auth_with_single_quoted_realm(self):
|
||||
- self.test_basic_auth(quote_char="'")
|
||||
-
|
||||
- def test_basic_auth_with_unquoted_realm(self):
|
||||
- opener = OpenerDirector()
|
||||
- password_manager = MockPasswordManager()
|
||||
- auth_handler = urllib.request.HTTPBasicAuthHandler(password_manager)
|
||||
- realm = "ACME Widget Store"
|
||||
- http_handler = MockHTTPHandler(
|
||||
- 401, 'WWW-Authenticate: Basic realm=%s\r\n\r\n' % realm)
|
||||
- opener.add_handler(auth_handler)
|
||||
- opener.add_handler(http_handler)
|
||||
- with self.assertWarns(UserWarning):
|
||||
+ def check_basic_auth(self, headers, realm):
|
||||
+ with self.subTest(realm=realm, headers=headers):
|
||||
+ opener = OpenerDirector()
|
||||
+ password_manager = MockPasswordManager()
|
||||
+ auth_handler = urllib.request.HTTPBasicAuthHandler(password_manager)
|
||||
+ body = '\r\n'.join(headers) + '\r\n\r\n'
|
||||
+ http_handler = MockHTTPHandler(401, body)
|
||||
+ opener.add_handler(auth_handler)
|
||||
+ opener.add_handler(http_handler)
|
||||
self._test_basic_auth(opener, auth_handler, "Authorization",
|
||||
- realm, http_handler, password_manager,
|
||||
- "http://acme.example.com/protected",
|
||||
- "http://acme.example.com/protected",
|
||||
- )
|
||||
+ realm, http_handler, password_manager,
|
||||
+ "http://acme.example.com/protected",
|
||||
+ "http://acme.example.com/protected")
|
||||
+
|
||||
+ def test_basic_auth(self):
|
||||
+ realm = "realm2@example.com"
|
||||
+ realm2 = "realm2@example.com"
|
||||
+ basic = f'Basic realm="{realm}"'
|
||||
+ basic2 = f'Basic realm="{realm2}"'
|
||||
+ other_no_realm = 'Otherscheme xxx'
|
||||
+ digest = (f'Digest realm="{realm2}", '
|
||||
+ f'qop="auth, auth-int", '
|
||||
+ f'nonce="dcd98b7102dd2f0e8b11d0f600bfb0c093", '
|
||||
+ f'opaque="5ccc069c403ebaf9f0171e9517f40e41"')
|
||||
+ for realm_str in (
|
||||
+ # test "quote" and 'quote'
|
||||
+ f'Basic realm="{realm}"',
|
||||
+ f"Basic realm='{realm}'",
|
||||
+
|
||||
+ # charset is ignored
|
||||
+ f'Basic realm="{realm}", charset="UTF-8"',
|
||||
+
|
||||
+ # Multiple challenges per header
|
||||
+ f'{basic}, {basic2}',
|
||||
+ f'{basic}, {other_no_realm}',
|
||||
+ f'{other_no_realm}, {basic}',
|
||||
+ f'{basic}, {digest}',
|
||||
+ f'{digest}, {basic}',
|
||||
+ ):
|
||||
+ headers = [f'WWW-Authenticate: {realm_str}']
|
||||
+ self.check_basic_auth(headers, realm)
|
||||
+
|
||||
+ # no quote: expect a warning
|
||||
+ with support.check_warnings(("Basic Auth Realm was unquoted",
|
||||
+ UserWarning)):
|
||||
+ headers = [f'WWW-Authenticate: Basic realm={realm}']
|
||||
+ self.check_basic_auth(headers, realm)
|
||||
+
|
||||
+ # Multiple headers: one challenge per header.
|
||||
+ # Use the first Basic realm.
|
||||
+ for challenges in (
|
||||
+ [basic, basic2],
|
||||
+ [basic, digest],
|
||||
+ [digest, basic],
|
||||
+ ):
|
||||
+ headers = [f'WWW-Authenticate: {challenge}'
|
||||
+ for challenge in challenges]
|
||||
+ self.check_basic_auth(headers, realm)
|
||||
|
||||
def test_proxy_basic_auth(self):
|
||||
opener = OpenerDirector()
|
||||
diff --git a/Lib/urllib/request.py b/Lib/urllib/request.py
|
||||
index c9945d9..6624e04 100644
|
||||
--- a/Lib/urllib/request.py
|
||||
+++ b/Lib/urllib/request.py
|
||||
@@ -945,8 +945,15 @@ class AbstractBasicAuthHandler:
|
||||
|
||||
# allow for double- and single-quoted realm values
|
||||
# (single quotes are a violation of the RFC, but appear in the wild)
|
||||
- rx = re.compile('(?:.*,)*[ \t]*([^ \t]+)[ \t]+'
|
||||
- 'realm=(["\']?)([^"\']*)\\2', re.I)
|
||||
+ rx = re.compile('(?:^|,)' # start of the string or ','
|
||||
+ '[ \t]*' # optional whitespaces
|
||||
+ '([^ \t]+)' # scheme like "Basic"
|
||||
+ '[ \t]+' # mandatory whitespaces
|
||||
+ # realm=xxx
|
||||
+ # realm='xxx'
|
||||
+ # realm="xxx"
|
||||
+ 'realm=(["\']?)([^"\']*)\\2',
|
||||
+ re.I)
|
||||
|
||||
# XXX could pre-emptively send auth info already accepted (RFC 2617,
|
||||
# end of section 2, and section 1.2 immediately after "credentials"
|
||||
@@ -958,27 +965,51 @@ class AbstractBasicAuthHandler:
|
||||
self.passwd = password_mgr
|
||||
self.add_password = self.passwd.add_password
|
||||
|
||||
+ def _parse_realm(self, header):
|
||||
+ # parse WWW-Authenticate header: accept multiple challenges per header
|
||||
+ found_challenge = False
|
||||
+ for mo in AbstractBasicAuthHandler.rx.finditer(header):
|
||||
+ scheme, quote, realm = mo.groups()
|
||||
+ if quote not in ['"', "'"]:
|
||||
+ warnings.warn("Basic Auth Realm was unquoted",
|
||||
+ UserWarning, 3)
|
||||
+
|
||||
+ yield (scheme, realm)
|
||||
+
|
||||
+ found_challenge = True
|
||||
+
|
||||
+ if not found_challenge:
|
||||
+ if header:
|
||||
+ scheme = header.split()[0]
|
||||
+ else:
|
||||
+ scheme = ''
|
||||
+ yield (scheme, None)
|
||||
+
|
||||
def http_error_auth_reqed(self, authreq, host, req, headers):
|
||||
# host may be an authority (without userinfo) or a URL with an
|
||||
# authority
|
||||
- # XXX could be multiple headers
|
||||
- authreq = headers.get(authreq, None)
|
||||
+ headers = headers.get_all(authreq)
|
||||
+ if not headers:
|
||||
+ # no header found
|
||||
+ return
|
||||
|
||||
- if authreq:
|
||||
- scheme = authreq.split()[0]
|
||||
- if scheme.lower() != 'basic':
|
||||
- raise ValueError("AbstractBasicAuthHandler does not"
|
||||
- " support the following scheme: '%s'" %
|
||||
- scheme)
|
||||
- else:
|
||||
- mo = AbstractBasicAuthHandler.rx.search(authreq)
|
||||
- if mo:
|
||||
- scheme, quote, realm = mo.groups()
|
||||
- if quote not in ['"',"'"]:
|
||||
- warnings.warn("Basic Auth Realm was unquoted",
|
||||
- UserWarning, 2)
|
||||
- if scheme.lower() == 'basic':
|
||||
- return self.retry_http_basic_auth(host, req, realm)
|
||||
+ unsupported = None
|
||||
+ for header in headers:
|
||||
+ for scheme, realm in self._parse_realm(header):
|
||||
+ if scheme.lower() != 'basic':
|
||||
+ unsupported = scheme
|
||||
+ continue
|
||||
+
|
||||
+ if realm is not None:
|
||||
+ # Use the first matching Basic challenge.
|
||||
+ # Ignore following challenges even if they use the Basic
|
||||
+ # scheme.
|
||||
+ return self.retry_http_basic_auth(host, req, realm)
|
||||
+
|
||||
+ if unsupported is not None:
|
||||
+ raise ValueError("AbstractBasicAuthHandler does not "
|
||||
+ "support the following scheme: %r"
|
||||
+ % (scheme,))
|
||||
|
||||
def retry_http_basic_auth(self, host, req, realm):
|
||||
user, pw = self.passwd.find_user_password(realm, host)
|
|
@ -0,0 +1,67 @@
|
|||
From 0000000000000000000000000000000000000000 Mon Sep 17 00:00:00 2001
|
||||
From: "Miss Islington (bot)"
|
||||
<31488909+miss-islington@users.noreply.github.com>
|
||||
Date: Wed, 15 Jul 2020 05:36:36 -0700
|
||||
Subject: [PATCH] 00351: Avoid infinite loop in the tarfile module
|
||||
|
||||
Avoid infinite loop when reading specially crafted TAR files using the tarfile module
|
||||
(CVE-2019-20907).
|
||||
Fixed upstream: https://bugs.python.org/issue39017
|
||||
---
|
||||
Lib/tarfile.py | 2 ++
|
||||
Lib/test/recursion.tar | Bin 0 -> 516 bytes
|
||||
Lib/test/test_tarfile.py | 7 +++++++
|
||||
.../2020-07-12-22-16-58.bpo-39017.x3Cg-9.rst | 1 +
|
||||
4 files changed, 10 insertions(+)
|
||||
create mode 100644 Lib/test/recursion.tar
|
||||
create mode 100644 Misc/NEWS.d/next/Library/2020-07-12-22-16-58.bpo-39017.x3Cg-9.rst
|
||||
|
||||
diff --git a/Lib/tarfile.py b/Lib/tarfile.py
|
||||
index 62d22150f5..2ea47978ff 100755
|
||||
--- a/Lib/tarfile.py
|
||||
+++ b/Lib/tarfile.py
|
||||
@@ -1231,6 +1231,8 @@ class TarInfo(object):
|
||||
|
||||
length, keyword = match.groups()
|
||||
length = int(length)
|
||||
+ if length == 0:
|
||||
+ raise InvalidHeaderError("invalid header")
|
||||
value = buf[match.end(2) + 1:match.start(1) + length - 1]
|
||||
|
||||
# Normally, we could just use "utf-8" as the encoding and "strict"
|
||||
diff --git a/Lib/test/recursion.tar b/Lib/test/recursion.tar
|
||||
new file mode 100644
|
||||
index 0000000000000000000000000000000000000000..b8237251964983f54ed1966297e887636cd0c5f4
|
||||
GIT binary patch
|
||||
literal 516
|
||||
zcmYdFPRz+kEn=W0Fn}74P8%Xw3X=l~85kIuo0>8xq$A1Gm}!7)KUsFc41m#O8A5+e
|
||||
I1_}|j06>QaCIA2c
|
||||
|
||||
literal 0
|
||||
HcmV?d00001
|
||||
|
||||
diff --git a/Lib/test/test_tarfile.py b/Lib/test/test_tarfile.py
|
||||
index 4cd7d5370f..573be812ea 100644
|
||||
--- a/Lib/test/test_tarfile.py
|
||||
+++ b/Lib/test/test_tarfile.py
|
||||
@@ -395,6 +395,13 @@ class CommonReadTest(ReadTest):
|
||||
with self.assertRaisesRegex(tarfile.ReadError, "unexpected end of data"):
|
||||
tar.extractfile(t).read()
|
||||
|
||||
+ def test_length_zero_header(self):
|
||||
+ # bpo-39017 (CVE-2019-20907): reading a zero-length header should fail
|
||||
+ # with an exception
|
||||
+ with self.assertRaisesRegex(tarfile.ReadError, "file could not be opened successfully"):
|
||||
+ with tarfile.open(support.findfile('recursion.tar')) as tar:
|
||||
+ pass
|
||||
+
|
||||
class MiscReadTestBase(CommonReadTest):
|
||||
def requires_name_attribute(self):
|
||||
pass
|
||||
diff --git a/Misc/NEWS.d/next/Library/2020-07-12-22-16-58.bpo-39017.x3Cg-9.rst b/Misc/NEWS.d/next/Library/2020-07-12-22-16-58.bpo-39017.x3Cg-9.rst
|
||||
new file mode 100644
|
||||
index 0000000000..ad26676f8b
|
||||
--- /dev/null
|
||||
+++ b/Misc/NEWS.d/next/Library/2020-07-12-22-16-58.bpo-39017.x3Cg-9.rst
|
||||
@@ -0,0 +1 @@
|
||||
+Avoid infinite loop when reading specially crafted TAR files using the tarfile module (CVE-2019-20907).
|
|
@ -0,0 +1,70 @@
|
|||
From 0000000000000000000000000000000000000000 Mon Sep 17 00:00:00 2001
|
||||
From: Tapas Kundu <39723251+tapakund@users.noreply.github.com>
|
||||
Date: Wed, 1 Jul 2020 01:00:22 +0530
|
||||
Subject: [PATCH] 00352: Resolve hash collisions for IPv4Interface and
|
||||
IPv6Interface
|
||||
|
||||
CVE-2020-14422
|
||||
The hash() methods of classes IPv4Interface and IPv6Interface had issue
|
||||
of generating constant hash values of 32 and 128 respectively causing hash collisions.
|
||||
The fix uses the hash() function to generate hash values for the objects
|
||||
instead of XOR operation.
|
||||
Fixed upstream: https://bugs.python.org/issue41004
|
||||
---
|
||||
Lib/ipaddress.py | 4 ++--
|
||||
Lib/test/test_ipaddress.py | 11 +++++++++++
|
||||
.../Security/2020-06-29-16-02-29.bpo-41004.ovF0KZ.rst | 1 +
|
||||
3 files changed, 14 insertions(+), 2 deletions(-)
|
||||
create mode 100644 Misc/NEWS.d/next/Security/2020-06-29-16-02-29.bpo-41004.ovF0KZ.rst
|
||||
|
||||
diff --git a/Lib/ipaddress.py b/Lib/ipaddress.py
|
||||
index 583f02ad54..98492136ca 100644
|
||||
--- a/Lib/ipaddress.py
|
||||
+++ b/Lib/ipaddress.py
|
||||
@@ -1418,7 +1418,7 @@ class IPv4Interface(IPv4Address):
|
||||
return False
|
||||
|
||||
def __hash__(self):
|
||||
- return self._ip ^ self._prefixlen ^ int(self.network.network_address)
|
||||
+ return hash((self._ip, self._prefixlen, int(self.network.network_address)))
|
||||
|
||||
__reduce__ = _IPAddressBase.__reduce__
|
||||
|
||||
@@ -2092,7 +2092,7 @@ class IPv6Interface(IPv6Address):
|
||||
return False
|
||||
|
||||
def __hash__(self):
|
||||
- return self._ip ^ self._prefixlen ^ int(self.network.network_address)
|
||||
+ return hash((self._ip, self._prefixlen, int(self.network.network_address)))
|
||||
|
||||
__reduce__ = _IPAddressBase.__reduce__
|
||||
|
||||
diff --git a/Lib/test/test_ipaddress.py b/Lib/test/test_ipaddress.py
|
||||
index 1cef4217bc..7de444af4a 100644
|
||||
--- a/Lib/test/test_ipaddress.py
|
||||
+++ b/Lib/test/test_ipaddress.py
|
||||
@@ -1990,6 +1990,17 @@ class IpaddrUnitTest(unittest.TestCase):
|
||||
sixtofouraddr.sixtofour)
|
||||
self.assertFalse(bad_addr.sixtofour)
|
||||
|
||||
+ # issue41004 Hash collisions in IPv4Interface and IPv6Interface
|
||||
+ def testV4HashIsNotConstant(self):
|
||||
+ ipv4_address1 = ipaddress.IPv4Interface("1.2.3.4")
|
||||
+ ipv4_address2 = ipaddress.IPv4Interface("2.3.4.5")
|
||||
+ self.assertNotEqual(ipv4_address1.__hash__(), ipv4_address2.__hash__())
|
||||
+
|
||||
+ # issue41004 Hash collisions in IPv4Interface and IPv6Interface
|
||||
+ def testV6HashIsNotConstant(self):
|
||||
+ ipv6_address1 = ipaddress.IPv6Interface("2001:658:22a:cafe:200:0:0:1")
|
||||
+ ipv6_address2 = ipaddress.IPv6Interface("2001:658:22a:cafe:200:0:0:2")
|
||||
+ self.assertNotEqual(ipv6_address1.__hash__(), ipv6_address2.__hash__())
|
||||
|
||||
if __name__ == '__main__':
|
||||
unittest.main()
|
||||
diff --git a/Misc/NEWS.d/next/Security/2020-06-29-16-02-29.bpo-41004.ovF0KZ.rst b/Misc/NEWS.d/next/Security/2020-06-29-16-02-29.bpo-41004.ovF0KZ.rst
|
||||
new file mode 100644
|
||||
index 0000000000..f5a9db52ff
|
||||
--- /dev/null
|
||||
+++ b/Misc/NEWS.d/next/Security/2020-06-29-16-02-29.bpo-41004.ovF0KZ.rst
|
||||
@@ -0,0 +1 @@
|
||||
+CVE-2020-14422: The __hash__() methods of ipaddress.IPv4Interface and ipaddress.IPv6Interface incorrectly generated constant hash values of 32 and 128 respectively. This resulted in always causing hash collisions. The fix uses hash() to generate hash values for the tuple of (address, mask length, network address).
|
|
@ -0,0 +1,97 @@
|
|||
From 0000000000000000000000000000000000000000 Mon Sep 17 00:00:00 2001
|
||||
From: Lumir Balhar <lbalhar@redhat.com>
|
||||
Date: Tue, 4 Aug 2020 12:04:03 +0200
|
||||
Subject: [PATCH] 00353: Original names for architectures with different names
|
||||
downstream
|
||||
MIME-Version: 1.0
|
||||
Content-Type: text/plain; charset=UTF-8
|
||||
Content-Transfer-Encoding: 8bit
|
||||
|
||||
https://fedoraproject.org/wiki/Changes/Python_Upstream_Architecture_Names
|
||||
|
||||
Pythons in RHEL/Fedora used different names for some architectures
|
||||
than upstream and other distros (for example ppc64 vs. powerpc64).
|
||||
This was patched in patch 274, now it is sedded if %with legacy_archnames.
|
||||
|
||||
That meant that an extension built with the default upstream settings
|
||||
(on other distro or as an manylinux wheel) could not been found by Python
|
||||
on RHEL/Fedora because it had a different suffix.
|
||||
This patch adds the legacy names to importlib so Python is able
|
||||
to import extensions with a legacy architecture name in its
|
||||
file name.
|
||||
It work both ways, so it support both %with and %without legacy_archnames.
|
||||
|
||||
WARNING: This patch has no effect on Python built with bootstrap
|
||||
enabled because Python/importlib_external.h is not regenerated
|
||||
and therefore Python during bootstrap contains importlib from
|
||||
upstream without this feature. It's possible to include
|
||||
Python/importlib_external.h to this patch but it'd make rebasing
|
||||
a nightmare because it's basically a binary file.
|
||||
|
||||
Co-authored-by: Miro Hrončok <miro@hroncok.cz>
|
||||
---
|
||||
Lib/importlib/_bootstrap_external.py | 40 ++++++++++++++++++++++++++--
|
||||
1 file changed, 38 insertions(+), 2 deletions(-)
|
||||
|
||||
diff --git a/Lib/importlib/_bootstrap_external.py b/Lib/importlib/_bootstrap_external.py
|
||||
index 9feec50842..5bb2454a5c 100644
|
||||
--- a/Lib/importlib/_bootstrap_external.py
|
||||
+++ b/Lib/importlib/_bootstrap_external.py
|
||||
@@ -1361,7 +1361,7 @@ def _get_supported_file_loaders():
|
||||
|
||||
Each item is a tuple (loader, suffixes).
|
||||
"""
|
||||
- extensions = ExtensionFileLoader, _imp.extension_suffixes()
|
||||
+ extensions = ExtensionFileLoader, _alternative_architectures(_imp.extension_suffixes())
|
||||
source = SourceFileLoader, SOURCE_SUFFIXES
|
||||
bytecode = SourcelessFileLoader, BYTECODE_SUFFIXES
|
||||
return [extensions, source, bytecode]
|
||||
@@ -1428,7 +1428,7 @@ def _setup(_bootstrap_module):
|
||||
|
||||
# Constants
|
||||
setattr(self_module, '_relax_case', _make_relax_case())
|
||||
- EXTENSION_SUFFIXES.extend(_imp.extension_suffixes())
|
||||
+ EXTENSION_SUFFIXES.extend(_alternative_architectures(_imp.extension_suffixes()))
|
||||
if builtin_os == 'nt':
|
||||
SOURCE_SUFFIXES.append('.pyw')
|
||||
if '_d.pyd' in EXTENSION_SUFFIXES:
|
||||
@@ -1441,3 +1441,39 @@ def _install(_bootstrap_module):
|
||||
supported_loaders = _get_supported_file_loaders()
|
||||
sys.path_hooks.extend([FileFinder.path_hook(*supported_loaders)])
|
||||
sys.meta_path.append(PathFinder)
|
||||
+
|
||||
+
|
||||
+_ARCH_MAP = {
|
||||
+ "-arm-linux-gnueabi.": "-arm-linux-gnueabihf.",
|
||||
+ "-armeb-linux-gnueabi.": "-armeb-linux-gnueabihf.",
|
||||
+ "-mips64-linux-gnu.": "-mips64-linux-gnuabi64.",
|
||||
+ "-mips64el-linux-gnu.": "-mips64el-linux-gnuabi64.",
|
||||
+ "-ppc-linux-gnu.": "-powerpc-linux-gnu.",
|
||||
+ "-ppc-linux-gnuspe.": "-powerpc-linux-gnuspe.",
|
||||
+ "-ppc64-linux-gnu.": "-powerpc64-linux-gnu.",
|
||||
+ "-ppc64le-linux-gnu.": "-powerpc64le-linux-gnu.",
|
||||
+ # The above, but the other way around:
|
||||
+ "-arm-linux-gnueabihf.": "-arm-linux-gnueabi.",
|
||||
+ "-armeb-linux-gnueabihf.": "-armeb-linux-gnueabi.",
|
||||
+ "-mips64-linux-gnuabi64.": "-mips64-linux-gnu.",
|
||||
+ "-mips64el-linux-gnuabi64.": "-mips64el-linux-gnu.",
|
||||
+ "-powerpc-linux-gnu.": "-ppc-linux-gnu.",
|
||||
+ "-powerpc-linux-gnuspe.": "-ppc-linux-gnuspe.",
|
||||
+ "-powerpc64-linux-gnu.": "-ppc64-linux-gnu.",
|
||||
+ "-powerpc64le-linux-gnu.": "-ppc64le-linux-gnu.",
|
||||
+}
|
||||
+
|
||||
+
|
||||
+def _alternative_architectures(suffixes):
|
||||
+ """Add a suffix with an alternative architecture name
|
||||
+ to the list of suffixes so an extension built with
|
||||
+ the default (upstream) setting is loadable with our Pythons
|
||||
+ """
|
||||
+
|
||||
+ for suffix in suffixes:
|
||||
+ for original, alternative in _ARCH_MAP.items():
|
||||
+ if original in suffix:
|
||||
+ suffixes.append(suffix.replace(original, alternative))
|
||||
+ return suffixes
|
||||
+
|
||||
+ return suffixes
|
|
@ -0,0 +1,73 @@
|
|||
diff --git a/Lib/http/client.py b/Lib/http/client.py
|
||||
index f0d2642..0a044e9 100644
|
||||
--- a/Lib/http/client.py
|
||||
+++ b/Lib/http/client.py
|
||||
@@ -151,6 +151,10 @@ _contains_disallowed_url_pchar_re = re.compile('[\x00-\x20\x7f]')
|
||||
# _is_allowed_url_pchars_re = re.compile(r"^[/!$&'()*+,;=:@%a-zA-Z0-9._~-]+$")
|
||||
# We are more lenient for assumed real world compatibility purposes.
|
||||
|
||||
+# These characters are not allowed within HTTP method names
|
||||
+# to prevent http header injection.
|
||||
+_contains_disallowed_method_pchar_re = re.compile('[\x00-\x1f]')
|
||||
+
|
||||
# We always set the Content-Length header for these methods because some
|
||||
# servers will otherwise respond with a 411
|
||||
_METHODS_EXPECTING_BODY = {'PATCH', 'POST', 'PUT'}
|
||||
@@ -1117,6 +1121,8 @@ class HTTPConnection:
|
||||
else:
|
||||
raise CannotSendRequest(self.__state)
|
||||
|
||||
+ self._validate_method(method)
|
||||
+
|
||||
# Save the method we use, we need it later in the response phase
|
||||
self._method = method
|
||||
if not url:
|
||||
@@ -1207,6 +1213,15 @@ class HTTPConnection:
|
||||
# For HTTP/1.0, the server will assume "not chunked"
|
||||
pass
|
||||
|
||||
+ def _validate_method(self, method):
|
||||
+ """Validate a method name for putrequest."""
|
||||
+ # prevent http header injection
|
||||
+ match = _contains_disallowed_method_pchar_re.search(method)
|
||||
+ if match:
|
||||
+ raise ValueError(
|
||||
+ f"method can't contain control characters. {method!r} "
|
||||
+ f"(found at least {match.group()!r})")
|
||||
+
|
||||
def putheader(self, header, *values):
|
||||
"""Send a request header line to the server.
|
||||
|
||||
diff --git a/Lib/test/test_httplib.py b/Lib/test/test_httplib.py
|
||||
index 5795b7a..af0350f 100644
|
||||
--- a/Lib/test/test_httplib.py
|
||||
+++ b/Lib/test/test_httplib.py
|
||||
@@ -359,6 +359,28 @@ class HeaderTests(TestCase):
|
||||
self.assertEqual(lines[2], "header: Second: val")
|
||||
|
||||
|
||||
+class HttpMethodTests(TestCase):
|
||||
+ def test_invalid_method_names(self):
|
||||
+ methods = (
|
||||
+ 'GET\r',
|
||||
+ 'POST\n',
|
||||
+ 'PUT\n\r',
|
||||
+ 'POST\nValue',
|
||||
+ 'POST\nHOST:abc',
|
||||
+ 'GET\nrHost:abc\n',
|
||||
+ 'POST\rRemainder:\r',
|
||||
+ 'GET\rHOST:\n',
|
||||
+ '\nPUT'
|
||||
+ )
|
||||
+
|
||||
+ for method in methods:
|
||||
+ with self.assertRaisesRegex(
|
||||
+ ValueError, "method can't contain control characters"):
|
||||
+ conn = client.HTTPConnection('example.com')
|
||||
+ conn.sock = FakeSocket(None)
|
||||
+ conn.request(method=method, url="/")
|
||||
+
|
||||
+
|
||||
class TransferEncodingTest(TestCase):
|
||||
expected_body = b"It's just a flesh wound"
|
||||
|
|
@ -0,0 +1,42 @@
|
|||
diff --git a/Lib/test/multibytecodec_support.py b/Lib/test/multibytecodec_support.py
|
||||
index f9884c6..98feec2 100644
|
||||
--- a/Lib/test/multibytecodec_support.py
|
||||
+++ b/Lib/test/multibytecodec_support.py
|
||||
@@ -300,29 +300,23 @@ class TestBase_Mapping(unittest.TestCase):
|
||||
self._test_mapping_file_plain()
|
||||
|
||||
def _test_mapping_file_plain(self):
|
||||
- unichrs = lambda s: ''.join(map(chr, map(eval, s.split('+'))))
|
||||
+ def unichrs(s):
|
||||
+ return ''.join(chr(int(x, 16)) for x in s.split('+'))
|
||||
+
|
||||
urt_wa = {}
|
||||
|
||||
with self.open_mapping_file() as f:
|
||||
for line in f:
|
||||
if not line:
|
||||
break
|
||||
- data = line.split('#')[0].strip().split()
|
||||
+ data = line.split('#')[0].split()
|
||||
if len(data) != 2:
|
||||
continue
|
||||
|
||||
- csetval = eval(data[0])
|
||||
- if csetval <= 0x7F:
|
||||
- csetch = bytes([csetval & 0xff])
|
||||
- elif csetval >= 0x1000000:
|
||||
- csetch = bytes([(csetval >> 24), ((csetval >> 16) & 0xff),
|
||||
- ((csetval >> 8) & 0xff), (csetval & 0xff)])
|
||||
- elif csetval >= 0x10000:
|
||||
- csetch = bytes([(csetval >> 16), ((csetval >> 8) & 0xff),
|
||||
- (csetval & 0xff)])
|
||||
- elif csetval >= 0x100:
|
||||
- csetch = bytes([(csetval >> 8), (csetval & 0xff)])
|
||||
- else:
|
||||
+ if data[0][:2] != '0x':
|
||||
+ self.fail(f"Invalid line: {line!r}")
|
||||
+ csetch = bytes.fromhex(data[0][2:])
|
||||
+ if len(csetch) == 1 and 0x80 <= csetch[0]:
|
||||
continue
|
||||
|
||||
unich = unichrs(data[1])
|
|
@ -0,0 +1,269 @@
|
|||
From 0cfd9a7f26488567b9a3e5ec192099a8b80ad9df Mon Sep 17 00:00:00 2001
|
||||
From: Lumir Balhar <lbalhar@redhat.com>
|
||||
Date: Tue, 19 Jan 2021 07:55:37 +0100
|
||||
Subject: [PATCH] [PATCH] bpo-37064: Add -k and -a options to pathfix.py tool
|
||||
(GH-16387)
|
||||
|
||||
* bpo-37064: Add option -k to Tools/scripts/pathfix.py (GH-15548)
|
||||
|
||||
Add flag -k to pathscript.py script: preserve shebang flags.
|
||||
|
||||
(cherry picked from commit 50254ac4c179cb412e90682098c97db786143929)
|
||||
|
||||
* bpo-37064: Add option -a to pathfix.py tool (GH-15717)
|
||||
|
||||
Add option -a to Tools/Scripts/pathfix.py script: add flags.
|
||||
|
||||
(cherry picked from commit 1dc1acbd73f05f14c974b7ce1041787d7abef31e)
|
||||
---
|
||||
Lib/test/test_tools/test_pathfix.py | 104 ++++++++++++++++++++++++++++
|
||||
Tools/scripts/pathfix.py | 64 +++++++++++++++--
|
||||
2 files changed, 163 insertions(+), 5 deletions(-)
|
||||
create mode 100644 Lib/test/test_tools/test_pathfix.py
|
||||
|
||||
diff --git a/Lib/test/test_tools/test_pathfix.py b/Lib/test/test_tools/test_pathfix.py
|
||||
new file mode 100644
|
||||
index 0000000..1f0585e
|
||||
--- /dev/null
|
||||
+++ b/Lib/test/test_tools/test_pathfix.py
|
||||
@@ -0,0 +1,104 @@
|
||||
+import os
|
||||
+import subprocess
|
||||
+import sys
|
||||
+import unittest
|
||||
+from test import support
|
||||
+from test.test_tools import import_tool, scriptsdir
|
||||
+
|
||||
+
|
||||
+class TestPathfixFunctional(unittest.TestCase):
|
||||
+ script = os.path.join(scriptsdir, 'pathfix.py')
|
||||
+
|
||||
+ def setUp(self):
|
||||
+ self.temp_file = support.TESTFN
|
||||
+ self.addCleanup(support.unlink, support.TESTFN)
|
||||
+
|
||||
+ def pathfix(self, shebang, pathfix_flags, exitcode=0, stdout='', stderr=''):
|
||||
+ with open(self.temp_file, 'w', encoding='utf8') as f:
|
||||
+ f.write(f'{shebang}\n' + 'print("Hello world")\n')
|
||||
+
|
||||
+ proc = subprocess.run(
|
||||
+ [sys.executable, self.script,
|
||||
+ *pathfix_flags, '-n', self.temp_file],
|
||||
+ universal_newlines=True, stdout=subprocess.PIPE,
|
||||
+ stderr=subprocess.PIPE)
|
||||
+
|
||||
+ if stdout == '' and proc.returncode == 0:
|
||||
+ stdout = f'{self.temp_file}: updating\n'
|
||||
+ self.assertEqual(proc.returncode, exitcode, proc)
|
||||
+ self.assertEqual(proc.stdout, stdout, proc)
|
||||
+ self.assertEqual(proc.stderr, stderr, proc)
|
||||
+
|
||||
+ with open(self.temp_file, 'r', encoding='utf8') as f:
|
||||
+ output = f.read()
|
||||
+
|
||||
+ lines = output.split('\n')
|
||||
+ self.assertEqual(lines[1:], ['print("Hello world")', ''])
|
||||
+ new_shebang = lines[0]
|
||||
+
|
||||
+ if proc.returncode != 0:
|
||||
+ self.assertEqual(shebang, new_shebang)
|
||||
+
|
||||
+ return new_shebang
|
||||
+
|
||||
+ def test_pathfix(self):
|
||||
+ self.assertEqual(
|
||||
+ self.pathfix(
|
||||
+ '#! /usr/bin/env python',
|
||||
+ ['-i', '/usr/bin/python3']),
|
||||
+ '#! /usr/bin/python3')
|
||||
+ self.assertEqual(
|
||||
+ self.pathfix(
|
||||
+ '#! /usr/bin/env python -R',
|
||||
+ ['-i', '/usr/bin/python3']),
|
||||
+ '#! /usr/bin/python3')
|
||||
+
|
||||
+ def test_pathfix_keeping_flags(self):
|
||||
+ self.assertEqual(
|
||||
+ self.pathfix(
|
||||
+ '#! /usr/bin/env python -R',
|
||||
+ ['-i', '/usr/bin/python3', '-k']),
|
||||
+ '#! /usr/bin/python3 -R')
|
||||
+ self.assertEqual(
|
||||
+ self.pathfix(
|
||||
+ '#! /usr/bin/env python',
|
||||
+ ['-i', '/usr/bin/python3', '-k']),
|
||||
+ '#! /usr/bin/python3')
|
||||
+
|
||||
+ def test_pathfix_adding_flag(self):
|
||||
+ self.assertEqual(
|
||||
+ self.pathfix(
|
||||
+ '#! /usr/bin/env python',
|
||||
+ ['-i', '/usr/bin/python3', '-a', 's']),
|
||||
+ '#! /usr/bin/python3 -s')
|
||||
+ self.assertEqual(
|
||||
+ self.pathfix(
|
||||
+ '#! /usr/bin/env python -S',
|
||||
+ ['-i', '/usr/bin/python3', '-a', 's']),
|
||||
+ '#! /usr/bin/python3 -s')
|
||||
+ self.assertEqual(
|
||||
+ self.pathfix(
|
||||
+ '#! /usr/bin/env python -V',
|
||||
+ ['-i', '/usr/bin/python3', '-a', 'v', '-k']),
|
||||
+ '#! /usr/bin/python3 -vV')
|
||||
+ self.assertEqual(
|
||||
+ self.pathfix(
|
||||
+ '#! /usr/bin/env python',
|
||||
+ ['-i', '/usr/bin/python3', '-a', 'Rs']),
|
||||
+ '#! /usr/bin/python3 -Rs')
|
||||
+ self.assertEqual(
|
||||
+ self.pathfix(
|
||||
+ '#! /usr/bin/env python -W default',
|
||||
+ ['-i', '/usr/bin/python3', '-a', 's', '-k']),
|
||||
+ '#! /usr/bin/python3 -sW default')
|
||||
+
|
||||
+ def test_pathfix_adding_errors(self):
|
||||
+ self.pathfix(
|
||||
+ '#! /usr/bin/env python -E',
|
||||
+ ['-i', '/usr/bin/python3', '-a', 'W default', '-k'],
|
||||
+ exitcode=2,
|
||||
+ stderr="-a option doesn't support whitespaces")
|
||||
+
|
||||
+
|
||||
+if __name__ == '__main__':
|
||||
+ unittest.main()
|
||||
diff --git a/Tools/scripts/pathfix.py b/Tools/scripts/pathfix.py
|
||||
index c5bf984..2dfa6e8 100755
|
||||
--- a/Tools/scripts/pathfix.py
|
||||
+++ b/Tools/scripts/pathfix.py
|
||||
@@ -1,6 +1,6 @@
|
||||
#!/usr/bin/env python3
|
||||
|
||||
-# Change the #! line occurring in Python scripts. The new interpreter
|
||||
+# Change the #! line (shebang) occurring in Python scripts. The new interpreter
|
||||
# pathname must be given with a -i option.
|
||||
#
|
||||
# Command line arguments are files or directories to be processed.
|
||||
@@ -10,7 +10,13 @@
|
||||
# arguments).
|
||||
# The original file is kept as a back-up (with a "~" attached to its name),
|
||||
# -n flag can be used to disable this.
|
||||
-#
|
||||
+
|
||||
+# Sometimes you may find shebangs with flags such as `#! /usr/bin/env python -si`.
|
||||
+# Normally, pathfix overwrites the entire line, including the flags.
|
||||
+# To change interpreter and keep flags from the original shebang line, use -k.
|
||||
+# If you want to keep flags and add to them one single literal flag, use option -a.
|
||||
+
|
||||
+
|
||||
# Undoubtedly you can do this using find and sed or perl, but this is
|
||||
# a nice example of Python code that recurses down a directory tree
|
||||
# and uses regular expressions. Also note several subtleties like
|
||||
@@ -33,16 +39,21 @@ rep = sys.stdout.write
|
||||
new_interpreter = None
|
||||
preserve_timestamps = False
|
||||
create_backup = True
|
||||
+keep_flags = False
|
||||
+add_flags = b''
|
||||
|
||||
|
||||
def main():
|
||||
global new_interpreter
|
||||
global preserve_timestamps
|
||||
global create_backup
|
||||
- usage = ('usage: %s -i /interpreter -p -n file-or-directory ...\n' %
|
||||
+ global keep_flags
|
||||
+ global add_flags
|
||||
+
|
||||
+ usage = ('usage: %s -i /interpreter -p -n -k -a file-or-directory ...\n' %
|
||||
sys.argv[0])
|
||||
try:
|
||||
- opts, args = getopt.getopt(sys.argv[1:], 'i:pn')
|
||||
+ opts, args = getopt.getopt(sys.argv[1:], 'i:a:kpn')
|
||||
except getopt.error as msg:
|
||||
err(str(msg) + '\n')
|
||||
err(usage)
|
||||
@@ -54,6 +65,13 @@ def main():
|
||||
preserve_timestamps = True
|
||||
if o == '-n':
|
||||
create_backup = False
|
||||
+ if o == '-k':
|
||||
+ keep_flags = True
|
||||
+ if o == '-a':
|
||||
+ add_flags = a.encode()
|
||||
+ if b' ' in add_flags:
|
||||
+ err("-a option doesn't support whitespaces")
|
||||
+ sys.exit(2)
|
||||
if not new_interpreter or not new_interpreter.startswith(b'/') or \
|
||||
not args:
|
||||
err('-i option or file-or-directory missing\n')
|
||||
@@ -70,10 +88,14 @@ def main():
|
||||
if fix(arg): bad = 1
|
||||
sys.exit(bad)
|
||||
|
||||
+
|
||||
ispythonprog = re.compile(r'^[a-zA-Z0-9_]+\.py$')
|
||||
+
|
||||
+
|
||||
def ispython(name):
|
||||
return bool(ispythonprog.match(name))
|
||||
|
||||
+
|
||||
def recursedown(dirname):
|
||||
dbg('recursedown(%r)\n' % (dirname,))
|
||||
bad = 0
|
||||
@@ -96,6 +118,7 @@ def recursedown(dirname):
|
||||
if recursedown(fullname): bad = 1
|
||||
return bad
|
||||
|
||||
+
|
||||
def fix(filename):
|
||||
## dbg('fix(%r)\n' % (filename,))
|
||||
try:
|
||||
@@ -166,12 +189,43 @@ def fix(filename):
|
||||
# Return success
|
||||
return 0
|
||||
|
||||
+
|
||||
+def parse_shebang(shebangline):
|
||||
+ shebangline = shebangline.rstrip(b'\n')
|
||||
+ start = shebangline.find(b' -')
|
||||
+ if start == -1:
|
||||
+ return b''
|
||||
+ return shebangline[start:]
|
||||
+
|
||||
+
|
||||
+def populate_flags(shebangline):
|
||||
+ old_flags = b''
|
||||
+ if keep_flags:
|
||||
+ old_flags = parse_shebang(shebangline)
|
||||
+ if old_flags:
|
||||
+ old_flags = old_flags[2:]
|
||||
+ if not (old_flags or add_flags):
|
||||
+ return b''
|
||||
+ # On Linux, the entire string following the interpreter name
|
||||
+ # is passed as a single argument to the interpreter.
|
||||
+ # e.g. "#! /usr/bin/python3 -W Error -s" runs "/usr/bin/python3 "-W Error -s"
|
||||
+ # so shebang should have single '-' where flags are given and
|
||||
+ # flag might need argument for that reasons adding new flags is
|
||||
+ # between '-' and original flags
|
||||
+ # e.g. #! /usr/bin/python3 -sW Error
|
||||
+ return b' -' + add_flags + old_flags
|
||||
+
|
||||
+
|
||||
def fixline(line):
|
||||
if not line.startswith(b'#!'):
|
||||
return line
|
||||
+
|
||||
if b"python" not in line:
|
||||
return line
|
||||
- return b'#! ' + new_interpreter + b'\n'
|
||||
+
|
||||
+ flags = populate_flags(line)
|
||||
+ return b'#! ' + new_interpreter + flags + b'\n'
|
||||
+
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
--
|
||||
2.29.2
|
||||
|
|
@ -0,0 +1,184 @@
|
|||
From e92381a0a6a3e1f000956e1f1e70e543b9c2bcd5 Mon Sep 17 00:00:00 2001
|
||||
From: Benjamin Peterson <benjamin@python.org>
|
||||
Date: Mon, 18 Jan 2021 14:47:05 -0600
|
||||
Subject: [PATCH] [3.6] closes bpo-42938: Replace snprintf with Python unicode
|
||||
formatting in ctypes param reprs. (24239). (cherry picked from commit
|
||||
916610ef90a0d0761f08747f7b0905541f0977c7)
|
||||
|
||||
Co-authored-by: Benjamin Peterson <benjamin@python.org>
|
||||
---
|
||||
Lib/ctypes/test/test_parameters.py | 43 +++++++++++++++
|
||||
.../2021-01-18-09-27-31.bpo-42938.4Zn4Mp.rst | 2 +
|
||||
Modules/_ctypes/callproc.c | 55 +++++++------------
|
||||
3 files changed, 66 insertions(+), 34 deletions(-)
|
||||
create mode 100644 Misc/NEWS.d/next/Security/2021-01-18-09-27-31.bpo-42938.4Zn4Mp.rst
|
||||
|
||||
diff --git a/Lib/ctypes/test/test_parameters.py b/Lib/ctypes/test/test_parameters.py
|
||||
index e4c25fd880cef..531894fdec838 100644
|
||||
--- a/Lib/ctypes/test/test_parameters.py
|
||||
+++ b/Lib/ctypes/test/test_parameters.py
|
||||
@@ -201,6 +201,49 @@ def __dict__(self):
|
||||
with self.assertRaises(ZeroDivisionError):
|
||||
WorseStruct().__setstate__({}, b'foo')
|
||||
|
||||
+ def test_parameter_repr(self):
|
||||
+ from ctypes import (
|
||||
+ c_bool,
|
||||
+ c_char,
|
||||
+ c_wchar,
|
||||
+ c_byte,
|
||||
+ c_ubyte,
|
||||
+ c_short,
|
||||
+ c_ushort,
|
||||
+ c_int,
|
||||
+ c_uint,
|
||||
+ c_long,
|
||||
+ c_ulong,
|
||||
+ c_longlong,
|
||||
+ c_ulonglong,
|
||||
+ c_float,
|
||||
+ c_double,
|
||||
+ c_longdouble,
|
||||
+ c_char_p,
|
||||
+ c_wchar_p,
|
||||
+ c_void_p,
|
||||
+ )
|
||||
+ self.assertRegex(repr(c_bool.from_param(True)), r"^<cparam '\?' at 0x[A-Fa-f0-9]+>$")
|
||||
+ self.assertEqual(repr(c_char.from_param(97)), "<cparam 'c' ('a')>")
|
||||
+ self.assertRegex(repr(c_wchar.from_param('a')), r"^<cparam 'u' at 0x[A-Fa-f0-9]+>$")
|
||||
+ self.assertEqual(repr(c_byte.from_param(98)), "<cparam 'b' (98)>")
|
||||
+ self.assertEqual(repr(c_ubyte.from_param(98)), "<cparam 'B' (98)>")
|
||||
+ self.assertEqual(repr(c_short.from_param(511)), "<cparam 'h' (511)>")
|
||||
+ self.assertEqual(repr(c_ushort.from_param(511)), "<cparam 'H' (511)>")
|
||||
+ self.assertRegex(repr(c_int.from_param(20000)), r"^<cparam '[li]' \(20000\)>$")
|
||||
+ self.assertRegex(repr(c_uint.from_param(20000)), r"^<cparam '[LI]' \(20000\)>$")
|
||||
+ self.assertRegex(repr(c_long.from_param(20000)), r"^<cparam '[li]' \(20000\)>$")
|
||||
+ self.assertRegex(repr(c_ulong.from_param(20000)), r"^<cparam '[LI]' \(20000\)>$")
|
||||
+ self.assertRegex(repr(c_longlong.from_param(20000)), r"^<cparam '[liq]' \(20000\)>$")
|
||||
+ self.assertRegex(repr(c_ulonglong.from_param(20000)), r"^<cparam '[LIQ]' \(20000\)>$")
|
||||
+ self.assertEqual(repr(c_float.from_param(1.5)), "<cparam 'f' (1.5)>")
|
||||
+ self.assertEqual(repr(c_double.from_param(1.5)), "<cparam 'd' (1.5)>")
|
||||
+ self.assertEqual(repr(c_double.from_param(1e300)), "<cparam 'd' (1e+300)>")
|
||||
+ self.assertRegex(repr(c_longdouble.from_param(1.5)), r"^<cparam ('d' \(1.5\)|'g' at 0x[A-Fa-f0-9]+)>$")
|
||||
+ self.assertRegex(repr(c_char_p.from_param(b'hihi')), "^<cparam 'z' \(0x[A-Fa-f0-9]+\)>$")
|
||||
+ self.assertRegex(repr(c_wchar_p.from_param('hihi')), "^<cparam 'Z' \(0x[A-Fa-f0-9]+\)>$")
|
||||
+ self.assertRegex(repr(c_void_p.from_param(0x12)), r"^<cparam 'P' \(0x0*12\)>$")
|
||||
+
|
||||
################################################################
|
||||
|
||||
if __name__ == '__main__':
|
||||
diff --git a/Misc/NEWS.d/next/Security/2021-01-18-09-27-31.bpo-42938.4Zn4Mp.rst b/Misc/NEWS.d/next/Security/2021-01-18-09-27-31.bpo-42938.4Zn4Mp.rst
|
||||
new file mode 100644
|
||||
index 0000000000000..7df65a156feab
|
||||
--- /dev/null
|
||||
+++ b/Misc/NEWS.d/next/Security/2021-01-18-09-27-31.bpo-42938.4Zn4Mp.rst
|
||||
@@ -0,0 +1,2 @@
|
||||
+Avoid static buffers when computing the repr of :class:`ctypes.c_double` and
|
||||
+:class:`ctypes.c_longdouble` values.
|
||||
diff --git a/Modules/_ctypes/callproc.c b/Modules/_ctypes/callproc.c
|
||||
index d1c190f359108..2bb289bce043f 100644
|
||||
--- a/Modules/_ctypes/callproc.c
|
||||
+++ b/Modules/_ctypes/callproc.c
|
||||
@@ -461,58 +461,47 @@ is_literal_char(unsigned char c)
|
||||
static PyObject *
|
||||
PyCArg_repr(PyCArgObject *self)
|
||||
{
|
||||
- char buffer[256];
|
||||
switch(self->tag) {
|
||||
case 'b':
|
||||
case 'B':
|
||||
- sprintf(buffer, "<cparam '%c' (%d)>",
|
||||
+ return PyUnicode_FromFormat("<cparam '%c' (%d)>",
|
||||
self->tag, self->value.b);
|
||||
- break;
|
||||
case 'h':
|
||||
case 'H':
|
||||
- sprintf(buffer, "<cparam '%c' (%d)>",
|
||||
+ return PyUnicode_FromFormat("<cparam '%c' (%d)>",
|
||||
self->tag, self->value.h);
|
||||
- break;
|
||||
case 'i':
|
||||
case 'I':
|
||||
- sprintf(buffer, "<cparam '%c' (%d)>",
|
||||
+ return PyUnicode_FromFormat("<cparam '%c' (%d)>",
|
||||
self->tag, self->value.i);
|
||||
- break;
|
||||
case 'l':
|
||||
case 'L':
|
||||
- sprintf(buffer, "<cparam '%c' (%ld)>",
|
||||
+ return PyUnicode_FromFormat("<cparam '%c' (%ld)>",
|
||||
self->tag, self->value.l);
|
||||
- break;
|
||||
|
||||
case 'q':
|
||||
case 'Q':
|
||||
- sprintf(buffer,
|
||||
-#ifdef MS_WIN32
|
||||
- "<cparam '%c' (%I64d)>",
|
||||
-#else
|
||||
- "<cparam '%c' (%lld)>",
|
||||
-#endif
|
||||
+ return PyUnicode_FromFormat("<cparam '%c' (%lld)>",
|
||||
self->tag, self->value.q);
|
||||
- break;
|
||||
case 'd':
|
||||
- sprintf(buffer, "<cparam '%c' (%f)>",
|
||||
- self->tag, self->value.d);
|
||||
- break;
|
||||
- case 'f':
|
||||
- sprintf(buffer, "<cparam '%c' (%f)>",
|
||||
- self->tag, self->value.f);
|
||||
- break;
|
||||
-
|
||||
+ case 'f': {
|
||||
+ PyObject *f = PyFloat_FromDouble((self->tag == 'f') ? self->value.f : self->value.d);
|
||||
+ if (f == NULL) {
|
||||
+ return NULL;
|
||||
+ }
|
||||
+ PyObject *result = PyUnicode_FromFormat("<cparam '%c' (%R)>", self->tag, f);
|
||||
+ Py_DECREF(f);
|
||||
+ return result;
|
||||
+ }
|
||||
case 'c':
|
||||
if (is_literal_char((unsigned char)self->value.c)) {
|
||||
- sprintf(buffer, "<cparam '%c' ('%c')>",
|
||||
+ return PyUnicode_FromFormat("<cparam '%c' ('%c')>",
|
||||
self->tag, self->value.c);
|
||||
}
|
||||
else {
|
||||
- sprintf(buffer, "<cparam '%c' ('\\x%02x')>",
|
||||
+ return PyUnicode_FromFormat("<cparam '%c' ('\\x%02x')>",
|
||||
self->tag, (unsigned char)self->value.c);
|
||||
}
|
||||
- break;
|
||||
|
||||
/* Hm, are these 'z' and 'Z' codes useful at all?
|
||||
Shouldn't they be replaced by the functionality of c_string
|
||||
@@ -521,22 +510,20 @@ PyCArg_repr(PyCArgObject *self)
|
||||
case 'z':
|
||||
case 'Z':
|
||||
case 'P':
|
||||
- sprintf(buffer, "<cparam '%c' (%p)>",
|
||||
+ return PyUnicode_FromFormat("<cparam '%c' (%p)>",
|
||||
self->tag, self->value.p);
|
||||
break;
|
||||
|
||||
default:
|
||||
if (is_literal_char((unsigned char)self->tag)) {
|
||||
- sprintf(buffer, "<cparam '%c' at %p>",
|
||||
- (unsigned char)self->tag, self);
|
||||
+ return PyUnicode_FromFormat("<cparam '%c' at %p>",
|
||||
+ (unsigned char)self->tag, (void *)self);
|
||||
}
|
||||
else {
|
||||
- sprintf(buffer, "<cparam 0x%02x at %p>",
|
||||
- (unsigned char)self->tag, self);
|
||||
+ return PyUnicode_FromFormat("<cparam 0x%02x at %p>",
|
||||
+ (unsigned char)self->tag, (void *)self);
|
||||
}
|
||||
- break;
|
||||
}
|
||||
- return PyUnicode_FromString(buffer);
|
||||
}
|
||||
|
||||
static PyMemberDef PyCArgType_members[] = {
|
|
@ -0,0 +1,684 @@
|
|||
commit 9e77ec82c40ab59846f9447b7c483e7b8e368b16
|
||||
Author: Petr Viktorin <pviktori@redhat.com>
|
||||
Date: Thu Mar 4 13:59:56 2021 +0100
|
||||
|
||||
CVE-2021-23336: Add `separator` argument to parse_qs; warn with default
|
||||
|
||||
Partially backports https://bugs.python.org/issue42967 : [security] Address a web cache-poisoning issue reported in urllib.parse.parse_qsl().
|
||||
However, this solution is different than the upstream solution in Python 3.6.13.
|
||||
|
||||
An optional argument seperator is added to specify the separator.
|
||||
It is recommended to set it to '&' or ';' to match the application or proxy in use.
|
||||
The default can be set with an env variable of a config file.
|
||||
If neither the argument, env var or config file specifies a separator, "&" is used
|
||||
but a warning is raised if parse_qs is used on input that contains ';'.
|
||||
|
||||
Co-authors of the upstream change (who do not necessarily agree with this):
|
||||
Co-authored-by: Adam Goldschmidt <adamgold7@gmail.com>
|
||||
Co-authored-by: Ken Jin <28750310+Fidget-Spinner@users.noreply.github.com>
|
||||
Co-authored-by: Éric Araujo <merwok@netwok.org>
|
||||
|
||||
diff --git a/Doc/library/cgi.rst b/Doc/library/cgi.rst
|
||||
index 41219eeaaba..ddecc0af23a 100644
|
||||
--- a/Doc/library/cgi.rst
|
||||
+++ b/Doc/library/cgi.rst
|
||||
@@ -277,13 +277,12 @@ These are useful if you want more control, or if you want to employ some of the
|
||||
algorithms implemented in this module in other circumstances.
|
||||
|
||||
|
||||
-.. function:: parse(fp=None, environ=os.environ, keep_blank_values=False, strict_parsing=False)
|
||||
+.. function:: parse(fp=None, environ=os.environ, keep_blank_values=False, strict_parsing=False, separator=None)
|
||||
|
||||
Parse a query in the environment or from a file (the file defaults to
|
||||
- ``sys.stdin``). The *keep_blank_values* and *strict_parsing* parameters are
|
||||
+ ``sys.stdin``). The *keep_blank_values*, *strict_parsing* and *separator* parameters are
|
||||
passed to :func:`urllib.parse.parse_qs` unchanged.
|
||||
|
||||
-
|
||||
.. function:: parse_qs(qs, keep_blank_values=False, strict_parsing=False)
|
||||
|
||||
This function is deprecated in this module. Use :func:`urllib.parse.parse_qs`
|
||||
@@ -308,7 +307,6 @@ algorithms implemented in this module in other circumstances.
|
||||
Note that this does not parse nested multipart parts --- use
|
||||
:class:`FieldStorage` for that.
|
||||
|
||||
-
|
||||
.. function:: parse_header(string)
|
||||
|
||||
Parse a MIME header (such as :mailheader:`Content-Type`) into a main value and a
|
||||
diff --git a/Doc/library/urllib.parse.rst b/Doc/library/urllib.parse.rst
|
||||
index 647af613a31..bcab7c142bc 100644
|
||||
--- a/Doc/library/urllib.parse.rst
|
||||
+++ b/Doc/library/urllib.parse.rst
|
||||
@@ -143,7 +143,7 @@ or on combining URL components into a URL string.
|
||||
now raise :exc:`ValueError`.
|
||||
|
||||
|
||||
-.. function:: parse_qs(qs, keep_blank_values=False, strict_parsing=False, encoding='utf-8', errors='replace', max_num_fields=None)
|
||||
+.. function:: parse_qs(qs, keep_blank_values=False, strict_parsing=False, encoding='utf-8', errors='replace', max_num_fields=None, separator=None)
|
||||
|
||||
Parse a query string given as a string argument (data of type
|
||||
:mimetype:`application/x-www-form-urlencoded`). Data are returned as a
|
||||
@@ -168,6 +168,15 @@ or on combining URL components into a URL string.
|
||||
read. If set, then throws a :exc:`ValueError` if there are more than
|
||||
*max_num_fields* fields read.
|
||||
|
||||
+ The optional argument *separator* is the symbol to use for separating the
|
||||
+ query arguments. It is recommended to set it to ``'&'`` or ``';'``.
|
||||
+ It defaults to ``'&'``; a warning is raised if this default is used.
|
||||
+ This default may be changed with the following environment variable settings:
|
||||
+
|
||||
+ - ``PYTHON_URLLIB_QS_SEPARATOR='&'``: use only ``&`` as separator, without warning (as in Python 3.6.13+ or 3.10)
|
||||
+ - ``PYTHON_URLLIB_QS_SEPARATOR=';'``: use only ``;`` as separator
|
||||
+ - ``PYTHON_URLLIB_QS_SEPARATOR=legacy``: use both ``&`` and ``;`` (as in previous versions of Python)
|
||||
+
|
||||
Use the :func:`urllib.parse.urlencode` function (with the ``doseq``
|
||||
parameter set to ``True``) to convert such dictionaries into query
|
||||
strings.
|
||||
@@ -204,6 +213,9 @@ or on combining URL components into a URL string.
|
||||
read. If set, then throws a :exc:`ValueError` if there are more than
|
||||
*max_num_fields* fields read.
|
||||
|
||||
+ The optional argument *separator* is the symbol to use for separating the
|
||||
+ query arguments. It works as in :py:func:`parse_qs`.
|
||||
+
|
||||
Use the :func:`urllib.parse.urlencode` function to convert such lists of pairs into
|
||||
query strings.
|
||||
|
||||
@@ -213,7 +225,6 @@ or on combining URL components into a URL string.
|
||||
.. versionchanged:: 3.6.8
|
||||
Added *max_num_fields* parameter.
|
||||
|
||||
-
|
||||
.. function:: urlunparse(parts)
|
||||
|
||||
Construct a URL from a tuple as returned by ``urlparse()``. The *parts*
|
||||
diff --git a/Lib/cgi.py b/Lib/cgi.py
|
||||
index 56f243e09f0..5ab2a5d6af6 100755
|
||||
--- a/Lib/cgi.py
|
||||
+++ b/Lib/cgi.py
|
||||
@@ -117,7 +117,8 @@ log = initlog # The current logging function
|
||||
# 0 ==> unlimited input
|
||||
maxlen = 0
|
||||
|
||||
-def parse(fp=None, environ=os.environ, keep_blank_values=0, strict_parsing=0):
|
||||
+def parse(fp=None, environ=os.environ, keep_blank_values=0,
|
||||
+ strict_parsing=0, separator=None):
|
||||
"""Parse a query in the environment or from a file (default stdin)
|
||||
|
||||
Arguments, all optional:
|
||||
@@ -136,6 +137,8 @@ def parse(fp=None, environ=os.environ, keep_blank_values=0, strict_parsing=0):
|
||||
strict_parsing: flag indicating what to do with parsing errors.
|
||||
If false (the default), errors are silently ignored.
|
||||
If true, errors raise a ValueError exception.
|
||||
+
|
||||
+ separator: str. The symbol to use for separating the query arguments.
|
||||
"""
|
||||
if fp is None:
|
||||
fp = sys.stdin
|
||||
@@ -156,7 +159,7 @@ def parse(fp=None, environ=os.environ, keep_blank_values=0, strict_parsing=0):
|
||||
if environ['REQUEST_METHOD'] == 'POST':
|
||||
ctype, pdict = parse_header(environ['CONTENT_TYPE'])
|
||||
if ctype == 'multipart/form-data':
|
||||
- return parse_multipart(fp, pdict)
|
||||
+ return parse_multipart(fp, pdict, separator=separator)
|
||||
elif ctype == 'application/x-www-form-urlencoded':
|
||||
clength = int(environ['CONTENT_LENGTH'])
|
||||
if maxlen and clength > maxlen:
|
||||
@@ -182,21 +185,21 @@ def parse(fp=None, environ=os.environ, keep_blank_values=0, strict_parsing=0):
|
||||
return urllib.parse.parse_qs(qs, keep_blank_values, strict_parsing,
|
||||
encoding=encoding)
|
||||
|
||||
-
|
||||
# parse query string function called from urlparse,
|
||||
# this is done in order to maintain backward compatibility.
|
||||
-
|
||||
-def parse_qs(qs, keep_blank_values=0, strict_parsing=0):
|
||||
+def parse_qs(qs, keep_blank_values=0, strict_parsing=0, separator=None):
|
||||
"""Parse a query given as a string argument."""
|
||||
warn("cgi.parse_qs is deprecated, use urllib.parse.parse_qs instead",
|
||||
DeprecationWarning, 2)
|
||||
- return urllib.parse.parse_qs(qs, keep_blank_values, strict_parsing)
|
||||
+ return urllib.parse.parse_qs(qs, keep_blank_values, strict_parsing,
|
||||
+ separator=separator)
|
||||
|
||||
-def parse_qsl(qs, keep_blank_values=0, strict_parsing=0):
|
||||
+def parse_qsl(qs, keep_blank_values=0, strict_parsing=0, separator=None):
|
||||
"""Parse a query given as a string argument."""
|
||||
warn("cgi.parse_qsl is deprecated, use urllib.parse.parse_qsl instead",
|
||||
DeprecationWarning, 2)
|
||||
- return urllib.parse.parse_qsl(qs, keep_blank_values, strict_parsing)
|
||||
+ return urllib.parse.parse_qsl(qs, keep_blank_values, strict_parsing,
|
||||
+ separator=separator)
|
||||
|
||||
def parse_multipart(fp, pdict):
|
||||
"""Parse multipart input.
|
||||
@@ -297,7 +300,6 @@ def parse_multipart(fp, pdict):
|
||||
|
||||
return partdict
|
||||
|
||||
-
|
||||
def _parseparam(s):
|
||||
while s[:1] == ';':
|
||||
s = s[1:]
|
||||
@@ -405,7 +407,7 @@ class FieldStorage:
|
||||
def __init__(self, fp=None, headers=None, outerboundary=b'',
|
||||
environ=os.environ, keep_blank_values=0, strict_parsing=0,
|
||||
limit=None, encoding='utf-8', errors='replace',
|
||||
- max_num_fields=None):
|
||||
+ max_num_fields=None, separator=None):
|
||||
"""Constructor. Read multipart/* until last part.
|
||||
|
||||
Arguments, all optional:
|
||||
@@ -453,6 +455,7 @@ class FieldStorage:
|
||||
self.keep_blank_values = keep_blank_values
|
||||
self.strict_parsing = strict_parsing
|
||||
self.max_num_fields = max_num_fields
|
||||
+ self.separator = separator
|
||||
if 'REQUEST_METHOD' in environ:
|
||||
method = environ['REQUEST_METHOD'].upper()
|
||||
self.qs_on_post = None
|
||||
@@ -678,7 +681,7 @@ class FieldStorage:
|
||||
query = urllib.parse.parse_qsl(
|
||||
qs, self.keep_blank_values, self.strict_parsing,
|
||||
encoding=self.encoding, errors=self.errors,
|
||||
- max_num_fields=self.max_num_fields)
|
||||
+ max_num_fields=self.max_num_fields, separator=self.separator)
|
||||
self.list = [MiniFieldStorage(key, value) for key, value in query]
|
||||
self.skip_lines()
|
||||
|
||||
@@ -694,7 +697,7 @@ class FieldStorage:
|
||||
query = urllib.parse.parse_qsl(
|
||||
self.qs_on_post, self.keep_blank_values, self.strict_parsing,
|
||||
encoding=self.encoding, errors=self.errors,
|
||||
- max_num_fields=self.max_num_fields)
|
||||
+ max_num_fields=self.max_num_fields, separator=self.separator)
|
||||
self.list.extend(MiniFieldStorage(key, value) for key, value in query)
|
||||
|
||||
klass = self.FieldStorageClass or self.__class__
|
||||
@@ -736,7 +739,8 @@ class FieldStorage:
|
||||
|
||||
part = klass(self.fp, headers, ib, environ, keep_blank_values,
|
||||
strict_parsing,self.limit-self.bytes_read,
|
||||
- self.encoding, self.errors, max_num_fields)
|
||||
+ self.encoding, self.errors, max_num_fields,
|
||||
+ separator=self.separator)
|
||||
|
||||
if max_num_fields is not None:
|
||||
max_num_fields -= 1
|
||||
diff --git a/Lib/test/test_cgi.py b/Lib/test/test_cgi.py
|
||||
index b3e2d4cce8e..5ae3e085e1e 100644
|
||||
--- a/Lib/test/test_cgi.py
|
||||
+++ b/Lib/test/test_cgi.py
|
||||
@@ -55,12 +55,9 @@ parse_strict_test_cases = [
|
||||
("", ValueError("bad query field: ''")),
|
||||
("&", ValueError("bad query field: ''")),
|
||||
("&&", ValueError("bad query field: ''")),
|
||||
- (";", ValueError("bad query field: ''")),
|
||||
- (";&;", ValueError("bad query field: ''")),
|
||||
# Should the next few really be valid?
|
||||
("=", {}),
|
||||
("=&=", {}),
|
||||
- ("=;=", {}),
|
||||
# This rest seem to make sense
|
||||
("=a", {'': ['a']}),
|
||||
("&=a", ValueError("bad query field: ''")),
|
||||
@@ -75,8 +72,6 @@ parse_strict_test_cases = [
|
||||
("a=a+b&b=b+c", {'a': ['a b'], 'b': ['b c']}),
|
||||
("a=a+b&a=b+a", {'a': ['a b', 'b a']}),
|
||||
("x=1&y=2.0&z=2-3.%2b0", {'x': ['1'], 'y': ['2.0'], 'z': ['2-3.+0']}),
|
||||
- ("x=1;y=2.0&z=2-3.%2b0", {'x': ['1'], 'y': ['2.0'], 'z': ['2-3.+0']}),
|
||||
- ("x=1;y=2.0;z=2-3.%2b0", {'x': ['1'], 'y': ['2.0'], 'z': ['2-3.+0']}),
|
||||
("Hbc5161168c542333633315dee1182227:key_store_seqid=400006&cuyer=r&view=bustomer&order_id=0bb2e248638833d48cb7fed300000f1b&expire=964546263&lobale=en-US&kid=130003.300038&ss=env",
|
||||
{'Hbc5161168c542333633315dee1182227:key_store_seqid': ['400006'],
|
||||
'cuyer': ['r'],
|
||||
@@ -164,6 +159,35 @@ class CgiTests(unittest.TestCase):
|
||||
|
||||
env = {'QUERY_STRING': orig}
|
||||
fs = cgi.FieldStorage(environ=env)
|
||||
+ if isinstance(expect, dict):
|
||||
+ # test dict interface
|
||||
+ self.assertEqual(len(expect), len(fs))
|
||||
+ self.assertCountEqual(expect.keys(), fs.keys())
|
||||
+ self.assertEqual(fs.getvalue("nonexistent field", "default"), "default")
|
||||
+ # test individual fields
|
||||
+ for key in expect.keys():
|
||||
+ expect_val = expect[key]
|
||||
+ self.assertIn(key, fs)
|
||||
+ if len(expect_val) > 1:
|
||||
+ self.assertEqual(fs.getvalue(key), expect_val)
|
||||
+ else:
|
||||
+ self.assertEqual(fs.getvalue(key), expect_val[0])
|
||||
+
|
||||
+ def test_separator(self):
|
||||
+ parse_semicolon = [
|
||||
+ ("x=1;y=2.0", {'x': ['1'], 'y': ['2.0']}),
|
||||
+ ("x=1;y=2.0;z=2-3.%2b0", {'x': ['1'], 'y': ['2.0'], 'z': ['2-3.+0']}),
|
||||
+ (";", ValueError("bad query field: ''")),
|
||||
+ (";;", ValueError("bad query field: ''")),
|
||||
+ ("=;a", ValueError("bad query field: 'a'")),
|
||||
+ (";b=a", ValueError("bad query field: ''")),
|
||||
+ ("b;=a", ValueError("bad query field: 'b'")),
|
||||
+ ("a=a+b;b=b+c", {'a': ['a b'], 'b': ['b c']}),
|
||||
+ ("a=a+b;a=b+a", {'a': ['a b', 'b a']}),
|
||||
+ ]
|
||||
+ for orig, expect in parse_semicolon:
|
||||
+ env = {'QUERY_STRING': orig}
|
||||
+ fs = cgi.FieldStorage(separator=';', environ=env)
|
||||
if isinstance(expect, dict):
|
||||
# test dict interface
|
||||
self.assertEqual(len(expect), len(fs))
|
||||
diff --git a/Lib/test/test_urlparse.py b/Lib/test/test_urlparse.py
|
||||
index 68f633ca3a7..1ec86ba0fc2 100644
|
||||
--- a/Lib/test/test_urlparse.py
|
||||
+++ b/Lib/test/test_urlparse.py
|
||||
@@ -2,6 +2,11 @@ import sys
|
||||
import unicodedata
|
||||
import unittest
|
||||
import urllib.parse
|
||||
+from test.support import EnvironmentVarGuard
|
||||
+from warnings import catch_warnings
|
||||
+import tempfile
|
||||
+import contextlib
|
||||
+import os.path
|
||||
|
||||
RFC1808_BASE = "http://a/b/c/d;p?q#f"
|
||||
RFC2396_BASE = "http://a/b/c/d;p?q"
|
||||
@@ -32,6 +37,9 @@ parse_qsl_test_cases = [
|
||||
(b"&a=b", [(b'a', b'b')]),
|
||||
(b"a=a+b&b=b+c", [(b'a', b'a b'), (b'b', b'b c')]),
|
||||
(b"a=1&a=2", [(b'a', b'1'), (b'a', b'2')]),
|
||||
+]
|
||||
+
|
||||
+parse_qsl_test_cases_semicolon = [
|
||||
(";", []),
|
||||
(";;", []),
|
||||
(";a=b", [('a', 'b')]),
|
||||
@@ -44,6 +52,21 @@ parse_qsl_test_cases = [
|
||||
(b"a=1;a=2", [(b'a', b'1'), (b'a', b'2')]),
|
||||
]
|
||||
|
||||
+parse_qsl_test_cases_legacy = [
|
||||
+ (b"a=1;a=2&a=3", [(b'a', b'1'), (b'a', b'2'), (b'a', b'3')]),
|
||||
+ (b"a=1;b=2&c=3", [(b'a', b'1'), (b'b', b'2'), (b'c', b'3')]),
|
||||
+ (b"a=1&b=2&c=3;", [(b'a', b'1'), (b'b', b'2'), (b'c', b'3')]),
|
||||
+]
|
||||
+
|
||||
+parse_qsl_test_cases_warn = [
|
||||
+ (";a=b", [(';a', 'b')]),
|
||||
+ ("a=a+b;b=b+c", [('a', 'a b;b=b c')]),
|
||||
+ (b";a=b", [(b';a', b'b')]),
|
||||
+ (b"a=a+b;b=b+c", [(b'a', b'a b;b=b c')]),
|
||||
+ ("a=1;a=2&a=3", [('a', '1;a=2'), ('a', '3')]),
|
||||
+ (b"a=1;a=2&a=3", [(b'a', b'1;a=2'), (b'a', b'3')]),
|
||||
+]
|
||||
+
|
||||
# Each parse_qs testcase is a two-tuple that contains
|
||||
# a string with the query and a dictionary with the expected result.
|
||||
|
||||
@@ -68,6 +91,9 @@ parse_qs_test_cases = [
|
||||
(b"&a=b", {b'a': [b'b']}),
|
||||
(b"a=a+b&b=b+c", {b'a': [b'a b'], b'b': [b'b c']}),
|
||||
(b"a=1&a=2", {b'a': [b'1', b'2']}),
|
||||
+]
|
||||
+
|
||||
+parse_qs_test_cases_semicolon = [
|
||||
(";", {}),
|
||||
(";;", {}),
|
||||
(";a=b", {'a': ['b']}),
|
||||
@@ -80,6 +106,24 @@ parse_qs_test_cases = [
|
||||
(b"a=1;a=2", {b'a': [b'1', b'2']}),
|
||||
]
|
||||
|
||||
+parse_qs_test_cases_legacy = [
|
||||
+ ("a=1;a=2&a=3", {'a': ['1', '2', '3']}),
|
||||
+ ("a=1;b=2&c=3", {'a': ['1'], 'b': ['2'], 'c': ['3']}),
|
||||
+ ("a=1&b=2&c=3;", {'a': ['1'], 'b': ['2'], 'c': ['3']}),
|
||||
+ (b"a=1;a=2&a=3", {b'a': [b'1', b'2', b'3']}),
|
||||
+ (b"a=1;b=2&c=3", {b'a': [b'1'], b'b': [b'2'], b'c': [b'3']}),
|
||||
+ (b"a=1&b=2&c=3;", {b'a': [b'1'], b'b': [b'2'], b'c': [b'3']}),
|
||||
+]
|
||||
+
|
||||
+parse_qs_test_cases_warn = [
|
||||
+ (";a=b", {';a': ['b']}),
|
||||
+ ("a=a+b;b=b+c", {'a': ['a b;b=b c']}),
|
||||
+ (b";a=b", {b';a': [b'b']}),
|
||||
+ (b"a=a+b;b=b+c", {b'a':[ b'a b;b=b c']}),
|
||||
+ ("a=1;a=2&a=3", {'a': ['1;a=2', '3']}),
|
||||
+ (b"a=1;a=2&a=3", {b'a': [b'1;a=2', b'3']}),
|
||||
+]
|
||||
+
|
||||
class UrlParseTestCase(unittest.TestCase):
|
||||
|
||||
def checkRoundtrips(self, url, parsed, split):
|
||||
@@ -152,6 +196,40 @@ class UrlParseTestCase(unittest.TestCase):
|
||||
self.assertEqual(result, expect_without_blanks,
|
||||
"Error parsing %r" % orig)
|
||||
|
||||
+ def test_qs_default_warn(self):
|
||||
+ for orig, expect in parse_qs_test_cases_warn:
|
||||
+ with self.subTest(orig=orig, expect=expect):
|
||||
+ with catch_warnings(record=True) as w:
|
||||
+ result = urllib.parse.parse_qs(orig, keep_blank_values=True)
|
||||
+ self.assertEqual(result, expect, "Error parsing %r" % orig)
|
||||
+ self.assertEqual(len(w), 1)
|
||||
+ self.assertEqual(w[0].category, urllib.parse._QueryStringSeparatorWarning)
|
||||
+
|
||||
+ def test_qsl_default_warn(self):
|
||||
+ for orig, expect in parse_qsl_test_cases_warn:
|
||||
+ with self.subTest(orig=orig, expect=expect):
|
||||
+ with catch_warnings(record=True) as w:
|
||||
+ result = urllib.parse.parse_qsl(orig, keep_blank_values=True)
|
||||
+ self.assertEqual(result, expect, "Error parsing %r" % orig)
|
||||
+ self.assertEqual(len(w), 1)
|
||||
+ self.assertEqual(w[0].category, urllib.parse._QueryStringSeparatorWarning)
|
||||
+
|
||||
+ def test_default_qs_no_warnings(self):
|
||||
+ for orig, expect in parse_qs_test_cases:
|
||||
+ with self.subTest(orig=orig, expect=expect):
|
||||
+ with catch_warnings(record=True) as w:
|
||||
+ result = urllib.parse.parse_qs(orig, keep_blank_values=True)
|
||||
+ self.assertEqual(result, expect, "Error parsing %r" % orig)
|
||||
+ self.assertEqual(len(w), 0)
|
||||
+
|
||||
+ def test_default_qsl_no_warnings(self):
|
||||
+ for orig, expect in parse_qsl_test_cases:
|
||||
+ with self.subTest(orig=orig, expect=expect):
|
||||
+ with catch_warnings(record=True) as w:
|
||||
+ result = urllib.parse.parse_qsl(orig, keep_blank_values=True)
|
||||
+ self.assertEqual(result, expect, "Error parsing %r" % orig)
|
||||
+ self.assertEqual(len(w), 0)
|
||||
+
|
||||
def test_roundtrips(self):
|
||||
str_cases = [
|
||||
('file:///tmp/junk.txt',
|
||||
@@ -885,8 +963,151 @@ class UrlParseTestCase(unittest.TestCase):
|
||||
with self.assertRaises(ValueError):
|
||||
urllib.parse.parse_qs('&'.join(['a=a']*11), max_num_fields=10)
|
||||
with self.assertRaises(ValueError):
|
||||
- urllib.parse.parse_qs(';'.join(['a=a']*11), max_num_fields=10)
|
||||
+ urllib.parse.parse_qs(';'.join(['a=a']*11), separator=';', max_num_fields=10)
|
||||
+ with self.assertRaises(ValueError):
|
||||
+ urllib.parse.parse_qs('SEP'.join(['a=a']*11), separator='SEP', max_num_fields=10)
|
||||
urllib.parse.parse_qs('&'.join(['a=a']*10), max_num_fields=10)
|
||||
+ urllib.parse.parse_qs(';'.join(['a=a']*10), separator=';', max_num_fields=10)
|
||||
+ urllib.parse.parse_qs('SEP'.join(['a=a']*10), separator='SEP', max_num_fields=10)
|
||||
+
|
||||
+ def test_parse_qs_separator_bytes(self):
|
||||
+ expected = {b'a': [b'1'], b'b': [b'2']}
|
||||
+
|
||||
+ result = urllib.parse.parse_qs(b'a=1;b=2', separator=b';')
|
||||
+ self.assertEqual(result, expected)
|
||||
+ result = urllib.parse.parse_qs(b'a=1;b=2', separator=';')
|
||||
+ self.assertEqual(result, expected)
|
||||
+ result = urllib.parse.parse_qs('a=1;b=2', separator=';')
|
||||
+ self.assertEqual(result, {'a': ['1'], 'b': ['2']})
|
||||
+
|
||||
+ @contextlib.contextmanager
|
||||
+ def _qsl_sep_config(self, sep):
|
||||
+ """Context for the given parse_qsl default separator configured in config file"""
|
||||
+ old_filename = urllib.parse._QS_SEPARATOR_CONFIG_FILENAME
|
||||
+ urllib.parse._default_qs_separator = None
|
||||
+ try:
|
||||
+ with tempfile.TemporaryDirectory() as tmpdirname:
|
||||
+ filename = os.path.join(tmpdirname, 'conf.cfg')
|
||||
+ with open(filename, 'w') as file:
|
||||
+ file.write(f'[parse_qs]\n')
|
||||
+ file.write(f'PYTHON_URLLIB_QS_SEPARATOR = {sep}')
|
||||
+ urllib.parse._QS_SEPARATOR_CONFIG_FILENAME = filename
|
||||
+ yield
|
||||
+ finally:
|
||||
+ urllib.parse._QS_SEPARATOR_CONFIG_FILENAME = old_filename
|
||||
+ urllib.parse._default_qs_separator = None
|
||||
+
|
||||
+ def test_parse_qs_separator_semicolon(self):
|
||||
+ for orig, expect in parse_qs_test_cases_semicolon:
|
||||
+ with self.subTest(orig=orig, expect=expect, method='arg'):
|
||||
+ result = urllib.parse.parse_qs(orig, separator=';')
|
||||
+ self.assertEqual(result, expect, "Error parsing %r" % orig)
|
||||
+ with self.subTest(orig=orig, expect=expect, method='env'):
|
||||
+ with EnvironmentVarGuard() as environ, catch_warnings(record=True) as w:
|
||||
+ environ['PYTHON_URLLIB_QS_SEPARATOR'] = ';'
|
||||
+ result = urllib.parse.parse_qs(orig)
|
||||
+ self.assertEqual(result, expect, "Error parsing %r" % orig)
|
||||
+ self.assertEqual(len(w), 0)
|
||||
+ with self.subTest(orig=orig, expect=expect, method='conf'):
|
||||
+ with self._qsl_sep_config(';'), catch_warnings(record=True) as w:
|
||||
+ result = urllib.parse.parse_qs(orig)
|
||||
+ self.assertEqual(result, expect, "Error parsing %r" % orig)
|
||||
+ self.assertEqual(len(w), 0)
|
||||
+
|
||||
+ def test_parse_qsl_separator_semicolon(self):
|
||||
+ for orig, expect in parse_qsl_test_cases_semicolon:
|
||||
+ with self.subTest(orig=orig, expect=expect, method='arg'):
|
||||
+ result = urllib.parse.parse_qsl(orig, separator=';')
|
||||
+ self.assertEqual(result, expect, "Error parsing %r" % orig)
|
||||
+ with self.subTest(orig=orig, expect=expect, method='env'):
|
||||
+ with EnvironmentVarGuard() as environ, catch_warnings(record=True) as w:
|
||||
+ environ['PYTHON_URLLIB_QS_SEPARATOR'] = ';'
|
||||
+ result = urllib.parse.parse_qsl(orig)
|
||||
+ self.assertEqual(result, expect, "Error parsing %r" % orig)
|
||||
+ self.assertEqual(len(w), 0)
|
||||
+ with self.subTest(orig=orig, expect=expect, method='conf'):
|
||||
+ with self._qsl_sep_config(';'), catch_warnings(record=True) as w:
|
||||
+ result = urllib.parse.parse_qsl(orig)
|
||||
+ self.assertEqual(result, expect, "Error parsing %r" % orig)
|
||||
+ self.assertEqual(len(w), 0)
|
||||
+
|
||||
+ def test_parse_qs_separator_legacy(self):
|
||||
+ for orig, expect in parse_qs_test_cases_legacy:
|
||||
+ with self.subTest(orig=orig, expect=expect, method='env'):
|
||||
+ with EnvironmentVarGuard() as environ, catch_warnings(record=True) as w:
|
||||
+ environ['PYTHON_URLLIB_QS_SEPARATOR'] = 'legacy'
|
||||
+ result = urllib.parse.parse_qs(orig)
|
||||
+ self.assertEqual(result, expect, "Error parsing %r" % orig)
|
||||
+ self.assertEqual(len(w), 0)
|
||||
+ with self.subTest(orig=orig, expect=expect, method='conf'):
|
||||
+ with self._qsl_sep_config('legacy'), catch_warnings(record=True) as w:
|
||||
+ result = urllib.parse.parse_qs(orig)
|
||||
+ self.assertEqual(result, expect, "Error parsing %r" % orig)
|
||||
+ self.assertEqual(len(w), 0)
|
||||
+
|
||||
+ def test_parse_qsl_separator_legacy(self):
|
||||
+ for orig, expect in parse_qsl_test_cases_legacy:
|
||||
+ with self.subTest(orig=orig, expect=expect, method='env'):
|
||||
+ with EnvironmentVarGuard() as environ, catch_warnings(record=True) as w:
|
||||
+ environ['PYTHON_URLLIB_QS_SEPARATOR'] = 'legacy'
|
||||
+ result = urllib.parse.parse_qsl(orig)
|
||||
+ self.assertEqual(result, expect, "Error parsing %r" % orig)
|
||||
+ self.assertEqual(len(w), 0)
|
||||
+ with self.subTest(orig=orig, expect=expect, method='conf'):
|
||||
+ with self._qsl_sep_config('legacy'), catch_warnings(record=True) as w:
|
||||
+ result = urllib.parse.parse_qsl(orig)
|
||||
+ self.assertEqual(result, expect, "Error parsing %r" % orig)
|
||||
+ self.assertEqual(len(w), 0)
|
||||
+
|
||||
+ def test_parse_qs_separator_bad_value_env_or_config(self):
|
||||
+ for bad_sep in '', 'abc', 'safe', '&;', 'SEP':
|
||||
+ with self.subTest(bad_sep, method='env'):
|
||||
+ with EnvironmentVarGuard() as environ, catch_warnings(record=True) as w:
|
||||
+ environ['PYTHON_URLLIB_QS_SEPARATOR'] = bad_sep
|
||||
+ with self.assertRaises(ValueError):
|
||||
+ urllib.parse.parse_qsl('a=1;b=2')
|
||||
+ with self.subTest(bad_sep, method='conf'):
|
||||
+ with self._qsl_sep_config('bad_sep'), catch_warnings(record=True) as w:
|
||||
+ with self.assertRaises(ValueError):
|
||||
+ urllib.parse.parse_qsl('a=1;b=2')
|
||||
+
|
||||
+ def test_parse_qs_separator_bad_value_arg(self):
|
||||
+ for bad_sep in True, {}, '':
|
||||
+ with self.subTest(bad_sep):
|
||||
+ with self.assertRaises(ValueError):
|
||||
+ urllib.parse.parse_qsl('a=1;b=2', separator=bad_sep)
|
||||
+
|
||||
+ def test_parse_qs_separator_num_fields(self):
|
||||
+ for qs, sep in (
|
||||
+ ('a&b&c', '&'),
|
||||
+ ('a;b;c', ';'),
|
||||
+ ('a&b;c', 'legacy'),
|
||||
+ ):
|
||||
+ with self.subTest(qs=qs, sep=sep):
|
||||
+ with EnvironmentVarGuard() as environ, catch_warnings(record=True) as w:
|
||||
+ if sep != 'legacy':
|
||||
+ with self.assertRaises(ValueError):
|
||||
+ urllib.parse.parse_qsl(qs, separator=sep, max_num_fields=2)
|
||||
+ if sep:
|
||||
+ environ['PYTHON_URLLIB_QS_SEPARATOR'] = sep
|
||||
+ with self.assertRaises(ValueError):
|
||||
+ urllib.parse.parse_qsl(qs, max_num_fields=2)
|
||||
+
|
||||
+ def test_parse_qs_separator_priority(self):
|
||||
+ # env variable trumps config file
|
||||
+ with self._qsl_sep_config('~'), EnvironmentVarGuard() as environ:
|
||||
+ environ['PYTHON_URLLIB_QS_SEPARATOR'] = '!'
|
||||
+ result = urllib.parse.parse_qs('a=1!b=2~c=3')
|
||||
+ self.assertEqual(result, {'a': ['1'], 'b': ['2~c=3']})
|
||||
+ # argument trumps config file
|
||||
+ with self._qsl_sep_config('~'):
|
||||
+ result = urllib.parse.parse_qs('a=1$b=2~c=3', separator='$')
|
||||
+ self.assertEqual(result, {'a': ['1'], 'b': ['2~c=3']})
|
||||
+ # argument trumps env variable
|
||||
+ with EnvironmentVarGuard() as environ:
|
||||
+ environ['PYTHON_URLLIB_QS_SEPARATOR'] = '~'
|
||||
+ result = urllib.parse.parse_qs('a=1$b=2~c=3', separator='$')
|
||||
+ self.assertEqual(result, {'a': ['1'], 'b': ['2~c=3']})
|
||||
|
||||
def test_urlencode_sequences(self):
|
||||
# Other tests incidentally urlencode things; test non-covered cases:
|
||||
diff --git a/Lib/urllib/parse.py b/Lib/urllib/parse.py
|
||||
index fa8827a9fa7..57b8fcf8bbd 100644
|
||||
--- a/Lib/urllib/parse.py
|
||||
+++ b/Lib/urllib/parse.py
|
||||
@@ -28,6 +28,7 @@ test_urlparse.py provides a good indicator of parsing behavior.
|
||||
"""
|
||||
|
||||
import re
|
||||
+import os
|
||||
import sys
|
||||
import collections
|
||||
|
||||
@@ -644,7 +645,8 @@ def unquote(string, encoding='utf-8', errors='replace'):
|
||||
|
||||
|
||||
def parse_qs(qs, keep_blank_values=False, strict_parsing=False,
|
||||
- encoding='utf-8', errors='replace', max_num_fields=None):
|
||||
+ encoding='utf-8', errors='replace', max_num_fields=None,
|
||||
+ separator=None):
|
||||
"""Parse a query given as a string argument.
|
||||
|
||||
Arguments:
|
||||
@@ -673,7 +675,8 @@ def parse_qs(qs, keep_blank_values=False, strict_parsing=False,
|
||||
parsed_result = {}
|
||||
pairs = parse_qsl(qs, keep_blank_values, strict_parsing,
|
||||
encoding=encoding, errors=errors,
|
||||
- max_num_fields=max_num_fields)
|
||||
+ max_num_fields=max_num_fields,
|
||||
+ separator=separator)
|
||||
for name, value in pairs:
|
||||
if name in parsed_result:
|
||||
parsed_result[name].append(value)
|
||||
@@ -681,9 +684,16 @@ def parse_qs(qs, keep_blank_values=False, strict_parsing=False,
|
||||
parsed_result[name] = [value]
|
||||
return parsed_result
|
||||
|
||||
+class _QueryStringSeparatorWarning(RuntimeWarning):
|
||||
+ """Warning for using default `separator` in parse_qs or parse_qsl"""
|
||||
+
|
||||
+# The default "separator" for parse_qsl can be specified in a config file.
|
||||
+# It's cached after first read.
|
||||
+_QS_SEPARATOR_CONFIG_FILENAME = '/etc/python/urllib.cfg'
|
||||
+_default_qs_separator = None
|
||||
|
||||
def parse_qsl(qs, keep_blank_values=False, strict_parsing=False,
|
||||
- encoding='utf-8', errors='replace', max_num_fields=None):
|
||||
+ encoding='utf-8', errors='replace', max_num_fields=None, separator=None):
|
||||
"""Parse a query given as a string argument.
|
||||
|
||||
Arguments:
|
||||
@@ -710,15 +720,77 @@ def parse_qsl(qs, keep_blank_values=False, strict_parsing=False,
|
||||
"""
|
||||
qs, _coerce_result = _coerce_args(qs)
|
||||
|
||||
+ if isinstance(separator, bytes):
|
||||
+ separator = separator.decode('ascii')
|
||||
+
|
||||
+ if (not separator or (not isinstance(separator, (str, bytes)))) and separator is not None:
|
||||
+ raise ValueError("Separator must be of type string or bytes.")
|
||||
+
|
||||
+ # Used when both "&" and ";" act as separators. (Need a non-string value.)
|
||||
+ _legacy = object()
|
||||
+
|
||||
+ if separator is None:
|
||||
+ global _default_qs_separator
|
||||
+ separator = _default_qs_separator
|
||||
+ envvar_name = 'PYTHON_URLLIB_QS_SEPARATOR'
|
||||
+ if separator is None:
|
||||
+ # Set default separator from environment variable
|
||||
+ separator = os.environ.get(envvar_name)
|
||||
+ config_source = 'environment variable'
|
||||
+ if separator is None:
|
||||
+ # Set default separator from the configuration file
|
||||
+ try:
|
||||
+ file = open(_QS_SEPARATOR_CONFIG_FILENAME)
|
||||
+ except FileNotFoundError:
|
||||
+ pass
|
||||
+ else:
|
||||
+ with file:
|
||||
+ import configparser
|
||||
+ config = configparser.ConfigParser(
|
||||
+ interpolation=None,
|
||||
+ comment_prefixes=('#', ),
|
||||
+ )
|
||||
+ config.read_file(file)
|
||||
+ separator = config.get('parse_qs', envvar_name, fallback=None)
|
||||
+ _default_qs_separator = separator
|
||||
+ config_source = _QS_SEPARATOR_CONFIG_FILENAME
|
||||
+ if separator is None:
|
||||
+ # The default is '&', but warn if not specified explicitly
|
||||
+ if ';' in qs:
|
||||
+ from warnings import warn
|
||||
+ warn("The default separator of urllib.parse.parse_qsl and "
|
||||
+ + "parse_qs was changed to '&' to avoid a web cache "
|
||||
+ + "poisoning issue (CVE-2021-23336). "
|
||||
+ + "By default, semicolons no longer act as query field "
|
||||
+ + "separators. "
|
||||
+ + "See https://access.redhat.com/articles/5860431 for "
|
||||
+ + "more details.",
|
||||
+ _QueryStringSeparatorWarning, stacklevel=2)
|
||||
+ separator = '&'
|
||||
+ elif separator == 'legacy':
|
||||
+ separator = _legacy
|
||||
+ elif len(separator) != 1:
|
||||
+ raise ValueError(
|
||||
+ f'{envvar_name} (from {config_source}) must contain '
|
||||
+ + '1 character, or "legacy". See '
|
||||
+ + 'https://access.redhat.com/articles/5860431 for more details.'
|
||||
+ )
|
||||
+
|
||||
# If max_num_fields is defined then check that the number of fields
|
||||
# is less than max_num_fields. This prevents a memory exhaustion DOS
|
||||
# attack via post bodies with many fields.
|
||||
if max_num_fields is not None:
|
||||
- num_fields = 1 + qs.count('&') + qs.count(';')
|
||||
+ if separator is _legacy:
|
||||
+ num_fields = 1 + qs.count('&') + qs.count(';')
|
||||
+ else:
|
||||
+ num_fields = 1 + qs.count(separator)
|
||||
if max_num_fields < num_fields:
|
||||
raise ValueError('Max number of fields exceeded')
|
||||
|
||||
- pairs = [s2 for s1 in qs.split('&') for s2 in s1.split(';')]
|
||||
+ if separator is _legacy:
|
||||
+ pairs = [s2 for s1 in qs.split('&') for s2 in s1.split(';')]
|
||||
+ else:
|
||||
+ pairs = [s1 for s1 in qs.split(separator)]
|
||||
r = []
|
||||
for name_value in pairs:
|
||||
if not name_value and not strict_parsing:
|
||||
diff --git a/Misc/NEWS.d/next/Security/2021-02-14-15-59-16.bpo-42967.YApqDS.rst b/Misc/NEWS.d/next/Security/2021-02-14-15-59-16.bpo-42967.YApqDS.rst
|
||||
new file mode 100644
|
||||
index 00000000000..bc82c963067
|
||||
--- /dev/null
|
||||
+++ b/Misc/NEWS.d/next/Security/2021-02-14-15-59-16.bpo-42967.YApqDS.rst
|
||||
@@ -0,0 +1 @@
|
||||
+Make it possible to fix web cache poisoning vulnerability by allowing the user to choose a custom separator query args.
|
|
@ -0,0 +1,101 @@
|
|||
From 5b1e50256b6532667b6d31debc350f6c7d3f30aa Mon Sep 17 00:00:00 2001
|
||||
From: "Miss Islington (bot)"
|
||||
<31488909+miss-islington@users.noreply.github.com>
|
||||
Date: Mon, 29 Mar 2021 08:40:53 -0700
|
||||
Subject: [PATCH] bpo-42988: Remove the pydoc getfile feature (GH-25015)
|
||||
(GH-25067)
|
||||
MIME-Version: 1.0
|
||||
Content-Type: text/plain; charset=UTF-8
|
||||
Content-Transfer-Encoding: 8bit
|
||||
|
||||
CVE-2021-3426: Remove the "getfile" feature of the pydoc module which
|
||||
could be abused to read arbitrary files on the disk (directory
|
||||
traversal vulnerability). Moreover, even source code of Python
|
||||
modules can contain sensitive data like passwords. Vulnerability
|
||||
reported by David Schwörer.
|
||||
(cherry picked from commit 9b999479c0022edfc9835a8a1f06e046f3881048)
|
||||
|
||||
Co-authored-by: Victor Stinner <vstinner@python.org>
|
||||
---
|
||||
Lib/pydoc.py | 18 ------------------
|
||||
Lib/test/test_pydoc.py | 6 ------
|
||||
.../2021-03-24-14-16-56.bpo-42988.P2aNco.rst | 4 ++++
|
||||
3 files changed, 4 insertions(+), 24 deletions(-)
|
||||
create mode 100644 Misc/NEWS.d/next/Security/2021-03-24-14-16-56.bpo-42988.P2aNco.rst
|
||||
|
||||
diff --git a/Lib/pydoc.py b/Lib/pydoc.py
|
||||
index b521a5504728c4..5247ef9ea27aa1 100644
|
||||
--- a/Lib/pydoc.py
|
||||
+++ b/Lib/pydoc.py
|
||||
@@ -2312,9 +2312,6 @@ def page(self, title, contents):
|
||||
%s</head><body bgcolor="#f0f0f8">%s<div style="clear:both;padding-top:.5em;">%s</div>
|
||||
</body></html>''' % (title, css_link, html_navbar(), contents)
|
||||
|
||||
- def filelink(self, url, path):
|
||||
- return '<a href="getfile?key=%s">%s</a>' % (url, path)
|
||||
-
|
||||
|
||||
html = _HTMLDoc()
|
||||
|
||||
@@ -2400,19 +2397,6 @@ def bltinlink(name):
|
||||
'key = %s' % key, '#ffffff', '#ee77aa', '<br>'.join(results))
|
||||
return 'Search Results', contents
|
||||
|
||||
- def html_getfile(path):
|
||||
- """Get and display a source file listing safely."""
|
||||
- path = urllib.parse.unquote(path)
|
||||
- with tokenize.open(path) as fp:
|
||||
- lines = html.escape(fp.read())
|
||||
- body = '<pre>%s</pre>' % lines
|
||||
- heading = html.heading(
|
||||
- '<big><big><strong>File Listing</strong></big></big>',
|
||||
- '#ffffff', '#7799ee')
|
||||
- contents = heading + html.bigsection(
|
||||
- 'File: %s' % path, '#ffffff', '#ee77aa', body)
|
||||
- return 'getfile %s' % path, contents
|
||||
-
|
||||
def html_topics():
|
||||
"""Index of topic texts available."""
|
||||
|
||||
@@ -2504,8 +2488,6 @@ def get_html_page(url):
|
||||
op, _, url = url.partition('=')
|
||||
if op == "search?key":
|
||||
title, content = html_search(url)
|
||||
- elif op == "getfile?key":
|
||||
- title, content = html_getfile(url)
|
||||
elif op == "topic?key":
|
||||
# try topics first, then objects.
|
||||
try:
|
||||
diff --git a/Lib/test/test_pydoc.py b/Lib/test/test_pydoc.py
|
||||
index 00803d3305cb53..49bc3eb164b19c 100644
|
||||
--- a/Lib/test/test_pydoc.py
|
||||
+++ b/Lib/test/test_pydoc.py
|
||||
@@ -1052,18 +1052,12 @@ def test_url_requests(self):
|
||||
("topic?key=def", "Pydoc: KEYWORD def"),
|
||||
("topic?key=STRINGS", "Pydoc: TOPIC STRINGS"),
|
||||
("foobar", "Pydoc: Error - foobar"),
|
||||
- ("getfile?key=foobar", "Pydoc: Error - getfile?key=foobar"),
|
||||
]
|
||||
|
||||
with self.restrict_walk_packages():
|
||||
for url, title in requests:
|
||||
self.call_url_handler(url, title)
|
||||
|
||||
- path = string.__file__
|
||||
- title = "Pydoc: getfile " + path
|
||||
- url = "getfile?key=" + path
|
||||
- self.call_url_handler(url, title)
|
||||
-
|
||||
|
||||
class TestHelper(unittest.TestCase):
|
||||
def test_keywords(self):
|
||||
diff --git a/Misc/NEWS.d/next/Security/2021-03-24-14-16-56.bpo-42988.P2aNco.rst b/Misc/NEWS.d/next/Security/2021-03-24-14-16-56.bpo-42988.P2aNco.rst
|
||||
new file mode 100644
|
||||
index 00000000000000..4b42dd05305a83
|
||||
--- /dev/null
|
||||
+++ b/Misc/NEWS.d/next/Security/2021-03-24-14-16-56.bpo-42988.P2aNco.rst
|
||||
@@ -0,0 +1,4 @@
|
||||
+CVE-2021-3426: Remove the ``getfile`` feature of the :mod:`pydoc` module which
|
||||
+could be abused to read arbitrary files on the disk (directory traversal
|
||||
+vulnerability). Moreover, even source code of Python modules can contain
|
||||
+sensitive data like passwords. Vulnerability reported by David Schwörer.
|
|
@ -0,0 +1,36 @@
|
|||
bpo-44422: Fix threading.enumerate() reentrant call (GH-26727)
|
||||
|
||||
The threading.enumerate() function now uses a reentrant lock to
|
||||
prevent a hang on reentrant call.
|
||||
|
||||
https://github.com/python/cpython/commit/243fd01047ddce1a7eb0f99a49732d123e942c63
|
||||
|
||||
Resolves: rhbz#1959459
|
||||
|
||||
diff --git a/Lib/threading.py b/Lib/threading.py
|
||||
index 0ab1e46..7ab9ad8 100644
|
||||
--- a/Lib/threading.py
|
||||
+++ b/Lib/threading.py
|
||||
@@ -727,8 +727,11 @@ _counter() # Consume 0 so first non-main thread has id 1.
|
||||
def _newname(template="Thread-%d"):
|
||||
return template % _counter()
|
||||
|
||||
-# Active thread administration
|
||||
-_active_limbo_lock = _allocate_lock()
|
||||
+# Active thread administration.
|
||||
+#
|
||||
+# bpo-44422: Use a reentrant lock to allow reentrant calls to functions like
|
||||
+# threading.enumerate().
|
||||
+_active_limbo_lock = RLock()
|
||||
_active = {} # maps thread id to Thread object
|
||||
_limbo = {}
|
||||
_dangling = WeakSet()
|
||||
@@ -1325,7 +1328,7 @@ def _after_fork():
|
||||
# Reset _active_limbo_lock, in case we forked while the lock was held
|
||||
# by another (non-forked) thread. http://bugs.python.org/issue874900
|
||||
global _active_limbo_lock, _main_thread
|
||||
- _active_limbo_lock = _allocate_lock()
|
||||
+ _active_limbo_lock = RLock()
|
||||
|
||||
# fork() only copied the current thread; clear references to others.
|
||||
new_active = {}
|
|
@ -0,0 +1,43 @@
|
|||
bpo-44434: Don't call PyThread_exit_thread() explicitly (GH-26758)
|
||||
|
||||
_thread.start_new_thread() no longer calls PyThread_exit_thread()
|
||||
explicitly at the thread exit, the call was redundant.
|
||||
|
||||
On Linux with the glibc, pthread_cancel() loads dynamically the
|
||||
libgcc_s.so.1 library. dlopen() can fail if there is no more
|
||||
available file descriptor to open the file. In this case, the process
|
||||
aborts with the error message:
|
||||
|
||||
"libgcc_s.so.1 must be installed for pthread_cancel to work"
|
||||
|
||||
pthread_cancel() unwinds back to the thread's wrapping function that
|
||||
calls the thread entry point.
|
||||
|
||||
The unwind function is dynamically loaded from the libgcc_s library
|
||||
since it is tightly coupled to the C compiler (GCC). The unwinder
|
||||
depends on DWARF, the compiler generates DWARF, so the unwinder
|
||||
belongs to the compiler.
|
||||
|
||||
Thanks Florian Weimer and Carlos O'Donell for their help on
|
||||
investigating this issue.
|
||||
|
||||
https://github.com/python/cpython/commit/45a78f906d2d5fe5381d78466b11763fc56d57ba
|
||||
|
||||
Resolves: rhbz#1972293
|
||||
|
||||
diff --git a/Modules/_threadmodule.c b/Modules/_threadmodule.c
|
||||
index a13b2e0..8cc035b 100644
|
||||
--- a/Modules/_threadmodule.c
|
||||
+++ b/Modules/_threadmodule.c
|
||||
@@ -1027,7 +1027,10 @@ t_bootstrap(void *boot_raw)
|
||||
nb_threads--;
|
||||
PyThreadState_Clear(tstate);
|
||||
PyThreadState_DeleteCurrent();
|
||||
- PyThread_exit_thread();
|
||||
+
|
||||
+ // bpo-44434: Don't call explicitly PyThread_exit_thread(). On Linux with
|
||||
+ // the glibc, pthread_exit() can abort the whole process if dlopen() fails
|
||||
+ // to open the libgcc_s.so library (ex: EMFILE error).
|
||||
}
|
||||
|
||||
static PyObject *
|
|
@ -0,0 +1,40 @@
|
|||
From 29c669440dddba61d18e1b7fdd57180cae9e4ae3 Mon Sep 17 00:00:00 2001
|
||||
From: Yeting Li <liyt@ios.ac.cn>
|
||||
Date: Wed, 7 Apr 2021 19:27:41 +0800
|
||||
Subject: [PATCH] bpo-43075: Fix ReDoS in urllib AbstractBasicAuthHandler
|
||||
(GH-24391)
|
||||
|
||||
Fix Regular Expression Denial of Service (ReDoS) vulnerability in
|
||||
urllib.request.AbstractBasicAuthHandler. The ReDoS-vulnerable regex
|
||||
has quadratic worst-case complexity and it allows cause a denial of
|
||||
service when identifying crafted invalid RFCs. This ReDoS issue is on
|
||||
the client side and needs remote attackers to control the HTTP server.
|
||||
(cherry picked from commit 7215d1ae25525c92b026166f9d5cac85fb1defe1)
|
||||
|
||||
Co-authored-by: Yeting Li <liyt@ios.ac.cn>
|
||||
---
|
||||
Lib/urllib/request.py | 2 +-
|
||||
.../next/Security/2021-01-31-05-28-14.bpo-43075.DoAXqO.rst | 1 +
|
||||
2 files changed, 2 insertions(+), 1 deletion(-)
|
||||
create mode 100644 Misc/NEWS.d/next/Security/2021-01-31-05-28-14.bpo-43075.DoAXqO.rst
|
||||
|
||||
diff --git a/Lib/urllib/request.py b/Lib/urllib/request.py
|
||||
index 6624e04317ba2..56565405a7097 100644
|
||||
--- a/Lib/urllib/request.py
|
||||
+++ b/Lib/urllib/request.py
|
||||
@@ -947,7 +947,7 @@ class AbstractBasicAuthHandler:
|
||||
# (single quotes are a violation of the RFC, but appear in the wild)
|
||||
rx = re.compile('(?:^|,)' # start of the string or ','
|
||||
'[ \t]*' # optional whitespaces
|
||||
- '([^ \t]+)' # scheme like "Basic"
|
||||
+ '([^ \t,]+)' # scheme like "Basic"
|
||||
'[ \t]+' # mandatory whitespaces
|
||||
# realm=xxx
|
||||
# realm='xxx'
|
||||
diff --git a/Misc/NEWS.d/next/Security/2021-01-31-05-28-14.bpo-43075.DoAXqO.rst b/Misc/NEWS.d/next/Security/2021-01-31-05-28-14.bpo-43075.DoAXqO.rst
|
||||
new file mode 100644
|
||||
index 0000000000000..1c9f727e965fb
|
||||
--- /dev/null
|
||||
+++ b/Misc/NEWS.d/next/Security/2021-01-31-05-28-14.bpo-43075.DoAXqO.rst
|
||||
@@ -0,0 +1 @@
|
||||
+Fix Regular Expression Denial of Service (ReDoS) vulnerability in :class:`urllib.request.AbstractBasicAuthHandler`. The ReDoS-vulnerable regex has quadratic worst-case complexity and it allows cause a denial of service when identifying crafted invalid RFCs. This ReDoS issue is on the client side and needs remote attackers to control the HTTP server.
|
|
@ -0,0 +1,119 @@
|
|||
From f7fb35b563a9182c22fbdd03c72ec3724dafe918 Mon Sep 17 00:00:00 2001
|
||||
From: Gen Xu <xgbarry@gmail.com>
|
||||
Date: Wed, 5 May 2021 15:42:41 -0700
|
||||
Subject: [PATCH] bpo-44022: Fix http client infinite line reading (DoS) after
|
||||
a HTTP 100 Continue (GH-25916)
|
||||
|
||||
Fixes http.client potential denial of service where it could get stuck reading lines from a malicious server after a 100 Continue response.
|
||||
|
||||
Co-authored-by: Gregory P. Smith <greg@krypto.org>
|
||||
(cherry picked from commit 47895e31b6f626bc6ce47d175fe9d43c1098909d)
|
||||
|
||||
Co-authored-by: Gen Xu <xgbarry@gmail.com>
|
||||
---
|
||||
Lib/http/client.py | 38 ++++++++++---------
|
||||
Lib/test/test_httplib.py | 10 ++++-
|
||||
.../2021-05-05-17-37-04.bpo-44022.bS3XJ9.rst | 2 +
|
||||
3 files changed, 32 insertions(+), 18 deletions(-)
|
||||
create mode 100644 Misc/NEWS.d/next/Security/2021-05-05-17-37-04.bpo-44022.bS3XJ9.rst
|
||||
|
||||
diff --git a/Lib/http/client.py b/Lib/http/client.py
|
||||
index 53581eca20587..07e675fac5981 100644
|
||||
--- a/Lib/http/client.py
|
||||
+++ b/Lib/http/client.py
|
||||
@@ -205,15 +205,11 @@ def getallmatchingheaders(self, name):
|
||||
lst.append(line)
|
||||
return lst
|
||||
|
||||
-def parse_headers(fp, _class=HTTPMessage):
|
||||
- """Parses only RFC2822 headers from a file pointer.
|
||||
-
|
||||
- email Parser wants to see strings rather than bytes.
|
||||
- But a TextIOWrapper around self.rfile would buffer too many bytes
|
||||
- from the stream, bytes which we later need to read as bytes.
|
||||
- So we read the correct bytes here, as bytes, for email Parser
|
||||
- to parse.
|
||||
+def _read_headers(fp):
|
||||
+ """Reads potential header lines into a list from a file pointer.
|
||||
|
||||
+ Length of line is limited by _MAXLINE, and number of
|
||||
+ headers is limited by _MAXHEADERS.
|
||||
"""
|
||||
headers = []
|
||||
while True:
|
||||
@@ -225,6 +221,19 @@ def parse_headers(fp, _class=HTTPMessage):
|
||||
raise HTTPException("got more than %d headers" % _MAXHEADERS)
|
||||
if line in (b'\r\n', b'\n', b''):
|
||||
break
|
||||
+ return headers
|
||||
+
|
||||
+def parse_headers(fp, _class=HTTPMessage):
|
||||
+ """Parses only RFC2822 headers from a file pointer.
|
||||
+
|
||||
+ email Parser wants to see strings rather than bytes.
|
||||
+ But a TextIOWrapper around self.rfile would buffer too many bytes
|
||||
+ from the stream, bytes which we later need to read as bytes.
|
||||
+ So we read the correct bytes here, as bytes, for email Parser
|
||||
+ to parse.
|
||||
+
|
||||
+ """
|
||||
+ headers = _read_headers(fp)
|
||||
hstring = b''.join(headers).decode('iso-8859-1')
|
||||
return email.parser.Parser(_class=_class).parsestr(hstring)
|
||||
|
||||
@@ -312,15 +321,10 @@ def begin(self):
|
||||
if status != CONTINUE:
|
||||
break
|
||||
# skip the header from the 100 response
|
||||
- while True:
|
||||
- skip = self.fp.readline(_MAXLINE + 1)
|
||||
- if len(skip) > _MAXLINE:
|
||||
- raise LineTooLong("header line")
|
||||
- skip = skip.strip()
|
||||
- if not skip:
|
||||
- break
|
||||
- if self.debuglevel > 0:
|
||||
- print("header:", skip)
|
||||
+ skipped_headers = _read_headers(self.fp)
|
||||
+ if self.debuglevel > 0:
|
||||
+ print("headers:", skipped_headers)
|
||||
+ del skipped_headers
|
||||
|
||||
self.code = self.status = status
|
||||
self.reason = reason.strip()
|
||||
diff --git a/Lib/test/test_httplib.py b/Lib/test/test_httplib.py
|
||||
index 03e049b13fd21..0db287507c7bf 100644
|
||||
--- a/Lib/test/test_httplib.py
|
||||
+++ b/Lib/test/test_httplib.py
|
||||
@@ -971,6 +971,14 @@ def test_overflowing_header_line(self):
|
||||
resp = client.HTTPResponse(FakeSocket(body))
|
||||
self.assertRaises(client.LineTooLong, resp.begin)
|
||||
|
||||
+ def test_overflowing_header_limit_after_100(self):
|
||||
+ body = (
|
||||
+ 'HTTP/1.1 100 OK\r\n'
|
||||
+ 'r\n' * 32768
|
||||
+ )
|
||||
+ resp = client.HTTPResponse(FakeSocket(body))
|
||||
+ self.assertRaises(client.HTTPException, resp.begin)
|
||||
+
|
||||
def test_overflowing_chunked_line(self):
|
||||
body = (
|
||||
'HTTP/1.1 200 OK\r\n'
|
||||
@@ -1377,7 +1385,7 @@ def readline(self, limit):
|
||||
class OfflineTest(TestCase):
|
||||
def test_all(self):
|
||||
# Documented objects defined in the module should be in __all__
|
||||
- expected = {"responses"} # White-list documented dict() object
|
||||
+ expected = {"responses"} # Allowlist documented dict() object
|
||||
# HTTPMessage, parse_headers(), and the HTTP status code constants are
|
||||
# intentionally omitted for simplicity
|
||||
blacklist = {"HTTPMessage", "parse_headers"}
|
||||
diff --git a/Misc/NEWS.d/next/Security/2021-05-05-17-37-04.bpo-44022.bS3XJ9.rst b/Misc/NEWS.d/next/Security/2021-05-05-17-37-04.bpo-44022.bS3XJ9.rst
|
||||
new file mode 100644
|
||||
index 0000000000000..cf6b63e396155
|
||||
--- /dev/null
|
||||
+++ b/Misc/NEWS.d/next/Security/2021-05-05-17-37-04.bpo-44022.bS3XJ9.rst
|
||||
@@ -0,0 +1,2 @@
|
||||
+mod:`http.client` now avoids infinitely reading potential HTTP headers after a
|
||||
+``100 Continue`` status response from the server.
|
|
@ -0,0 +1,74 @@
|
|||
diff --git a/Lib/logging/handlers.py b/Lib/logging/handlers.py
|
||||
index 11ebcf1..ee3d960 100644
|
||||
--- a/Lib/logging/handlers.py
|
||||
+++ b/Lib/logging/handlers.py
|
||||
@@ -181,14 +181,17 @@ class RotatingFileHandler(BaseRotatingHandler):
|
||||
Basically, see if the supplied record would cause the file to exceed
|
||||
the size limit we have.
|
||||
"""
|
||||
+ # See bpo-45401: Never rollover anything other than regular files
|
||||
+ if os.path.exists(self.baseFilename) and not os.path.isfile(self.baseFilename):
|
||||
+ return False
|
||||
if self.stream is None: # delay was set...
|
||||
self.stream = self._open()
|
||||
if self.maxBytes > 0: # are we rolling over?
|
||||
msg = "%s\n" % self.format(record)
|
||||
self.stream.seek(0, 2) #due to non-posix-compliant Windows feature
|
||||
if self.stream.tell() + len(msg) >= self.maxBytes:
|
||||
- return 1
|
||||
- return 0
|
||||
+ return True
|
||||
+ return False
|
||||
|
||||
class TimedRotatingFileHandler(BaseRotatingHandler):
|
||||
"""
|
||||
@@ -335,10 +338,13 @@ class TimedRotatingFileHandler(BaseRotatingHandler):
|
||||
record is not used, as we are just comparing times, but it is needed so
|
||||
the method signatures are the same
|
||||
"""
|
||||
+ # See bpo-45401: Never rollover anything other than regular files
|
||||
+ if os.path.exists(self.baseFilename) and not os.path.isfile(self.baseFilename):
|
||||
+ return False
|
||||
t = int(time.time())
|
||||
if t >= self.rolloverAt:
|
||||
- return 1
|
||||
- return 0
|
||||
+ return True
|
||||
+ return False
|
||||
|
||||
def getFilesToDelete(self):
|
||||
"""
|
||||
diff --git a/Lib/test/test_logging.py b/Lib/test/test_logging.py
|
||||
index 45b72e3..055b8e3 100644
|
||||
--- a/Lib/test/test_logging.py
|
||||
+++ b/Lib/test/test_logging.py
|
||||
@@ -4219,6 +4219,13 @@ class RotatingFileHandlerTest(BaseFileTest):
|
||||
rh = logging.handlers.RotatingFileHandler(self.fn, maxBytes=0)
|
||||
self.assertFalse(rh.shouldRollover(None))
|
||||
rh.close()
|
||||
+ # bpo-45401 - test with special file
|
||||
+ # We set maxBytes to 1 so that rollover would normally happen, except
|
||||
+ # for the check for regular files
|
||||
+ rh = logging.handlers.RotatingFileHandler(
|
||||
+ os.devnull, encoding="utf-8", maxBytes=1)
|
||||
+ self.assertFalse(rh.shouldRollover(self.next_rec()))
|
||||
+ rh.close()
|
||||
|
||||
def test_should_rollover(self):
|
||||
rh = logging.handlers.RotatingFileHandler(self.fn, maxBytes=1)
|
||||
@@ -4294,6 +4301,15 @@ class RotatingFileHandlerTest(BaseFileTest):
|
||||
rh.close()
|
||||
|
||||
class TimedRotatingFileHandlerTest(BaseFileTest):
|
||||
+ def test_should_not_rollover(self):
|
||||
+ # See bpo-45401. Should only ever rollover regular files
|
||||
+ fh = logging.handlers.TimedRotatingFileHandler(
|
||||
+ os.devnull, 'S', encoding="utf-8", backupCount=1)
|
||||
+ time.sleep(1.1) # a little over a second ...
|
||||
+ r = logging.makeLogRecord({'msg': 'testing - device file'})
|
||||
+ self.assertFalse(fh.shouldRollover(r))
|
||||
+ fh.close()
|
||||
+
|
||||
# other test methods added below
|
||||
def test_rollover(self):
|
||||
fh = logging.handlers.TimedRotatingFileHandler(self.fn, 'S',
|
|
@ -0,0 +1,267 @@
|
|||
diff --git a/Makefile.pre.in b/Makefile.pre.in
|
||||
index 8da1965..9864fe2 100644
|
||||
--- a/Makefile.pre.in
|
||||
+++ b/Makefile.pre.in
|
||||
@@ -884,7 +884,8 @@ regen-opcode-targets:
|
||||
$(srcdir)/Python/opcode_targets.h.new
|
||||
$(UPDATE_FILE) $(srcdir)/Python/opcode_targets.h $(srcdir)/Python/opcode_targets.h.new
|
||||
|
||||
-Python/ceval.o: $(srcdir)/Python/opcode_targets.h $(srcdir)/Python/ceval_gil.h
|
||||
+Python/ceval.o: $(srcdir)/Python/opcode_targets.h $(srcdir)/Python/ceval_gil.h \
|
||||
+ $(srcdir)/Python/condvar.h
|
||||
|
||||
Python/frozen.o: $(srcdir)/Python/importlib.h $(srcdir)/Python/importlib_external.h
|
||||
|
||||
@@ -1706,7 +1707,7 @@ patchcheck: @DEF_MAKE_RULE@
|
||||
|
||||
# Dependencies
|
||||
|
||||
-Python/thread.o: @THREADHEADERS@
|
||||
+Python/thread.o: @THREADHEADERS@ $(srcdir)/Python/condvar.h
|
||||
|
||||
# Declare targets that aren't real files
|
||||
.PHONY: all build_all sharedmods check-clean-src oldsharedmods test quicktest
|
||||
diff --git a/Python/ceval.c b/Python/ceval.c
|
||||
index 0b30cc1..3f1300c 100644
|
||||
--- a/Python/ceval.c
|
||||
+++ b/Python/ceval.c
|
||||
@@ -232,6 +232,7 @@ PyEval_InitThreads(void)
|
||||
{
|
||||
if (gil_created())
|
||||
return;
|
||||
+ PyThread_init_thread();
|
||||
create_gil();
|
||||
take_gil(PyThreadState_GET());
|
||||
main_thread = PyThread_get_thread_ident();
|
||||
diff --git a/Python/condvar.h b/Python/condvar.h
|
||||
index 9a71b17..39a420f 100644
|
||||
--- a/Python/condvar.h
|
||||
+++ b/Python/condvar.h
|
||||
@@ -59,20 +59,6 @@
|
||||
|
||||
#include <pthread.h>
|
||||
|
||||
-#define PyCOND_ADD_MICROSECONDS(tv, interval) \
|
||||
-do { /* TODO: add overflow and truncation checks */ \
|
||||
- tv.tv_usec += (long) interval; \
|
||||
- tv.tv_sec += tv.tv_usec / 1000000; \
|
||||
- tv.tv_usec %= 1000000; \
|
||||
-} while (0)
|
||||
-
|
||||
-/* We assume all modern POSIX systems have gettimeofday() */
|
||||
-#ifdef GETTIMEOFDAY_NO_TZ
|
||||
-#define PyCOND_GETTIMEOFDAY(ptv) gettimeofday(ptv)
|
||||
-#else
|
||||
-#define PyCOND_GETTIMEOFDAY(ptv) gettimeofday(ptv, (struct timezone *)NULL)
|
||||
-#endif
|
||||
-
|
||||
/* The following functions return 0 on success, nonzero on error */
|
||||
#define PyMUTEX_T pthread_mutex_t
|
||||
#define PyMUTEX_INIT(mut) pthread_mutex_init((mut), NULL)
|
||||
@@ -81,32 +67,30 @@ do { /* TODO: add overflow and truncation checks */ \
|
||||
#define PyMUTEX_UNLOCK(mut) pthread_mutex_unlock(mut)
|
||||
|
||||
#define PyCOND_T pthread_cond_t
|
||||
-#define PyCOND_INIT(cond) pthread_cond_init((cond), NULL)
|
||||
+#define PyCOND_INIT(cond) _PyThread_cond_init(cond)
|
||||
#define PyCOND_FINI(cond) pthread_cond_destroy(cond)
|
||||
#define PyCOND_SIGNAL(cond) pthread_cond_signal(cond)
|
||||
#define PyCOND_BROADCAST(cond) pthread_cond_broadcast(cond)
|
||||
#define PyCOND_WAIT(cond, mut) pthread_cond_wait((cond), (mut))
|
||||
|
||||
+/* These private functions are implemented in Python/thread_pthread.h */
|
||||
+int _PyThread_cond_init(PyCOND_T *cond);
|
||||
+void _PyThread_cond_after(long long us, struct timespec *abs);
|
||||
+
|
||||
/* return 0 for success, 1 on timeout, -1 on error */
|
||||
Py_LOCAL_INLINE(int)
|
||||
PyCOND_TIMEDWAIT(PyCOND_T *cond, PyMUTEX_T *mut, long long us)
|
||||
{
|
||||
- int r;
|
||||
- struct timespec ts;
|
||||
- struct timeval deadline;
|
||||
-
|
||||
- PyCOND_GETTIMEOFDAY(&deadline);
|
||||
- PyCOND_ADD_MICROSECONDS(deadline, us);
|
||||
- ts.tv_sec = deadline.tv_sec;
|
||||
- ts.tv_nsec = deadline.tv_usec * 1000;
|
||||
-
|
||||
- r = pthread_cond_timedwait((cond), (mut), &ts);
|
||||
- if (r == ETIMEDOUT)
|
||||
+ struct timespec abs;
|
||||
+ _PyThread_cond_after(us, &abs);
|
||||
+ int ret = pthread_cond_timedwait(cond, mut, &abs);
|
||||
+ if (ret == ETIMEDOUT) {
|
||||
return 1;
|
||||
- else if (r)
|
||||
+ }
|
||||
+ if (ret) {
|
||||
return -1;
|
||||
- else
|
||||
- return 0;
|
||||
+ }
|
||||
+ return 0;
|
||||
}
|
||||
|
||||
#elif defined(NT_THREADS)
|
||||
diff --git a/Python/thread.c b/Python/thread.c
|
||||
index 63eeb1e..c5d0e59 100644
|
||||
--- a/Python/thread.c
|
||||
+++ b/Python/thread.c
|
||||
@@ -6,6 +6,7 @@
|
||||
Stuff shared by all thread_*.h files is collected here. */
|
||||
|
||||
#include "Python.h"
|
||||
+#include "condvar.h"
|
||||
|
||||
#ifndef _POSIX_THREADS
|
||||
/* This means pthreads are not implemented in libc headers, hence the macro
|
||||
diff --git a/Python/thread_pthread.h b/Python/thread_pthread.h
|
||||
index baea71f..7dc295e 100644
|
||||
--- a/Python/thread_pthread.h
|
||||
+++ b/Python/thread_pthread.h
|
||||
@@ -66,16 +66,6 @@
|
||||
#endif
|
||||
#endif
|
||||
|
||||
-#if !defined(pthread_attr_default)
|
||||
-# define pthread_attr_default ((pthread_attr_t *)NULL)
|
||||
-#endif
|
||||
-#if !defined(pthread_mutexattr_default)
|
||||
-# define pthread_mutexattr_default ((pthread_mutexattr_t *)NULL)
|
||||
-#endif
|
||||
-#if !defined(pthread_condattr_default)
|
||||
-# define pthread_condattr_default ((pthread_condattr_t *)NULL)
|
||||
-#endif
|
||||
-
|
||||
|
||||
/* Whether or not to use semaphores directly rather than emulating them with
|
||||
* mutexes and condition variables:
|
||||
@@ -120,6 +110,56 @@ do { \
|
||||
} while(0)
|
||||
|
||||
|
||||
+/*
|
||||
+ * pthread_cond support
|
||||
+ */
|
||||
+
|
||||
+#if defined(HAVE_PTHREAD_CONDATTR_SETCLOCK) && defined(HAVE_CLOCK_GETTIME) && defined(CLOCK_MONOTONIC)
|
||||
+// monotonic is supported statically. It doesn't mean it works on runtime.
|
||||
+#define CONDATTR_MONOTONIC
|
||||
+#endif
|
||||
+
|
||||
+// NULL when pthread_condattr_setclock(CLOCK_MONOTONIC) is not supported.
|
||||
+static pthread_condattr_t *condattr_monotonic = NULL;
|
||||
+
|
||||
+static void
|
||||
+init_condattr()
|
||||
+{
|
||||
+#ifdef CONDATTR_MONOTONIC
|
||||
+ static pthread_condattr_t ca;
|
||||
+ pthread_condattr_init(&ca);
|
||||
+ if (pthread_condattr_setclock(&ca, CLOCK_MONOTONIC) == 0) {
|
||||
+ condattr_monotonic = &ca; // Use monotonic clock
|
||||
+ }
|
||||
+#endif
|
||||
+}
|
||||
+
|
||||
+int
|
||||
+_PyThread_cond_init(PyCOND_T *cond)
|
||||
+{
|
||||
+ return pthread_cond_init(cond, condattr_monotonic);
|
||||
+}
|
||||
+
|
||||
+void
|
||||
+_PyThread_cond_after(long long us, struct timespec *abs)
|
||||
+{
|
||||
+#ifdef CONDATTR_MONOTONIC
|
||||
+ if (condattr_monotonic) {
|
||||
+ clock_gettime(CLOCK_MONOTONIC, abs);
|
||||
+ abs->tv_sec += us / 1000000;
|
||||
+ abs->tv_nsec += (us % 1000000) * 1000;
|
||||
+ abs->tv_sec += abs->tv_nsec / 1000000000;
|
||||
+ abs->tv_nsec %= 1000000000;
|
||||
+ return;
|
||||
+ }
|
||||
+#endif
|
||||
+
|
||||
+ struct timespec ts;
|
||||
+ MICROSECONDS_TO_TIMESPEC(us, ts);
|
||||
+ *abs = ts;
|
||||
+}
|
||||
+
|
||||
+
|
||||
/* A pthread mutex isn't sufficient to model the Python lock type
|
||||
* because, according to Draft 5 of the docs (P1003.4a/D5), both of the
|
||||
* following are undefined:
|
||||
@@ -175,6 +215,7 @@ PyThread__init_thread(void)
|
||||
extern void pthread_init(void);
|
||||
pthread_init();
|
||||
#endif
|
||||
+ init_condattr();
|
||||
}
|
||||
|
||||
#endif /* !_HAVE_BSDI */
|
||||
@@ -449,8 +490,7 @@ PyThread_allocate_lock(void)
|
||||
memset((void *)lock, '\0', sizeof(pthread_lock));
|
||||
lock->locked = 0;
|
||||
|
||||
- status = pthread_mutex_init(&lock->mut,
|
||||
- pthread_mutexattr_default);
|
||||
+ status = pthread_mutex_init(&lock->mut, NULL);
|
||||
CHECK_STATUS_PTHREAD("pthread_mutex_init");
|
||||
/* Mark the pthread mutex underlying a Python mutex as
|
||||
pure happens-before. We can't simply mark the
|
||||
@@ -459,8 +499,7 @@ PyThread_allocate_lock(void)
|
||||
will cause errors. */
|
||||
_Py_ANNOTATE_PURE_HAPPENS_BEFORE_MUTEX(&lock->mut);
|
||||
|
||||
- status = pthread_cond_init(&lock->lock_released,
|
||||
- pthread_condattr_default);
|
||||
+ status = _PyThread_cond_init(&lock->lock_released);
|
||||
CHECK_STATUS_PTHREAD("pthread_cond_init");
|
||||
|
||||
if (error) {
|
||||
@@ -519,9 +558,10 @@ PyThread_acquire_lock_timed(PyThread_type_lock lock, PY_TIMEOUT_T microseconds,
|
||||
success = PY_LOCK_ACQUIRED;
|
||||
}
|
||||
else if (microseconds != 0) {
|
||||
- struct timespec ts;
|
||||
- if (microseconds > 0)
|
||||
- MICROSECONDS_TO_TIMESPEC(microseconds, ts);
|
||||
+ struct timespec abs;
|
||||
+ if (microseconds > 0) {
|
||||
+ _PyThread_cond_after(microseconds, &abs);
|
||||
+ }
|
||||
/* continue trying until we get the lock */
|
||||
|
||||
/* mut must be locked by me -- part of the condition
|
||||
@@ -530,10 +570,13 @@ PyThread_acquire_lock_timed(PyThread_type_lock lock, PY_TIMEOUT_T microseconds,
|
||||
if (microseconds > 0) {
|
||||
status = pthread_cond_timedwait(
|
||||
&thelock->lock_released,
|
||||
- &thelock->mut, &ts);
|
||||
+ &thelock->mut, &abs);
|
||||
+ if (status == 1) {
|
||||
+ break;
|
||||
+ }
|
||||
if (status == ETIMEDOUT)
|
||||
break;
|
||||
- CHECK_STATUS_PTHREAD("pthread_cond_timed_wait");
|
||||
+ CHECK_STATUS_PTHREAD("pthread_cond_timedwait");
|
||||
}
|
||||
else {
|
||||
status = pthread_cond_wait(
|
||||
diff --git a/configure.ac b/configure.ac
|
||||
index a0e3613..8a17559 100644
|
||||
--- a/configure.ac
|
||||
+++ b/configure.ac
|
||||
@@ -3582,7 +3582,7 @@ AC_CHECK_FUNCS(alarm accept4 setitimer getitimer bind_textdomain_codeset chown \
|
||||
memrchr mbrtowc mkdirat mkfifo \
|
||||
mkfifoat mknod mknodat mktime mremap nice openat pathconf pause pipe2 plock poll \
|
||||
posix_fallocate posix_fadvise pread \
|
||||
- pthread_init pthread_kill putenv pwrite readlink readlinkat readv realpath renameat \
|
||||
+ pthread_condattr_setclock pthread_init pthread_kill putenv pwrite readlink readlinkat readv realpath renameat \
|
||||
select sem_open sem_timedwait sem_getvalue sem_unlink sendfile setegid seteuid \
|
||||
setgid sethostname \
|
||||
setlocale setregid setreuid setresuid setresgid setsid setpgid setpgrp setpriority setuid setvbuf \
|
|
@ -0,0 +1,80 @@
|
|||
diff --git a/Lib/ftplib.py b/Lib/ftplib.py
|
||||
index 2ff251a..385e432 100644
|
||||
--- a/Lib/ftplib.py
|
||||
+++ b/Lib/ftplib.py
|
||||
@@ -104,6 +104,8 @@ class FTP:
|
||||
welcome = None
|
||||
passiveserver = 1
|
||||
encoding = "latin-1"
|
||||
+ # Disables https://bugs.python.org/issue43285 security if set to True.
|
||||
+ trust_server_pasv_ipv4_address = False
|
||||
|
||||
# Initialization method (called by class instantiation).
|
||||
# Initialize host to localhost, port to standard ftp port
|
||||
@@ -333,8 +335,13 @@ class FTP:
|
||||
return sock
|
||||
|
||||
def makepasv(self):
|
||||
+ """Internal: Does the PASV or EPSV handshake -> (address, port)"""
|
||||
if self.af == socket.AF_INET:
|
||||
- host, port = parse227(self.sendcmd('PASV'))
|
||||
+ untrusted_host, port = parse227(self.sendcmd('PASV'))
|
||||
+ if self.trust_server_pasv_ipv4_address:
|
||||
+ host = untrusted_host
|
||||
+ else:
|
||||
+ host = self.sock.getpeername()[0]
|
||||
else:
|
||||
host, port = parse229(self.sendcmd('EPSV'), self.sock.getpeername())
|
||||
return host, port
|
||||
diff --git a/Lib/test/test_ftplib.py b/Lib/test/test_ftplib.py
|
||||
index 4ff2f71..3ca7cc1 100644
|
||||
--- a/Lib/test/test_ftplib.py
|
||||
+++ b/Lib/test/test_ftplib.py
|
||||
@@ -94,6 +94,10 @@ class DummyFTPHandler(asynchat.async_chat):
|
||||
self.rest = None
|
||||
self.next_retr_data = RETR_DATA
|
||||
self.push('220 welcome')
|
||||
+ # We use this as the string IPv4 address to direct the client
|
||||
+ # to in response to a PASV command. To test security behavior.
|
||||
+ # https://bugs.python.org/issue43285/.
|
||||
+ self.fake_pasv_server_ip = '252.253.254.255'
|
||||
|
||||
def collect_incoming_data(self, data):
|
||||
self.in_buffer.append(data)
|
||||
@@ -136,7 +140,8 @@ class DummyFTPHandler(asynchat.async_chat):
|
||||
sock.bind((self.socket.getsockname()[0], 0))
|
||||
sock.listen()
|
||||
sock.settimeout(TIMEOUT)
|
||||
- ip, port = sock.getsockname()[:2]
|
||||
+ port = sock.getsockname()[1]
|
||||
+ ip = self.fake_pasv_server_ip
|
||||
ip = ip.replace('.', ','); p1 = port / 256; p2 = port % 256
|
||||
self.push('227 entering passive mode (%s,%d,%d)' %(ip, p1, p2))
|
||||
conn, addr = sock.accept()
|
||||
@@ -694,6 +699,26 @@ class TestFTPClass(TestCase):
|
||||
# IPv4 is in use, just make sure send_epsv has not been used
|
||||
self.assertEqual(self.server.handler_instance.last_received_cmd, 'pasv')
|
||||
|
||||
+ def test_makepasv_issue43285_security_disabled(self):
|
||||
+ """Test the opt-in to the old vulnerable behavior."""
|
||||
+ self.client.trust_server_pasv_ipv4_address = True
|
||||
+ bad_host, port = self.client.makepasv()
|
||||
+ self.assertEqual(
|
||||
+ bad_host, self.server.handler_instance.fake_pasv_server_ip)
|
||||
+ # Opening and closing a connection keeps the dummy server happy
|
||||
+ # instead of timing out on accept.
|
||||
+ socket.create_connection((self.client.sock.getpeername()[0], port),
|
||||
+ timeout=TIMEOUT).close()
|
||||
+
|
||||
+ def test_makepasv_issue43285_security_enabled_default(self):
|
||||
+ self.assertFalse(self.client.trust_server_pasv_ipv4_address)
|
||||
+ trusted_host, port = self.client.makepasv()
|
||||
+ self.assertNotEqual(
|
||||
+ trusted_host, self.server.handler_instance.fake_pasv_server_ip)
|
||||
+ # Opening and closing a connection keeps the dummy server happy
|
||||
+ # instead of timing out on accept.
|
||||
+ socket.create_connection((trusted_host, port), timeout=TIMEOUT).close()
|
||||
+
|
||||
def test_with_statement(self):
|
||||
self.client.quit()
|
||||
|
|
@ -0,0 +1,170 @@
|
|||
From 6c472d3a1d334d4eeb4a25eba7bf3b01611bf667 Mon Sep 17 00:00:00 2001
|
||||
From: "Miss Islington (bot)"
|
||||
<31488909+miss-islington@users.noreply.github.com>
|
||||
Date: Thu, 6 May 2021 09:56:01 -0700
|
||||
Subject: [PATCH] [3.6] bpo-43882 - urllib.parse should sanitize urls
|
||||
containing ASCII newline and tabs (GH-25924)
|
||||
|
||||
Co-authored-by: Gregory P. Smith <greg@krypto.org>
|
||||
Co-authored-by: Serhiy Storchaka <storchaka@gmail.com>
|
||||
(cherry picked from commit 76cd81d60310d65d01f9d7b48a8985d8ab89c8b4)
|
||||
Co-authored-by: Senthil Kumaran <senthil@uthcode.com>
|
||||
(cherry picked from commit 515a7bc4e13645d0945b46a8e1d9102b918cd407)
|
||||
|
||||
Co-authored-by: Miss Islington (bot) <31488909+miss-islington@users.noreply.github.com>
|
||||
---
|
||||
Doc/library/urllib.parse.rst | 13 +++++
|
||||
Lib/test/test_urlparse.py | 48 +++++++++++++++++++
|
||||
Lib/urllib/parse.py | 10 ++++
|
||||
.../2021-04-25-07-46-37.bpo-43882.Jpwx85.rst | 6 +++
|
||||
4 files changed, 77 insertions(+)
|
||||
create mode 100644 Misc/NEWS.d/next/Security/2021-04-25-07-46-37.bpo-43882.Jpwx85.rst
|
||||
|
||||
diff --git a/Doc/library/urllib.parse.rst b/Doc/library/urllib.parse.rst
|
||||
index 3c2e37ef2093a..b717d7cc05b2e 100644
|
||||
--- a/Doc/library/urllib.parse.rst
|
||||
+++ b/Doc/library/urllib.parse.rst
|
||||
@@ -288,6 +288,9 @@ or on combining URL components into a URL string.
|
||||
``#``, ``@``, or ``:`` will raise a :exc:`ValueError`. If the URL is
|
||||
decomposed before parsing, no error will be raised.
|
||||
|
||||
+ Following the `WHATWG spec`_ that updates RFC 3986, ASCII newline
|
||||
+ ``\n``, ``\r`` and tab ``\t`` characters are stripped from the URL.
|
||||
+
|
||||
.. versionchanged:: 3.6
|
||||
Out-of-range port numbers now raise :exc:`ValueError`, instead of
|
||||
returning :const:`None`.
|
||||
@@ -296,6 +299,10 @@ or on combining URL components into a URL string.
|
||||
Characters that affect netloc parsing under NFKC normalization will
|
||||
now raise :exc:`ValueError`.
|
||||
|
||||
+ .. versionchanged:: 3.6.14
|
||||
+ ASCII newline and tab characters are stripped from the URL.
|
||||
+
|
||||
+.. _WHATWG spec: https://url.spec.whatwg.org/#concept-basic-url-parser
|
||||
|
||||
.. function:: urlunsplit(parts)
|
||||
|
||||
@@ -633,6 +640,10 @@ task isn't already covered by the URL parsing functions above.
|
||||
|
||||
.. seealso::
|
||||
|
||||
+ `WHATWG`_ - URL Living standard
|
||||
+ Working Group for the URL Standard that defines URLs, domains, IP addresses, the
|
||||
+ application/x-www-form-urlencoded format, and their API.
|
||||
+
|
||||
:rfc:`3986` - Uniform Resource Identifiers
|
||||
This is the current standard (STD66). Any changes to urllib.parse module
|
||||
should conform to this. Certain deviations could be observed, which are
|
||||
@@ -656,3 +667,5 @@ task isn't already covered by the URL parsing functions above.
|
||||
|
||||
:rfc:`1738` - Uniform Resource Locators (URL)
|
||||
This specifies the formal syntax and semantics of absolute URLs.
|
||||
+
|
||||
+.. _WHATWG: https://url.spec.whatwg.org/
|
||||
diff --git a/Lib/test/test_urlparse.py b/Lib/test/test_urlparse.py
|
||||
index e3088b2f39bd7..3509278a01694 100644
|
||||
--- a/Lib/test/test_urlparse.py
|
||||
+++ b/Lib/test/test_urlparse.py
|
||||
@@ -612,6 +612,54 @@ def test_urlsplit_attributes(self):
|
||||
with self.assertRaisesRegex(ValueError, "out of range"):
|
||||
p.port
|
||||
|
||||
+ def test_urlsplit_remove_unsafe_bytes(self):
|
||||
+ # Remove ASCII tabs and newlines from input, for http common case scenario.
|
||||
+ url = "h\nttp://www.python\n.org\t/java\nscript:\talert('msg\r\n')/?query\n=\tsomething#frag\nment"
|
||||
+ p = urllib.parse.urlsplit(url)
|
||||
+ self.assertEqual(p.scheme, "http")
|
||||
+ self.assertEqual(p.netloc, "www.python.org")
|
||||
+ self.assertEqual(p.path, "/javascript:alert('msg')/")
|
||||
+ self.assertEqual(p.query, "query=something")
|
||||
+ self.assertEqual(p.fragment, "fragment")
|
||||
+ self.assertEqual(p.username, None)
|
||||
+ self.assertEqual(p.password, None)
|
||||
+ self.assertEqual(p.hostname, "www.python.org")
|
||||
+ self.assertEqual(p.port, None)
|
||||
+ self.assertEqual(p.geturl(), "http://www.python.org/javascript:alert('msg')/?query=something#fragment")
|
||||
+
|
||||
+ # Remove ASCII tabs and newlines from input as bytes, for http common case scenario.
|
||||
+ url = b"h\nttp://www.python\n.org\t/java\nscript:\talert('msg\r\n')/?query\n=\tsomething#frag\nment"
|
||||
+ p = urllib.parse.urlsplit(url)
|
||||
+ self.assertEqual(p.scheme, b"http")
|
||||
+ self.assertEqual(p.netloc, b"www.python.org")
|
||||
+ self.assertEqual(p.path, b"/javascript:alert('msg')/")
|
||||
+ self.assertEqual(p.query, b"query=something")
|
||||
+ self.assertEqual(p.fragment, b"fragment")
|
||||
+ self.assertEqual(p.username, None)
|
||||
+ self.assertEqual(p.password, None)
|
||||
+ self.assertEqual(p.hostname, b"www.python.org")
|
||||
+ self.assertEqual(p.port, None)
|
||||
+ self.assertEqual(p.geturl(), b"http://www.python.org/javascript:alert('msg')/?query=something#fragment")
|
||||
+
|
||||
+ # any scheme
|
||||
+ url = "x-new-scheme\t://www.python\n.org\t/java\nscript:\talert('msg\r\n')/?query\n=\tsomething#frag\nment"
|
||||
+ p = urllib.parse.urlsplit(url)
|
||||
+ self.assertEqual(p.geturl(), "x-new-scheme://www.python.org/javascript:alert('msg')/?query=something#fragment")
|
||||
+
|
||||
+ # Remove ASCII tabs and newlines from input as bytes, any scheme.
|
||||
+ url = b"x-new-scheme\t://www.python\n.org\t/java\nscript:\talert('msg\r\n')/?query\n=\tsomething#frag\nment"
|
||||
+ p = urllib.parse.urlsplit(url)
|
||||
+ self.assertEqual(p.geturl(), b"x-new-scheme://www.python.org/javascript:alert('msg')/?query=something#fragment")
|
||||
+
|
||||
+ # Unsafe bytes is not returned from urlparse cache.
|
||||
+ # scheme is stored after parsing, sending an scheme with unsafe bytes *will not* return an unsafe scheme
|
||||
+ url = "https://www.python\n.org\t/java\nscript:\talert('msg\r\n')/?query\n=\tsomething#frag\nment"
|
||||
+ scheme = "htt\nps"
|
||||
+ for _ in range(2):
|
||||
+ p = urllib.parse.urlsplit(url, scheme=scheme)
|
||||
+ self.assertEqual(p.scheme, "https")
|
||||
+ self.assertEqual(p.geturl(), "https://www.python.org/javascript:alert('msg')/?query=something#fragment")
|
||||
+
|
||||
def test_attributes_bad_port(self):
|
||||
"""Check handling of invalid ports."""
|
||||
for bytes in (False, True):
|
||||
diff --git a/Lib/urllib/parse.py b/Lib/urllib/parse.py
|
||||
index 66056bf589bf6..ac6e7a9cee0b9 100644
|
||||
--- a/Lib/urllib/parse.py
|
||||
+++ b/Lib/urllib/parse.py
|
||||
@@ -76,6 +76,9 @@
|
||||
'0123456789'
|
||||
'+-.')
|
||||
|
||||
+# Unsafe bytes to be removed per WHATWG spec
|
||||
+_UNSAFE_URL_BYTES_TO_REMOVE = ['\t', '\r', '\n']
|
||||
+
|
||||
# XXX: Consider replacing with functools.lru_cache
|
||||
MAX_CACHE_SIZE = 20
|
||||
_parse_cache = {}
|
||||
@@ -409,6 +412,11 @@ def _checknetloc(netloc):
|
||||
raise ValueError("netloc '" + netloc + "' contains invalid " +
|
||||
"characters under NFKC normalization")
|
||||
|
||||
+def _remove_unsafe_bytes_from_url(url):
|
||||
+ for b in _UNSAFE_URL_BYTES_TO_REMOVE:
|
||||
+ url = url.replace(b, "")
|
||||
+ return url
|
||||
+
|
||||
def urlsplit(url, scheme='', allow_fragments=True):
|
||||
"""Parse a URL into 5 components:
|
||||
<scheme>://<netloc>/<path>?<query>#<fragment>
|
||||
@@ -416,6 +424,8 @@ def urlsplit(url, scheme='', allow_fragments=True):
|
||||
Note that we don't break the components up in smaller bits
|
||||
(e.g. netloc is a single string) and we don't expand % escapes."""
|
||||
url, scheme, _coerce_result = _coerce_args(url, scheme)
|
||||
+ url = _remove_unsafe_bytes_from_url(url)
|
||||
+ scheme = _remove_unsafe_bytes_from_url(scheme)
|
||||
allow_fragments = bool(allow_fragments)
|
||||
key = url, scheme, allow_fragments, type(url), type(scheme)
|
||||
cached = _parse_cache.get(key, None)
|
||||
diff --git a/Misc/NEWS.d/next/Security/2021-04-25-07-46-37.bpo-43882.Jpwx85.rst b/Misc/NEWS.d/next/Security/2021-04-25-07-46-37.bpo-43882.Jpwx85.rst
|
||||
new file mode 100644
|
||||
index 0000000000000..a326d079dff4a
|
||||
--- /dev/null
|
||||
+++ b/Misc/NEWS.d/next/Security/2021-04-25-07-46-37.bpo-43882.Jpwx85.rst
|
||||
@@ -0,0 +1,6 @@
|
||||
+The presence of newline or tab characters in parts of a URL could allow
|
||||
+some forms of attacks.
|
||||
+
|
||||
+Following the controlling specification for URLs defined by WHATWG
|
||||
+:func:`urllib.parse` now removes ASCII newlines and tabs from URLs,
|
||||
+preventing such attacks.
|
|
@ -0,0 +1,98 @@
|
|||
From a5b78c6f1c802f6023bd4d7a248dc83be1eef6a3 Mon Sep 17 00:00:00 2001
|
||||
From: Sebastian Pipping <sebastian@pipping.org>
|
||||
Date: Mon, 21 Feb 2022 15:48:32 +0100
|
||||
Subject: [PATCH] 00378: Support expat 2.4.5
|
||||
|
||||
Curly brackets were never allowed in namespace URIs
|
||||
according to RFC 3986, and so-called namespace-validating
|
||||
XML parsers have the right to reject them a invalid URIs.
|
||||
|
||||
libexpat >=2.4.5 has become strcter in that regard due to
|
||||
related security issues; with ET.XML instantiating a
|
||||
namespace-aware parser under the hood, this test has no
|
||||
future in CPython.
|
||||
|
||||
References:
|
||||
- https://datatracker.ietf.org/doc/html/rfc3968
|
||||
- https://www.w3.org/TR/xml-names/
|
||||
|
||||
Also, test_minidom.py: Support Expat >=2.4.5
|
||||
|
||||
Upstream: https://bugs.python.org/issue46811
|
||||
|
||||
Co-authored-by: Sebastian Pipping <sebastian@pipping.org>
|
||||
---
|
||||
Lib/test/test_minidom.py | 12 +++++++++---
|
||||
Lib/test/test_xml_etree.py | 6 ------
|
||||
.../Library/2022-02-20-21-03-31.bpo-46811.8BxgdQ.rst | 1 +
|
||||
3 files changed, 10 insertions(+), 9 deletions(-)
|
||||
create mode 100644 Misc/NEWS.d/next/Library/2022-02-20-21-03-31.bpo-46811.8BxgdQ.rst
|
||||
|
||||
diff --git a/Lib/test/test_minidom.py b/Lib/test/test_minidom.py
|
||||
index d55e25e..e947382 100644
|
||||
--- a/Lib/test/test_minidom.py
|
||||
+++ b/Lib/test/test_minidom.py
|
||||
@@ -5,10 +5,12 @@ import pickle
|
||||
from test import support
|
||||
import unittest
|
||||
|
||||
+import pyexpat
|
||||
import xml.dom.minidom
|
||||
|
||||
from xml.dom.minidom import parse, Node, Document, parseString
|
||||
from xml.dom.minidom import getDOMImplementation
|
||||
+from xml.parsers.expat import ExpatError
|
||||
|
||||
|
||||
tstfile = support.findfile("test.xml", subdir="xmltestdata")
|
||||
@@ -1156,8 +1158,10 @@ class MinidomTest(unittest.TestCase):
|
||||
|
||||
# Verify that character decoding errors raise exceptions instead
|
||||
# of crashing
|
||||
- self.assertRaises(UnicodeDecodeError, parseString,
|
||||
- b'<fran\xe7ais>Comment \xe7a va ? Tr\xe8s bien ?</fran\xe7ais>')
|
||||
+ self.assertRaises(ExpatError, parseString,
|
||||
+ b'<fran\xe7ais></fran\xe7ais>')
|
||||
+ self.assertRaises(ExpatError, parseString,
|
||||
+ b'<franais>Comment \xe7a va ? Tr\xe8s bien ?</franais>')
|
||||
|
||||
doc.unlink()
|
||||
|
||||
@@ -1602,7 +1606,9 @@ class MinidomTest(unittest.TestCase):
|
||||
self.confirm(doc2.namespaceURI == xml.dom.EMPTY_NAMESPACE)
|
||||
|
||||
def testExceptionOnSpacesInXMLNSValue(self):
|
||||
- with self.assertRaisesRegex(ValueError, 'Unsupported syntax'):
|
||||
+ context = self.assertRaisesRegex(ExpatError, 'syntax error')
|
||||
+
|
||||
+ with context:
|
||||
parseString('<element xmlns:abc="http:abc.com/de f g/hi/j k"><abc:foo /></element>')
|
||||
|
||||
def testDocRemoveChild(self):
|
||||
diff --git a/Lib/test/test_xml_etree.py b/Lib/test/test_xml_etree.py
|
||||
index b01709e..acaa519 100644
|
||||
--- a/Lib/test/test_xml_etree.py
|
||||
+++ b/Lib/test/test_xml_etree.py
|
||||
@@ -1668,12 +1668,6 @@ class BugsTest(unittest.TestCase):
|
||||
b"<?xml version='1.0' encoding='ascii'?>\n"
|
||||
b'<body>tãg</body>')
|
||||
|
||||
- def test_issue3151(self):
|
||||
- e = ET.XML('<prefix:localname xmlns:prefix="${stuff}"/>')
|
||||
- self.assertEqual(e.tag, '{${stuff}}localname')
|
||||
- t = ET.ElementTree(e)
|
||||
- self.assertEqual(ET.tostring(e), b'<ns0:localname xmlns:ns0="${stuff}" />')
|
||||
-
|
||||
def test_issue6565(self):
|
||||
elem = ET.XML("<body><tag/></body>")
|
||||
self.assertEqual(summarize_list(elem), ['tag'])
|
||||
diff --git a/Misc/NEWS.d/next/Library/2022-02-20-21-03-31.bpo-46811.8BxgdQ.rst b/Misc/NEWS.d/next/Library/2022-02-20-21-03-31.bpo-46811.8BxgdQ.rst
|
||||
new file mode 100644
|
||||
index 0000000..6969bd1
|
||||
--- /dev/null
|
||||
+++ b/Misc/NEWS.d/next/Library/2022-02-20-21-03-31.bpo-46811.8BxgdQ.rst
|
||||
@@ -0,0 +1 @@
|
||||
+Make test suite support Expat >=2.4.5
|
||||
--
|
||||
2.35.1
|
||||
|
|
@ -0,0 +1,150 @@
|
|||
From 0000000000000000000000000000000000000000 Mon Sep 17 00:00:00 2001
|
||||
From: Petr Viktorin <encukou@gmail.com>
|
||||
Date: Fri, 3 Jun 2022 11:43:35 +0200
|
||||
Subject: [PATCH] 00382: CVE-2015-20107
|
||||
|
||||
Make mailcap refuse to match unsafe filenames/types/params (GH-91993)
|
||||
|
||||
Upstream: https://github.com/python/cpython/issues/68966
|
||||
|
||||
Tracker bug: https://bugzilla.redhat.com/show_bug.cgi?id=2075390
|
||||
---
|
||||
Doc/library/mailcap.rst | 12 +++++++++
|
||||
Lib/mailcap.py | 26 +++++++++++++++++--
|
||||
Lib/test/test_mailcap.py | 8 ++++--
|
||||
...2-04-27-18-25-30.gh-issue-68966.gjS8zs.rst | 4 +++
|
||||
4 files changed, 46 insertions(+), 4 deletions(-)
|
||||
create mode 100644 Misc/NEWS.d/next/Security/2022-04-27-18-25-30.gh-issue-68966.gjS8zs.rst
|
||||
|
||||
diff --git a/Doc/library/mailcap.rst b/Doc/library/mailcap.rst
|
||||
index 896afd1d73..849d0bc05f 100644
|
||||
--- a/Doc/library/mailcap.rst
|
||||
+++ b/Doc/library/mailcap.rst
|
||||
@@ -54,6 +54,18 @@ standard. However, mailcap files are supported on most Unix systems.
|
||||
use) to determine whether or not the mailcap line applies. :func:`findmatch`
|
||||
will automatically check such conditions and skip the entry if the check fails.
|
||||
|
||||
+ .. versionchanged:: 3.11
|
||||
+
|
||||
+ To prevent security issues with shell metacharacters (symbols that have
|
||||
+ special effects in a shell command line), ``findmatch`` will refuse
|
||||
+ to inject ASCII characters other than alphanumerics and ``@+=:,./-_``
|
||||
+ into the returned command line.
|
||||
+
|
||||
+ If a disallowed character appears in *filename*, ``findmatch`` will always
|
||||
+ return ``(None, None)`` as if no entry was found.
|
||||
+ If such a character appears elsewhere (a value in *plist* or in *MIMEtype*),
|
||||
+ ``findmatch`` will ignore all mailcap entries which use that value.
|
||||
+ A :mod:`warning <warnings>` will be raised in either case.
|
||||
|
||||
.. function:: getcaps()
|
||||
|
||||
diff --git a/Lib/mailcap.py b/Lib/mailcap.py
|
||||
index bd0fc0981c..dcd4b449e8 100644
|
||||
--- a/Lib/mailcap.py
|
||||
+++ b/Lib/mailcap.py
|
||||
@@ -2,6 +2,7 @@
|
||||
|
||||
import os
|
||||
import warnings
|
||||
+import re
|
||||
|
||||
__all__ = ["getcaps","findmatch"]
|
||||
|
||||
@@ -13,6 +14,11 @@ def lineno_sort_key(entry):
|
||||
else:
|
||||
return 1, 0
|
||||
|
||||
+_find_unsafe = re.compile(r'[^\xa1-\U0010FFFF\w@+=:,./-]').search
|
||||
+
|
||||
+class UnsafeMailcapInput(Warning):
|
||||
+ """Warning raised when refusing unsafe input"""
|
||||
+
|
||||
|
||||
# Part 1: top-level interface.
|
||||
|
||||
@@ -165,15 +171,22 @@ def findmatch(caps, MIMEtype, key='view', filename="/dev/null", plist=[]):
|
||||
entry to use.
|
||||
|
||||
"""
|
||||
+ if _find_unsafe(filename):
|
||||
+ msg = "Refusing to use mailcap with filename %r. Use a safe temporary filename." % (filename,)
|
||||
+ warnings.warn(msg, UnsafeMailcapInput)
|
||||
+ return None, None
|
||||
entries = lookup(caps, MIMEtype, key)
|
||||
# XXX This code should somehow check for the needsterminal flag.
|
||||
for e in entries:
|
||||
if 'test' in e:
|
||||
test = subst(e['test'], filename, plist)
|
||||
+ if test is None:
|
||||
+ continue
|
||||
if test and os.system(test) != 0:
|
||||
continue
|
||||
command = subst(e[key], MIMEtype, filename, plist)
|
||||
- return command, e
|
||||
+ if command is not None:
|
||||
+ return command, e
|
||||
return None, None
|
||||
|
||||
def lookup(caps, MIMEtype, key=None):
|
||||
@@ -206,6 +219,10 @@ def subst(field, MIMEtype, filename, plist=[]):
|
||||
elif c == 's':
|
||||
res = res + filename
|
||||
elif c == 't':
|
||||
+ if _find_unsafe(MIMEtype):
|
||||
+ msg = "Refusing to substitute MIME type %r into a shell command." % (MIMEtype,)
|
||||
+ warnings.warn(msg, UnsafeMailcapInput)
|
||||
+ return None
|
||||
res = res + MIMEtype
|
||||
elif c == '{':
|
||||
start = i
|
||||
@@ -213,7 +230,12 @@ def subst(field, MIMEtype, filename, plist=[]):
|
||||
i = i+1
|
||||
name = field[start:i]
|
||||
i = i+1
|
||||
- res = res + findparam(name, plist)
|
||||
+ param = findparam(name, plist)
|
||||
+ if _find_unsafe(param):
|
||||
+ msg = "Refusing to substitute parameter %r (%s) into a shell command" % (param, name)
|
||||
+ warnings.warn(msg, UnsafeMailcapInput)
|
||||
+ return None
|
||||
+ res = res + param
|
||||
# XXX To do:
|
||||
# %n == number of parts if type is multipart/*
|
||||
# %F == list of alternating type and filename for parts
|
||||
diff --git a/Lib/test/test_mailcap.py b/Lib/test/test_mailcap.py
|
||||
index c08423c670..920283d9a2 100644
|
||||
--- a/Lib/test/test_mailcap.py
|
||||
+++ b/Lib/test/test_mailcap.py
|
||||
@@ -121,7 +121,8 @@ class HelperFunctionTest(unittest.TestCase):
|
||||
(["", "audio/*", "foo.txt"], ""),
|
||||
(["echo foo", "audio/*", "foo.txt"], "echo foo"),
|
||||
(["echo %s", "audio/*", "foo.txt"], "echo foo.txt"),
|
||||
- (["echo %t", "audio/*", "foo.txt"], "echo audio/*"),
|
||||
+ (["echo %t", "audio/*", "foo.txt"], None),
|
||||
+ (["echo %t", "audio/wav", "foo.txt"], "echo audio/wav"),
|
||||
(["echo \\%t", "audio/*", "foo.txt"], "echo %t"),
|
||||
(["echo foo", "audio/*", "foo.txt", plist], "echo foo"),
|
||||
(["echo %{total}", "audio/*", "foo.txt", plist], "echo 3")
|
||||
@@ -205,7 +206,10 @@ class FindmatchTest(unittest.TestCase):
|
||||
('"An audio fragment"', audio_basic_entry)),
|
||||
([c, "audio/*"],
|
||||
{"filename": fname},
|
||||
- ("/usr/local/bin/showaudio audio/*", audio_entry)),
|
||||
+ (None, None)),
|
||||
+ ([c, "audio/wav"],
|
||||
+ {"filename": fname},
|
||||
+ ("/usr/local/bin/showaudio audio/wav", audio_entry)),
|
||||
([c, "message/external-body"],
|
||||
{"plist": plist},
|
||||
("showexternal /dev/null default john python.org /tmp foo bar", message_entry))
|
||||
diff --git a/Misc/NEWS.d/next/Security/2022-04-27-18-25-30.gh-issue-68966.gjS8zs.rst b/Misc/NEWS.d/next/Security/2022-04-27-18-25-30.gh-issue-68966.gjS8zs.rst
|
||||
new file mode 100644
|
||||
index 0000000000..da81a1f699
|
||||
--- /dev/null
|
||||
+++ b/Misc/NEWS.d/next/Security/2022-04-27-18-25-30.gh-issue-68966.gjS8zs.rst
|
||||
@@ -0,0 +1,4 @@
|
||||
+The deprecated mailcap module now refuses to inject unsafe text (filenames,
|
||||
+MIME types, parameters) into shell commands. Instead of using such text, it
|
||||
+will warn and act as if a match was not found (or for test commands, as if
|
||||
+the test failed).
|
|
@ -0,0 +1,130 @@
|
|||
From 0000000000000000000000000000000000000000 Mon Sep 17 00:00:00 2001
|
||||
From: "Miss Islington (bot)"
|
||||
<31488909+miss-islington@users.noreply.github.com>
|
||||
Date: Wed, 22 Jun 2022 15:05:00 -0700
|
||||
Subject: [PATCH] 00386: CVE-2021-28861
|
||||
|
||||
Fix an open redirection vulnerability in the `http.server` module when
|
||||
an URI path starts with `//` that could produce a 301 Location header
|
||||
with a misleading target. Vulnerability discovered, and logic fix
|
||||
proposed, by Hamza Avvan (@hamzaavvan).
|
||||
|
||||
Test and comments authored by Gregory P. Smith [Google].
|
||||
(cherry picked from commit 4abab6b603dd38bec1168e9a37c40a48ec89508e)
|
||||
|
||||
Upstream: https://github.com/python/cpython/pull/93879
|
||||
Tracking bugzilla: https://bugzilla.redhat.com/show_bug.cgi?id=2120642
|
||||
|
||||
Co-authored-by: Gregory P. Smith <greg@krypto.org>
|
||||
---
|
||||
Lib/http/server.py | 7 +++
|
||||
Lib/test/test_httpservers.py | 53 ++++++++++++++++++-
|
||||
...2-06-15-20-09-23.gh-issue-87389.QVaC3f.rst | 3 ++
|
||||
3 files changed, 61 insertions(+), 2 deletions(-)
|
||||
create mode 100644 Misc/NEWS.d/next/Security/2022-06-15-20-09-23.gh-issue-87389.QVaC3f.rst
|
||||
|
||||
diff --git a/Lib/http/server.py b/Lib/http/server.py
|
||||
index 60a4dadf03..ce05be13d3 100644
|
||||
--- a/Lib/http/server.py
|
||||
+++ b/Lib/http/server.py
|
||||
@@ -323,6 +323,13 @@ class BaseHTTPRequestHandler(socketserver.StreamRequestHandler):
|
||||
return False
|
||||
self.command, self.path, self.request_version = command, path, version
|
||||
|
||||
+ # gh-87389: The purpose of replacing '//' with '/' is to protect
|
||||
+ # against open redirect attacks possibly triggered if the path starts
|
||||
+ # with '//' because http clients treat //path as an absolute URI
|
||||
+ # without scheme (similar to http://path) rather than a path.
|
||||
+ if self.path.startswith('//'):
|
||||
+ self.path = '/' + self.path.lstrip('/') # Reduce to a single /
|
||||
+
|
||||
# Examine the headers and look for a Connection directive.
|
||||
try:
|
||||
self.headers = http.client.parse_headers(self.rfile,
|
||||
diff --git a/Lib/test/test_httpservers.py b/Lib/test/test_httpservers.py
|
||||
index 66e937e04b..5a0a7c3f74 100644
|
||||
--- a/Lib/test/test_httpservers.py
|
||||
+++ b/Lib/test/test_httpservers.py
|
||||
@@ -324,7 +324,7 @@ class SimpleHTTPServerTestCase(BaseTestCase):
|
||||
pass
|
||||
|
||||
def setUp(self):
|
||||
- BaseTestCase.setUp(self)
|
||||
+ super().setUp()
|
||||
self.cwd = os.getcwd()
|
||||
basetempdir = tempfile.gettempdir()
|
||||
os.chdir(basetempdir)
|
||||
@@ -343,7 +343,7 @@ class SimpleHTTPServerTestCase(BaseTestCase):
|
||||
except:
|
||||
pass
|
||||
finally:
|
||||
- BaseTestCase.tearDown(self)
|
||||
+ super().tearDown()
|
||||
|
||||
def check_status_and_reason(self, response, status, data=None):
|
||||
def close_conn():
|
||||
@@ -399,6 +399,55 @@ class SimpleHTTPServerTestCase(BaseTestCase):
|
||||
self.check_status_and_reason(response, HTTPStatus.OK,
|
||||
data=support.TESTFN_UNDECODABLE)
|
||||
|
||||
+ def test_get_dir_redirect_location_domain_injection_bug(self):
|
||||
+ """Ensure //evil.co/..%2f../../X does not put //evil.co/ in Location.
|
||||
+
|
||||
+ //netloc/ in a Location header is a redirect to a new host.
|
||||
+ https://github.com/python/cpython/issues/87389
|
||||
+
|
||||
+ This checks that a path resolving to a directory on our server cannot
|
||||
+ resolve into a redirect to another server.
|
||||
+ """
|
||||
+ os.mkdir(os.path.join(self.tempdir, 'existing_directory'))
|
||||
+ url = f'/python.org/..%2f..%2f..%2f..%2f..%2f../%0a%0d/../{self.tempdir_name}/existing_directory'
|
||||
+ expected_location = f'{url}/' # /python.org.../ single slash single prefix, trailing slash
|
||||
+ # Canonicalizes to /tmp/tempdir_name/existing_directory which does
|
||||
+ # exist and is a dir, triggering the 301 redirect logic.
|
||||
+ response = self.request(url)
|
||||
+ self.check_status_and_reason(response, HTTPStatus.MOVED_PERMANENTLY)
|
||||
+ location = response.getheader('Location')
|
||||
+ self.assertEqual(location, expected_location, msg='non-attack failed!')
|
||||
+
|
||||
+ # //python.org... multi-slash prefix, no trailing slash
|
||||
+ attack_url = f'/{url}'
|
||||
+ response = self.request(attack_url)
|
||||
+ self.check_status_and_reason(response, HTTPStatus.MOVED_PERMANENTLY)
|
||||
+ location = response.getheader('Location')
|
||||
+ self.assertFalse(location.startswith('//'), msg=location)
|
||||
+ self.assertEqual(location, expected_location,
|
||||
+ msg='Expected Location header to start with a single / and '
|
||||
+ 'end with a / as this is a directory redirect.')
|
||||
+
|
||||
+ # ///python.org... triple-slash prefix, no trailing slash
|
||||
+ attack3_url = f'//{url}'
|
||||
+ response = self.request(attack3_url)
|
||||
+ self.check_status_and_reason(response, HTTPStatus.MOVED_PERMANENTLY)
|
||||
+ self.assertEqual(response.getheader('Location'), expected_location)
|
||||
+
|
||||
+ # If the second word in the http request (Request-URI for the http
|
||||
+ # method) is a full URI, we don't worry about it, as that'll be parsed
|
||||
+ # and reassembled as a full URI within BaseHTTPRequestHandler.send_head
|
||||
+ # so no errant scheme-less //netloc//evil.co/ domain mixup can happen.
|
||||
+ attack_scheme_netloc_2slash_url = f'https://pypi.org/{url}'
|
||||
+ expected_scheme_netloc_location = f'{attack_scheme_netloc_2slash_url}/'
|
||||
+ response = self.request(attack_scheme_netloc_2slash_url)
|
||||
+ self.check_status_and_reason(response, HTTPStatus.MOVED_PERMANENTLY)
|
||||
+ location = response.getheader('Location')
|
||||
+ # We're just ensuring that the scheme and domain make it through, if
|
||||
+ # there are or aren't multiple slashes at the start of the path that
|
||||
+ # follows that isn't important in this Location: header.
|
||||
+ self.assertTrue(location.startswith('https://pypi.org/'), msg=location)
|
||||
+
|
||||
def test_get(self):
|
||||
#constructs the path relative to the root directory of the HTTPServer
|
||||
response = self.request(self.base_url + '/test')
|
||||
diff --git a/Misc/NEWS.d/next/Security/2022-06-15-20-09-23.gh-issue-87389.QVaC3f.rst b/Misc/NEWS.d/next/Security/2022-06-15-20-09-23.gh-issue-87389.QVaC3f.rst
|
||||
new file mode 100644
|
||||
index 0000000000..029d437190
|
||||
--- /dev/null
|
||||
+++ b/Misc/NEWS.d/next/Security/2022-06-15-20-09-23.gh-issue-87389.QVaC3f.rst
|
||||
@@ -0,0 +1,3 @@
|
||||
+:mod:`http.server`: Fix an open redirection vulnerability in the HTTP server
|
||||
+when an URI path starts with ``//``. Vulnerability discovered, and initial
|
||||
+fix proposed, by Hamza Avvan.
|
File diff suppressed because it is too large
Load Diff
|
@ -0,0 +1,95 @@
|
|||
From 0000000000000000000000000000000000000000 Mon Sep 17 00:00:00 2001
|
||||
From: "Miss Islington (bot)"
|
||||
<31488909+miss-islington@users.noreply.github.com>
|
||||
Date: Mon, 7 Nov 2022 19:22:14 -0800
|
||||
Subject: [PATCH] 00394: CVE-2022-45061: CPU denial of service via inefficient
|
||||
IDNA decoder
|
||||
|
||||
gh-98433: Fix quadratic time idna decoding.
|
||||
|
||||
There was an unnecessary quadratic loop in idna decoding. This restores
|
||||
the behavior to linear.
|
||||
|
||||
(cherry picked from commit a6f6c3a3d6f2b580f2d87885c9b8a9350ad7bf15)
|
||||
|
||||
Co-authored-by: Miss Islington (bot) <31488909+miss-islington@users.noreply.github.com>
|
||||
Co-authored-by: Gregory P. Smith <greg@krypto.org>
|
||||
---
|
||||
Lib/encodings/idna.py | 32 +++++++++----------
|
||||
Lib/test/test_codecs.py | 6 ++++
|
||||
...2-11-04-09-29-36.gh-issue-98433.l76c5G.rst | 6 ++++
|
||||
3 files changed, 27 insertions(+), 17 deletions(-)
|
||||
create mode 100644 Misc/NEWS.d/next/Security/2022-11-04-09-29-36.gh-issue-98433.l76c5G.rst
|
||||
|
||||
diff --git a/Lib/encodings/idna.py b/Lib/encodings/idna.py
|
||||
index ea4058512f..bf98f51336 100644
|
||||
--- a/Lib/encodings/idna.py
|
||||
+++ b/Lib/encodings/idna.py
|
||||
@@ -39,23 +39,21 @@ def nameprep(label):
|
||||
|
||||
# Check bidi
|
||||
RandAL = [stringprep.in_table_d1(x) for x in label]
|
||||
- for c in RandAL:
|
||||
- if c:
|
||||
- # There is a RandAL char in the string. Must perform further
|
||||
- # tests:
|
||||
- # 1) The characters in section 5.8 MUST be prohibited.
|
||||
- # This is table C.8, which was already checked
|
||||
- # 2) If a string contains any RandALCat character, the string
|
||||
- # MUST NOT contain any LCat character.
|
||||
- if any(stringprep.in_table_d2(x) for x in label):
|
||||
- raise UnicodeError("Violation of BIDI requirement 2")
|
||||
-
|
||||
- # 3) If a string contains any RandALCat character, a
|
||||
- # RandALCat character MUST be the first character of the
|
||||
- # string, and a RandALCat character MUST be the last
|
||||
- # character of the string.
|
||||
- if not RandAL[0] or not RandAL[-1]:
|
||||
- raise UnicodeError("Violation of BIDI requirement 3")
|
||||
+ if any(RandAL):
|
||||
+ # There is a RandAL char in the string. Must perform further
|
||||
+ # tests:
|
||||
+ # 1) The characters in section 5.8 MUST be prohibited.
|
||||
+ # This is table C.8, which was already checked
|
||||
+ # 2) If a string contains any RandALCat character, the string
|
||||
+ # MUST NOT contain any LCat character.
|
||||
+ if any(stringprep.in_table_d2(x) for x in label):
|
||||
+ raise UnicodeError("Violation of BIDI requirement 2")
|
||||
+ # 3) If a string contains any RandALCat character, a
|
||||
+ # RandALCat character MUST be the first character of the
|
||||
+ # string, and a RandALCat character MUST be the last
|
||||
+ # character of the string.
|
||||
+ if not RandAL[0] or not RandAL[-1]:
|
||||
+ raise UnicodeError("Violation of BIDI requirement 3")
|
||||
|
||||
return label
|
||||
|
||||
diff --git a/Lib/test/test_codecs.py b/Lib/test/test_codecs.py
|
||||
index 56485de3f6..a798d1f287 100644
|
||||
--- a/Lib/test/test_codecs.py
|
||||
+++ b/Lib/test/test_codecs.py
|
||||
@@ -1640,6 +1640,12 @@ class IDNACodecTest(unittest.TestCase):
|
||||
self.assertEqual("pyth\xf6n.org".encode("idna"), b"xn--pythn-mua.org")
|
||||
self.assertEqual("pyth\xf6n.org.".encode("idna"), b"xn--pythn-mua.org.")
|
||||
|
||||
+ def test_builtin_decode_length_limit(self):
|
||||
+ with self.assertRaisesRegex(UnicodeError, "too long"):
|
||||
+ (b"xn--016c"+b"a"*1100).decode("idna")
|
||||
+ with self.assertRaisesRegex(UnicodeError, "too long"):
|
||||
+ (b"xn--016c"+b"a"*70).decode("idna")
|
||||
+
|
||||
def test_stream(self):
|
||||
r = codecs.getreader("idna")(io.BytesIO(b"abc"))
|
||||
r.read(3)
|
||||
diff --git a/Misc/NEWS.d/next/Security/2022-11-04-09-29-36.gh-issue-98433.l76c5G.rst b/Misc/NEWS.d/next/Security/2022-11-04-09-29-36.gh-issue-98433.l76c5G.rst
|
||||
new file mode 100644
|
||||
index 0000000000..5185fac2e2
|
||||
--- /dev/null
|
||||
+++ b/Misc/NEWS.d/next/Security/2022-11-04-09-29-36.gh-issue-98433.l76c5G.rst
|
||||
@@ -0,0 +1,6 @@
|
||||
+The IDNA codec decoder used on DNS hostnames by :mod:`socket` or :mod:`asyncio`
|
||||
+related name resolution functions no longer involves a quadratic algorithm.
|
||||
+This prevents a potential CPU denial of service if an out-of-spec excessive
|
||||
+length hostname involving bidirectional characters were decoded. Some protocols
|
||||
+such as :mod:`urllib` http ``3xx`` redirects potentially allow for an attacker
|
||||
+to supply such a name.
|
File diff suppressed because it is too large
Load Diff
|
@ -0,0 +1,223 @@
|
|||
From 0000000000000000000000000000000000000000 Mon Sep 17 00:00:00 2001
|
||||
From: "Miss Islington (bot)"
|
||||
<31488909+miss-islington@users.noreply.github.com>
|
||||
Date: Mon, 22 May 2023 03:42:37 -0700
|
||||
Subject: [PATCH] 00399: CVE-2023-24329
|
||||
|
||||
gh-102153: Start stripping C0 control and space chars in `urlsplit` (GH-102508)
|
||||
|
||||
`urllib.parse.urlsplit` has already been respecting the WHATWG spec a bit GH-25595.
|
||||
|
||||
This adds more sanitizing to respect the "Remove any leading C0 control or space from input" [rule](https://url.spec.whatwg.org/GH-url-parsing:~:text=Remove%20any%20leading%20and%20trailing%20C0%20control%20or%20space%20from%20input.) in response to [CVE-2023-24329](https://nvd.nist.gov/vuln/detail/CVE-2023-24329).
|
||||
|
||||
Backported from Python 3.12
|
||||
|
||||
(cherry picked from commit f48a96a28012d28ae37a2f4587a780a5eb779946)
|
||||
|
||||
Co-authored-by: Illia Volochii <illia.volochii@gmail.com>
|
||||
Co-authored-by: Gregory P. Smith [Google] <greg@krypto.org>
|
||||
---
|
||||
Doc/library/urllib.parse.rst | 40 +++++++++++-
|
||||
Lib/test/test_urlparse.py | 61 ++++++++++++++++++-
|
||||
Lib/urllib/parse.py | 12 ++++
|
||||
...-03-07-20-59-17.gh-issue-102153.14CLSZ.rst | 3 +
|
||||
4 files changed, 113 insertions(+), 3 deletions(-)
|
||||
create mode 100644 Misc/NEWS.d/next/Security/2023-03-07-20-59-17.gh-issue-102153.14CLSZ.rst
|
||||
|
||||
diff --git a/Doc/library/urllib.parse.rst b/Doc/library/urllib.parse.rst
|
||||
index b717d7cc05..83a7a82089 100644
|
||||
--- a/Doc/library/urllib.parse.rst
|
||||
+++ b/Doc/library/urllib.parse.rst
|
||||
@@ -126,6 +126,12 @@ or on combining URL components into a URL string.
|
||||
``#``, ``@``, or ``:`` will raise a :exc:`ValueError`. If the URL is
|
||||
decomposed before parsing, no error will be raised.
|
||||
|
||||
+
|
||||
+ .. warning::
|
||||
+
|
||||
+ :func:`urlparse` does not perform validation. See :ref:`URL parsing
|
||||
+ security <url-parsing-security>` for details.
|
||||
+
|
||||
.. versionchanged:: 3.2
|
||||
Added IPv6 URL parsing capabilities.
|
||||
|
||||
@@ -288,8 +294,14 @@ or on combining URL components into a URL string.
|
||||
``#``, ``@``, or ``:`` will raise a :exc:`ValueError`. If the URL is
|
||||
decomposed before parsing, no error will be raised.
|
||||
|
||||
- Following the `WHATWG spec`_ that updates RFC 3986, ASCII newline
|
||||
- ``\n``, ``\r`` and tab ``\t`` characters are stripped from the URL.
|
||||
+ Following some of the `WHATWG spec`_ that updates RFC 3986, leading C0
|
||||
+ control and space characters are stripped from the URL. ``\n``,
|
||||
+ ``\r`` and tab ``\t`` characters are removed from the URL at any position.
|
||||
+
|
||||
+ .. warning::
|
||||
+
|
||||
+ :func:`urlsplit` does not perform validation. See :ref:`URL parsing
|
||||
+ security <url-parsing-security>` for details.
|
||||
|
||||
.. versionchanged:: 3.6
|
||||
Out-of-range port numbers now raise :exc:`ValueError`, instead of
|
||||
@@ -302,6 +314,9 @@ or on combining URL components into a URL string.
|
||||
.. versionchanged:: 3.6.14
|
||||
ASCII newline and tab characters are stripped from the URL.
|
||||
|
||||
+ .. versionchanged:: 3.6.15
|
||||
+ Leading WHATWG C0 control and space characters are stripped from the URL.
|
||||
+
|
||||
.. _WHATWG spec: https://url.spec.whatwg.org/#concept-basic-url-parser
|
||||
|
||||
.. function:: urlunsplit(parts)
|
||||
@@ -371,6 +386,27 @@ or on combining URL components into a URL string.
|
||||
.. versionchanged:: 3.2
|
||||
Result is a structured object rather than a simple 2-tuple.
|
||||
|
||||
+.. _url-parsing-security:
|
||||
+
|
||||
+URL parsing security
|
||||
+--------------------
|
||||
+
|
||||
+The :func:`urlsplit` and :func:`urlparse` APIs do not perform **validation** of
|
||||
+inputs. They may not raise errors on inputs that other applications consider
|
||||
+invalid. They may also succeed on some inputs that might not be considered
|
||||
+URLs elsewhere. Their purpose is for practical functionality rather than
|
||||
+purity.
|
||||
+
|
||||
+Instead of raising an exception on unusual input, they may instead return some
|
||||
+component parts as empty strings. Or components may contain more than perhaps
|
||||
+they should.
|
||||
+
|
||||
+We recommend that users of these APIs where the values may be used anywhere
|
||||
+with security implications code defensively. Do some verification within your
|
||||
+code before trusting a returned component part. Does that ``scheme`` make
|
||||
+sense? Is that a sensible ``path``? Is there anything strange about that
|
||||
+``hostname``? etc.
|
||||
+
|
||||
.. _parsing-ascii-encoded-bytes:
|
||||
|
||||
Parsing ASCII Encoded Bytes
|
||||
diff --git a/Lib/test/test_urlparse.py b/Lib/test/test_urlparse.py
|
||||
index 3509278a01..7fd61ffea9 100644
|
||||
--- a/Lib/test/test_urlparse.py
|
||||
+++ b/Lib/test/test_urlparse.py
|
||||
@@ -660,6 +660,65 @@ class UrlParseTestCase(unittest.TestCase):
|
||||
self.assertEqual(p.scheme, "https")
|
||||
self.assertEqual(p.geturl(), "https://www.python.org/javascript:alert('msg')/?query=something#fragment")
|
||||
|
||||
+ def test_urlsplit_strip_url(self):
|
||||
+ noise = bytes(range(0, 0x20 + 1))
|
||||
+ base_url = "http://User:Pass@www.python.org:080/doc/?query=yes#frag"
|
||||
+
|
||||
+ url = noise.decode("utf-8") + base_url
|
||||
+ p = urllib.parse.urlsplit(url)
|
||||
+ self.assertEqual(p.scheme, "http")
|
||||
+ self.assertEqual(p.netloc, "User:Pass@www.python.org:080")
|
||||
+ self.assertEqual(p.path, "/doc/")
|
||||
+ self.assertEqual(p.query, "query=yes")
|
||||
+ self.assertEqual(p.fragment, "frag")
|
||||
+ self.assertEqual(p.username, "User")
|
||||
+ self.assertEqual(p.password, "Pass")
|
||||
+ self.assertEqual(p.hostname, "www.python.org")
|
||||
+ self.assertEqual(p.port, 80)
|
||||
+ self.assertEqual(p.geturl(), base_url)
|
||||
+
|
||||
+ url = noise + base_url.encode("utf-8")
|
||||
+ p = urllib.parse.urlsplit(url)
|
||||
+ self.assertEqual(p.scheme, b"http")
|
||||
+ self.assertEqual(p.netloc, b"User:Pass@www.python.org:080")
|
||||
+ self.assertEqual(p.path, b"/doc/")
|
||||
+ self.assertEqual(p.query, b"query=yes")
|
||||
+ self.assertEqual(p.fragment, b"frag")
|
||||
+ self.assertEqual(p.username, b"User")
|
||||
+ self.assertEqual(p.password, b"Pass")
|
||||
+ self.assertEqual(p.hostname, b"www.python.org")
|
||||
+ self.assertEqual(p.port, 80)
|
||||
+ self.assertEqual(p.geturl(), base_url.encode("utf-8"))
|
||||
+
|
||||
+ # Test that trailing space is preserved as some applications rely on
|
||||
+ # this within query strings.
|
||||
+ query_spaces_url = "https://www.python.org:88/doc/?query= "
|
||||
+ p = urllib.parse.urlsplit(noise.decode("utf-8") + query_spaces_url)
|
||||
+ self.assertEqual(p.scheme, "https")
|
||||
+ self.assertEqual(p.netloc, "www.python.org:88")
|
||||
+ self.assertEqual(p.path, "/doc/")
|
||||
+ self.assertEqual(p.query, "query= ")
|
||||
+ self.assertEqual(p.port, 88)
|
||||
+ self.assertEqual(p.geturl(), query_spaces_url)
|
||||
+
|
||||
+ p = urllib.parse.urlsplit("www.pypi.org ")
|
||||
+ # That "hostname" gets considered a "path" due to the
|
||||
+ # trailing space and our existing logic... YUCK...
|
||||
+ # and re-assembles via geturl aka unurlsplit into the original.
|
||||
+ # django.core.validators.URLValidator (at least through v3.2) relies on
|
||||
+ # this, for better or worse, to catch it in a ValidationError via its
|
||||
+ # regular expressions.
|
||||
+ # Here we test the basic round trip concept of such a trailing space.
|
||||
+ self.assertEqual(urllib.parse.urlunsplit(p), "www.pypi.org ")
|
||||
+
|
||||
+ # with scheme as cache-key
|
||||
+ url = "//www.python.org/"
|
||||
+ scheme = noise.decode("utf-8") + "https" + noise.decode("utf-8")
|
||||
+ for _ in range(2):
|
||||
+ p = urllib.parse.urlsplit(url, scheme=scheme)
|
||||
+ self.assertEqual(p.scheme, "https")
|
||||
+ self.assertEqual(p.geturl(), "https://www.python.org/")
|
||||
+
|
||||
def test_attributes_bad_port(self):
|
||||
"""Check handling of invalid ports."""
|
||||
for bytes in (False, True):
|
||||
@@ -667,7 +726,7 @@ class UrlParseTestCase(unittest.TestCase):
|
||||
for port in ("foo", "1.5", "-1", "0x10"):
|
||||
with self.subTest(bytes=bytes, parse=parse, port=port):
|
||||
netloc = "www.example.net:" + port
|
||||
- url = "http://" + netloc
|
||||
+ url = "http://" + netloc + "/"
|
||||
if bytes:
|
||||
netloc = netloc.encode("ascii")
|
||||
url = url.encode("ascii")
|
||||
diff --git a/Lib/urllib/parse.py b/Lib/urllib/parse.py
|
||||
index ac6e7a9cee..717e990997 100644
|
||||
--- a/Lib/urllib/parse.py
|
||||
+++ b/Lib/urllib/parse.py
|
||||
@@ -25,6 +25,10 @@ currently not entirely compliant with this RFC due to defacto
|
||||
scenarios for parsing, and for backward compatibility purposes, some
|
||||
parsing quirks from older RFCs are retained. The testcases in
|
||||
test_urlparse.py provides a good indicator of parsing behavior.
|
||||
+
|
||||
+The WHATWG URL Parser spec should also be considered. We are not compliant with
|
||||
+it either due to existing user code API behavior expectations (Hyrum's Law).
|
||||
+It serves as a useful guide when making changes.
|
||||
"""
|
||||
|
||||
import re
|
||||
@@ -76,6 +80,10 @@ scheme_chars = ('abcdefghijklmnopqrstuvwxyz'
|
||||
'0123456789'
|
||||
'+-.')
|
||||
|
||||
+# Leading and trailing C0 control and space to be stripped per WHATWG spec.
|
||||
+# == "".join([chr(i) for i in range(0, 0x20 + 1)])
|
||||
+_WHATWG_C0_CONTROL_OR_SPACE = '\x00\x01\x02\x03\x04\x05\x06\x07\x08\t\n\x0b\x0c\r\x0e\x0f\x10\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f '
|
||||
+
|
||||
# Unsafe bytes to be removed per WHATWG spec
|
||||
_UNSAFE_URL_BYTES_TO_REMOVE = ['\t', '\r', '\n']
|
||||
|
||||
@@ -426,6 +434,10 @@ def urlsplit(url, scheme='', allow_fragments=True):
|
||||
url, scheme, _coerce_result = _coerce_args(url, scheme)
|
||||
url = _remove_unsafe_bytes_from_url(url)
|
||||
scheme = _remove_unsafe_bytes_from_url(scheme)
|
||||
+ # Only lstrip url as some applications rely on preserving trailing space.
|
||||
+ # (https://url.spec.whatwg.org/#concept-basic-url-parser would strip both)
|
||||
+ url = url.lstrip(_WHATWG_C0_CONTROL_OR_SPACE)
|
||||
+ scheme = scheme.strip(_WHATWG_C0_CONTROL_OR_SPACE)
|
||||
allow_fragments = bool(allow_fragments)
|
||||
key = url, scheme, allow_fragments, type(url), type(scheme)
|
||||
cached = _parse_cache.get(key, None)
|
||||
diff --git a/Misc/NEWS.d/next/Security/2023-03-07-20-59-17.gh-issue-102153.14CLSZ.rst b/Misc/NEWS.d/next/Security/2023-03-07-20-59-17.gh-issue-102153.14CLSZ.rst
|
||||
new file mode 100644
|
||||
index 0000000000..e57ac4ed3a
|
||||
--- /dev/null
|
||||
+++ b/Misc/NEWS.d/next/Security/2023-03-07-20-59-17.gh-issue-102153.14CLSZ.rst
|
||||
@@ -0,0 +1,3 @@
|
||||
+:func:`urllib.parse.urlsplit` now strips leading C0 control and space
|
||||
+characters following the specification for URLs defined by WHATWG in
|
||||
+response to CVE-2023-24329. Patch by Illia Volochii.
|
|
@ -0,0 +1,648 @@
|
|||
From 9f39318072b5775cf527f83daf8cb5d64678ac86 Mon Sep 17 00:00:00 2001
|
||||
From: =?UTF-8?q?=C5=81ukasz=20Langa?= <lukasz@langa.pl>
|
||||
Date: Tue, 22 Aug 2023 19:57:01 +0200
|
||||
Subject: [PATCH 1/4] gh-108310: Fix CVE-2023-40217: Check for & avoid the ssl
|
||||
pre-close flaw (#108321)
|
||||
MIME-Version: 1.0
|
||||
Content-Type: text/plain; charset=UTF-8
|
||||
Content-Transfer-Encoding: 8bit
|
||||
|
||||
gh-108310: Fix CVE-2023-40217: Check for & avoid the ssl pre-close flaw
|
||||
|
||||
Instances of `ssl.SSLSocket` were vulnerable to a bypass of the TLS handshake
|
||||
and included protections (like certificate verification) and treating sent
|
||||
unencrypted data as if it were post-handshake TLS encrypted data.
|
||||
|
||||
The vulnerability is caused when a socket is connected, data is sent by the
|
||||
malicious peer and stored in a buffer, and then the malicious peer closes the
|
||||
socket within a small timing window before the other peers’ TLS handshake can
|
||||
begin. After this sequence of events the closed socket will not immediately
|
||||
attempt a TLS handshake due to not being connected but will also allow the
|
||||
buffered data to be read as if a successful TLS handshake had occurred.
|
||||
|
||||
Co-authored-by: Gregory P. Smith [Google LLC] <greg@krypto.org>
|
||||
---
|
||||
Lib/ssl.py | 31 ++-
|
||||
Lib/test/test_ssl.py | 214 ++++++++++++++++++
|
||||
...-08-22-17-39-12.gh-issue-108310.fVM3sg.rst | 7 +
|
||||
3 files changed, 251 insertions(+), 1 deletion(-)
|
||||
create mode 100644 Misc/NEWS.d/next/Security/2023-08-22-17-39-12.gh-issue-108310.fVM3sg.rst
|
||||
|
||||
diff --git a/Lib/ssl.py b/Lib/ssl.py
|
||||
index c5c5529..288f237 100644
|
||||
--- a/Lib/ssl.py
|
||||
+++ b/Lib/ssl.py
|
||||
@@ -741,7 +741,7 @@ class SSLSocket(socket):
|
||||
type=sock.type,
|
||||
proto=sock.proto,
|
||||
fileno=sock.fileno())
|
||||
- self.settimeout(sock.gettimeout())
|
||||
+ sock_timeout = sock.gettimeout()
|
||||
sock.detach()
|
||||
elif fileno is not None:
|
||||
socket.__init__(self, fileno=fileno)
|
||||
@@ -755,9 +755,38 @@ class SSLSocket(socket):
|
||||
if e.errno != errno.ENOTCONN:
|
||||
raise
|
||||
connected = False
|
||||
+ blocking = self.getblocking()
|
||||
+ self.setblocking(False)
|
||||
+ try:
|
||||
+ # We are not connected so this is not supposed to block, but
|
||||
+ # testing revealed otherwise on macOS and Windows so we do
|
||||
+ # the non-blocking dance regardless. Our raise when any data
|
||||
+ # is found means consuming the data is harmless.
|
||||
+ notconn_pre_handshake_data = self.recv(1)
|
||||
+ except OSError as e:
|
||||
+ # EINVAL occurs for recv(1) on non-connected on unix sockets.
|
||||
+ if e.errno not in (errno.ENOTCONN, errno.EINVAL):
|
||||
+ raise
|
||||
+ notconn_pre_handshake_data = b''
|
||||
+ self.setblocking(blocking)
|
||||
+ if notconn_pre_handshake_data:
|
||||
+ # This prevents pending data sent to the socket before it was
|
||||
+ # closed from escaping to the caller who could otherwise
|
||||
+ # presume it came through a successful TLS connection.
|
||||
+ reason = "Closed before TLS handshake with data in recv buffer."
|
||||
+ notconn_pre_handshake_data_error = SSLError(e.errno, reason)
|
||||
+ # Add the SSLError attributes that _ssl.c always adds.
|
||||
+ notconn_pre_handshake_data_error.reason = reason
|
||||
+ notconn_pre_handshake_data_error.library = None
|
||||
+ try:
|
||||
+ self.close()
|
||||
+ except OSError:
|
||||
+ pass
|
||||
+ raise notconn_pre_handshake_data_error
|
||||
else:
|
||||
connected = True
|
||||
|
||||
+ self.settimeout(sock_timeout) # Must come after setblocking() calls.
|
||||
self._closed = False
|
||||
self._sslobj = None
|
||||
self._connected = connected
|
||||
diff --git a/Lib/test/test_ssl.py b/Lib/test/test_ssl.py
|
||||
index b35db25..e24d11b 100644
|
||||
--- a/Lib/test/test_ssl.py
|
||||
+++ b/Lib/test/test_ssl.py
|
||||
@@ -3,11 +3,14 @@
|
||||
import sys
|
||||
import unittest
|
||||
from test import support
|
||||
+import re
|
||||
import socket
|
||||
import select
|
||||
+import struct
|
||||
import time
|
||||
import datetime
|
||||
import gc
|
||||
+import http.client
|
||||
import os
|
||||
import errno
|
||||
import pprint
|
||||
@@ -3940,6 +3943,217 @@ class TestPostHandshakeAuth(unittest.TestCase):
|
||||
# server cert has not been validated
|
||||
self.assertEqual(s.getpeercert(), {})
|
||||
|
||||
+def set_socket_so_linger_on_with_zero_timeout(sock):
|
||||
+ sock.setsockopt(socket.SOL_SOCKET, socket.SO_LINGER, struct.pack('ii', 1, 0))
|
||||
+
|
||||
+
|
||||
+class TestPreHandshakeClose(unittest.TestCase):
|
||||
+ """Verify behavior of close sockets with received data before to the handshake.
|
||||
+ """
|
||||
+
|
||||
+ class SingleConnectionTestServerThread(threading.Thread):
|
||||
+
|
||||
+ def __init__(self, *, name, call_after_accept):
|
||||
+ self.call_after_accept = call_after_accept
|
||||
+ self.received_data = b'' # set by .run()
|
||||
+ self.wrap_error = None # set by .run()
|
||||
+ self.listener = None # set by .start()
|
||||
+ self.port = None # set by .start()
|
||||
+ super().__init__(name=name)
|
||||
+
|
||||
+ def __enter__(self):
|
||||
+ self.start()
|
||||
+ return self
|
||||
+
|
||||
+ def __exit__(self, *args):
|
||||
+ try:
|
||||
+ if self.listener:
|
||||
+ self.listener.close()
|
||||
+ except OSError:
|
||||
+ pass
|
||||
+ self.join()
|
||||
+ self.wrap_error = None # avoid dangling references
|
||||
+
|
||||
+ def start(self):
|
||||
+ self.ssl_ctx = ssl.create_default_context(ssl.Purpose.CLIENT_AUTH)
|
||||
+ self.ssl_ctx.verify_mode = ssl.CERT_REQUIRED
|
||||
+ self.ssl_ctx.load_verify_locations(cafile=ONLYCERT)
|
||||
+ self.ssl_ctx.load_cert_chain(certfile=ONLYCERT, keyfile=ONLYKEY)
|
||||
+ self.listener = socket.socket()
|
||||
+ self.port = support.bind_port(self.listener)
|
||||
+ self.listener.settimeout(2.0)
|
||||
+ self.listener.listen(1)
|
||||
+ super().start()
|
||||
+
|
||||
+ def run(self):
|
||||
+ conn, address = self.listener.accept()
|
||||
+ self.listener.close()
|
||||
+ with conn:
|
||||
+ if self.call_after_accept(conn):
|
||||
+ return
|
||||
+ try:
|
||||
+ tls_socket = self.ssl_ctx.wrap_socket(conn, server_side=True)
|
||||
+ except OSError as err: # ssl.SSLError inherits from OSError
|
||||
+ self.wrap_error = err
|
||||
+ else:
|
||||
+ try:
|
||||
+ self.received_data = tls_socket.recv(400)
|
||||
+ except OSError:
|
||||
+ pass # closed, protocol error, etc.
|
||||
+
|
||||
+ def non_linux_skip_if_other_okay_error(self, err):
|
||||
+ if sys.platform == "linux":
|
||||
+ return # Expect the full test setup to always work on Linux.
|
||||
+ if (isinstance(err, ConnectionResetError) or
|
||||
+ (isinstance(err, OSError) and err.errno == errno.EINVAL) or
|
||||
+ re.search('wrong.version.number', getattr(err, "reason", ""), re.I)):
|
||||
+ # On Windows the TCP RST leads to a ConnectionResetError
|
||||
+ # (ECONNRESET) which Linux doesn't appear to surface to userspace.
|
||||
+ # If wrap_socket() winds up on the "if connected:" path and doing
|
||||
+ # the actual wrapping... we get an SSLError from OpenSSL. Typically
|
||||
+ # WRONG_VERSION_NUMBER. While appropriate, neither is the scenario
|
||||
+ # we're specifically trying to test. The way this test is written
|
||||
+ # is known to work on Linux. We'll skip it anywhere else that it
|
||||
+ # does not present as doing so.
|
||||
+ self.skipTest("Could not recreate conditions on {}: \
|
||||
+ err={}".format(sys.platform,err))
|
||||
+ # If maintaining this conditional winds up being a problem.
|
||||
+ # just turn this into an unconditional skip anything but Linux.
|
||||
+ # The important thing is that our CI has the logic covered.
|
||||
+
|
||||
+ def test_preauth_data_to_tls_server(self):
|
||||
+ server_accept_called = threading.Event()
|
||||
+ ready_for_server_wrap_socket = threading.Event()
|
||||
+
|
||||
+ def call_after_accept(unused):
|
||||
+ server_accept_called.set()
|
||||
+ if not ready_for_server_wrap_socket.wait(2.0):
|
||||
+ raise RuntimeError("wrap_socket event never set, test may fail.")
|
||||
+ return False # Tell the server thread to continue.
|
||||
+
|
||||
+ server = self.SingleConnectionTestServerThread(
|
||||
+ call_after_accept=call_after_accept,
|
||||
+ name="preauth_data_to_tls_server")
|
||||
+ server.__enter__() # starts it
|
||||
+ self.addCleanup(server.__exit__) # ... & unittest.TestCase stops it.
|
||||
+
|
||||
+ with socket.socket() as client:
|
||||
+ client.connect(server.listener.getsockname())
|
||||
+ # This forces an immediate connection close via RST on .close().
|
||||
+ set_socket_so_linger_on_with_zero_timeout(client)
|
||||
+ client.setblocking(False)
|
||||
+
|
||||
+ server_accept_called.wait()
|
||||
+ client.send(b"DELETE /data HTTP/1.0\r\n\r\n")
|
||||
+ client.close() # RST
|
||||
+
|
||||
+ ready_for_server_wrap_socket.set()
|
||||
+ server.join()
|
||||
+ wrap_error = server.wrap_error
|
||||
+ self.assertEqual(b"", server.received_data)
|
||||
+ self.assertIsInstance(wrap_error, OSError) # All platforms.
|
||||
+ self.non_linux_skip_if_other_okay_error(wrap_error)
|
||||
+ self.assertIsInstance(wrap_error, ssl.SSLError)
|
||||
+ self.assertIn("before TLS handshake with data", wrap_error.args[1])
|
||||
+ self.assertIn("before TLS handshake with data", wrap_error.reason)
|
||||
+ self.assertNotEqual(0, wrap_error.args[0])
|
||||
+ self.assertIsNone(wrap_error.library, msg="attr must exist")
|
||||
+
|
||||
+ def test_preauth_data_to_tls_client(self):
|
||||
+ client_can_continue_with_wrap_socket = threading.Event()
|
||||
+
|
||||
+ def call_after_accept(conn_to_client):
|
||||
+ # This forces an immediate connection close via RST on .close().
|
||||
+ set_socket_so_linger_on_with_zero_timeout(conn_to_client)
|
||||
+ conn_to_client.send(
|
||||
+ b"HTTP/1.0 307 Temporary Redirect\r\n"
|
||||
+ b"Location: https://example.com/someone-elses-server\r\n"
|
||||
+ b"\r\n")
|
||||
+ conn_to_client.close() # RST
|
||||
+ client_can_continue_with_wrap_socket.set()
|
||||
+ return True # Tell the server to stop.
|
||||
+
|
||||
+ server = self.SingleConnectionTestServerThread(
|
||||
+ call_after_accept=call_after_accept,
|
||||
+ name="preauth_data_to_tls_client")
|
||||
+ server.__enter__() # starts it
|
||||
+ self.addCleanup(server.__exit__) # ... & unittest.TestCase stops it.
|
||||
+
|
||||
+ # Redundant; call_after_accept sets SO_LINGER on the accepted conn.
|
||||
+ set_socket_so_linger_on_with_zero_timeout(server.listener)
|
||||
+
|
||||
+ with socket.socket() as client:
|
||||
+ client.connect(server.listener.getsockname())
|
||||
+ if not client_can_continue_with_wrap_socket.wait(2.0):
|
||||
+ self.fail("test server took too long.")
|
||||
+ ssl_ctx = ssl.create_default_context()
|
||||
+ try:
|
||||
+ tls_client = ssl_ctx.wrap_socket(
|
||||
+ client, server_hostname="localhost")
|
||||
+ except OSError as err: # SSLError inherits from OSError
|
||||
+ wrap_error = err
|
||||
+ received_data = b""
|
||||
+ else:
|
||||
+ wrap_error = None
|
||||
+ received_data = tls_client.recv(400)
|
||||
+ tls_client.close()
|
||||
+
|
||||
+ server.join()
|
||||
+ self.assertEqual(b"", received_data)
|
||||
+ self.assertIsInstance(wrap_error, OSError) # All platforms.
|
||||
+ self.non_linux_skip_if_other_okay_error(wrap_error)
|
||||
+ self.assertIsInstance(wrap_error, ssl.SSLError)
|
||||
+ self.assertIn("before TLS handshake with data", wrap_error.args[1])
|
||||
+ self.assertIn("before TLS handshake with data", wrap_error.reason)
|
||||
+ self.assertNotEqual(0, wrap_error.args[0])
|
||||
+ self.assertIsNone(wrap_error.library, msg="attr must exist")
|
||||
+
|
||||
+ def test_https_client_non_tls_response_ignored(self):
|
||||
+
|
||||
+ server_responding = threading.Event()
|
||||
+
|
||||
+ class SynchronizedHTTPSConnection(http.client.HTTPSConnection):
|
||||
+ def connect(self):
|
||||
+ http.client.HTTPConnection.connect(self)
|
||||
+ # Wait for our fault injection server to have done its thing.
|
||||
+ if not server_responding.wait(1.0) and support.verbose:
|
||||
+ sys.stdout.write("server_responding event never set.")
|
||||
+ self.sock = self._context.wrap_socket(
|
||||
+ self.sock, server_hostname=self.host)
|
||||
+
|
||||
+ def call_after_accept(conn_to_client):
|
||||
+ # This forces an immediate connection close via RST on .close().
|
||||
+ set_socket_so_linger_on_with_zero_timeout(conn_to_client)
|
||||
+ conn_to_client.send(
|
||||
+ b"HTTP/1.0 402 Payment Required\r\n"
|
||||
+ b"\r\n")
|
||||
+ conn_to_client.close() # RST
|
||||
+ server_responding.set()
|
||||
+ return True # Tell the server to stop.
|
||||
+
|
||||
+ server = self.SingleConnectionTestServerThread(
|
||||
+ call_after_accept=call_after_accept,
|
||||
+ name="non_tls_http_RST_responder")
|
||||
+ server.__enter__() # starts it
|
||||
+ self.addCleanup(server.__exit__) # ... & unittest.TestCase stops it.
|
||||
+ # Redundant; call_after_accept sets SO_LINGER on the accepted conn.
|
||||
+ set_socket_so_linger_on_with_zero_timeout(server.listener)
|
||||
+
|
||||
+ connection = SynchronizedHTTPSConnection(
|
||||
+ f"localhost",
|
||||
+ port=server.port,
|
||||
+ context=ssl.create_default_context(),
|
||||
+ timeout=2.0,
|
||||
+ )
|
||||
+ # There are lots of reasons this raises as desired, long before this
|
||||
+ # test was added. Sending the request requires a successful TLS wrapped
|
||||
+ # socket; that fails if the connection is broken. It may seem pointless
|
||||
+ # to test this. It serves as an illustration of something that we never
|
||||
+ # want to happen... properly not happening.
|
||||
+ with self.assertRaises(OSError) as err_ctx:
|
||||
+ connection.request("HEAD", "/test", headers={"Host": "localhost"})
|
||||
+ response = connection.getresponse()
|
||||
+
|
||||
|
||||
def test_main(verbose=False):
|
||||
if support.verbose:
|
||||
diff --git a/Misc/NEWS.d/next/Security/2023-08-22-17-39-12.gh-issue-108310.fVM3sg.rst b/Misc/NEWS.d/next/Security/2023-08-22-17-39-12.gh-issue-108310.fVM3sg.rst
|
||||
new file mode 100644
|
||||
index 0000000..403c77a
|
||||
--- /dev/null
|
||||
+++ b/Misc/NEWS.d/next/Security/2023-08-22-17-39-12.gh-issue-108310.fVM3sg.rst
|
||||
@@ -0,0 +1,7 @@
|
||||
+Fixed an issue where instances of :class:`ssl.SSLSocket` were vulnerable to
|
||||
+a bypass of the TLS handshake and included protections (like certificate
|
||||
+verification) and treating sent unencrypted data as if it were
|
||||
+post-handshake TLS encrypted data. Security issue reported as
|
||||
+`CVE-2023-40217
|
||||
+<https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2023-40217>`_ by
|
||||
+Aapo Oksman. Patch by Gregory P. Smith.
|
||||
--
|
||||
2.41.0
|
||||
|
||||
|
||||
From 6fbb37c0b7ab8ce1db8e2e78df62d6e5c1e56766 Mon Sep 17 00:00:00 2001
|
||||
From: "Miss Islington (bot)"
|
||||
<31488909+miss-islington@users.noreply.github.com>
|
||||
Date: Wed, 23 Aug 2023 03:10:56 -0700
|
||||
Subject: [PATCH 2/4] gh-108342: Break ref cycle in SSLSocket._create() exc
|
||||
(GH-108344) (#108352)
|
||||
|
||||
Explicitly break a reference cycle when SSLSocket._create() raises an
|
||||
exception. Clear the variable storing the exception, since the
|
||||
exception traceback contains the variables and so creates a reference
|
||||
cycle.
|
||||
|
||||
This test leak was introduced by the test added for the fix of GH-108310.
|
||||
(cherry picked from commit 64f99350351bc46e016b2286f36ba7cd669b79e3)
|
||||
|
||||
Co-authored-by: Victor Stinner <vstinner@python.org>
|
||||
---
|
||||
Lib/ssl.py | 6 +++++-
|
||||
1 file changed, 5 insertions(+), 1 deletion(-)
|
||||
|
||||
diff --git a/Lib/ssl.py b/Lib/ssl.py
|
||||
index 288f237..67869c9 100644
|
||||
--- a/Lib/ssl.py
|
||||
+++ b/Lib/ssl.py
|
||||
@@ -782,7 +782,11 @@ class SSLSocket(socket):
|
||||
self.close()
|
||||
except OSError:
|
||||
pass
|
||||
- raise notconn_pre_handshake_data_error
|
||||
+ try:
|
||||
+ raise notconn_pre_handshake_data_error
|
||||
+ finally:
|
||||
+ # Explicitly break the reference cycle.
|
||||
+ notconn_pre_handshake_data_error = None
|
||||
else:
|
||||
connected = True
|
||||
|
||||
--
|
||||
2.41.0
|
||||
|
||||
|
||||
From 579d809e60e569f06c00844cc3a3f06a0e359603 Mon Sep 17 00:00:00 2001
|
||||
From: =?UTF-8?q?=C5=81ukasz=20Langa?= <lukasz@langa.pl>
|
||||
Date: Thu, 24 Aug 2023 12:09:30 +0200
|
||||
Subject: [PATCH 3/4] gh-108342: Make ssl TestPreHandshakeClose more reliable
|
||||
(GH-108370) (#108408)
|
||||
|
||||
* In preauth tests of test_ssl, explicitly break reference cycles
|
||||
invoving SingleConnectionTestServerThread to make sure that the
|
||||
thread is deleted. Otherwise, the test marks the environment as
|
||||
altered because the threading module sees a "dangling thread"
|
||||
(SingleConnectionTestServerThread). This test leak was introduced
|
||||
by the test added for the fix of issue gh-108310.
|
||||
* Use support.SHORT_TIMEOUT instead of hardcoded 1.0 or 2.0 seconds
|
||||
timeout.
|
||||
* SingleConnectionTestServerThread.run() catchs TimeoutError
|
||||
* Fix a race condition (missing synchronization) in
|
||||
test_preauth_data_to_tls_client(): the server now waits until the
|
||||
client connect() completed in call_after_accept().
|
||||
* test_https_client_non_tls_response_ignored() calls server.join()
|
||||
explicitly.
|
||||
* Replace "localhost" with server.listener.getsockname()[0].
|
||||
(cherry picked from commit 592bacb6fc0833336c0453e818e9b95016e9fd47)
|
||||
---
|
||||
Lib/test/test_ssl.py | 102 ++++++++++++++++++++++++++++++-------------
|
||||
1 file changed, 71 insertions(+), 31 deletions(-)
|
||||
|
||||
diff --git a/Lib/test/test_ssl.py b/Lib/test/test_ssl.py
|
||||
index e24d11b..6332330 100644
|
||||
--- a/Lib/test/test_ssl.py
|
||||
+++ b/Lib/test/test_ssl.py
|
||||
@@ -3953,12 +3953,16 @@ class TestPreHandshakeClose(unittest.TestCase):
|
||||
|
||||
class SingleConnectionTestServerThread(threading.Thread):
|
||||
|
||||
- def __init__(self, *, name, call_after_accept):
|
||||
+ def __init__(self, *, name, call_after_accept, timeout=None):
|
||||
self.call_after_accept = call_after_accept
|
||||
self.received_data = b'' # set by .run()
|
||||
self.wrap_error = None # set by .run()
|
||||
self.listener = None # set by .start()
|
||||
self.port = None # set by .start()
|
||||
+ if timeout is None:
|
||||
+ self.timeout = support.SHORT_TIMEOUT
|
||||
+ else:
|
||||
+ self.timeout = timeout
|
||||
super().__init__(name=name)
|
||||
|
||||
def __enter__(self):
|
||||
@@ -3981,13 +3985,19 @@ class TestPreHandshakeClose(unittest.TestCase):
|
||||
self.ssl_ctx.load_cert_chain(certfile=ONLYCERT, keyfile=ONLYKEY)
|
||||
self.listener = socket.socket()
|
||||
self.port = support.bind_port(self.listener)
|
||||
- self.listener.settimeout(2.0)
|
||||
+ self.listener.settimeout(self.timeout)
|
||||
self.listener.listen(1)
|
||||
super().start()
|
||||
|
||||
def run(self):
|
||||
- conn, address = self.listener.accept()
|
||||
- self.listener.close()
|
||||
+ try:
|
||||
+ conn, address = self.listener.accept()
|
||||
+ except TimeoutError:
|
||||
+ # on timeout, just close the listener
|
||||
+ return
|
||||
+ finally:
|
||||
+ self.listener.close()
|
||||
+
|
||||
with conn:
|
||||
if self.call_after_accept(conn):
|
||||
return
|
||||
@@ -4015,8 +4025,13 @@ class TestPreHandshakeClose(unittest.TestCase):
|
||||
# we're specifically trying to test. The way this test is written
|
||||
# is known to work on Linux. We'll skip it anywhere else that it
|
||||
# does not present as doing so.
|
||||
- self.skipTest("Could not recreate conditions on {}: \
|
||||
- err={}".format(sys.platform,err))
|
||||
+ try:
|
||||
+ self.skipTest("Could not recreate conditions on {}: \
|
||||
+ err={}".format(sys.platform,err))
|
||||
+ finally:
|
||||
+ # gh-108342: Explicitly break the reference cycle
|
||||
+ err = None
|
||||
+
|
||||
# If maintaining this conditional winds up being a problem.
|
||||
# just turn this into an unconditional skip anything but Linux.
|
||||
# The important thing is that our CI has the logic covered.
|
||||
@@ -4027,7 +4042,7 @@ class TestPreHandshakeClose(unittest.TestCase):
|
||||
|
||||
def call_after_accept(unused):
|
||||
server_accept_called.set()
|
||||
- if not ready_for_server_wrap_socket.wait(2.0):
|
||||
+ if not ready_for_server_wrap_socket.wait(support.SHORT_TIMEOUT):
|
||||
raise RuntimeError("wrap_socket event never set, test may fail.")
|
||||
return False # Tell the server thread to continue.
|
||||
|
||||
@@ -4049,20 +4064,31 @@ class TestPreHandshakeClose(unittest.TestCase):
|
||||
|
||||
ready_for_server_wrap_socket.set()
|
||||
server.join()
|
||||
+
|
||||
wrap_error = server.wrap_error
|
||||
- self.assertEqual(b"", server.received_data)
|
||||
- self.assertIsInstance(wrap_error, OSError) # All platforms.
|
||||
- self.non_linux_skip_if_other_okay_error(wrap_error)
|
||||
- self.assertIsInstance(wrap_error, ssl.SSLError)
|
||||
- self.assertIn("before TLS handshake with data", wrap_error.args[1])
|
||||
- self.assertIn("before TLS handshake with data", wrap_error.reason)
|
||||
- self.assertNotEqual(0, wrap_error.args[0])
|
||||
- self.assertIsNone(wrap_error.library, msg="attr must exist")
|
||||
+ server.wrap_error = None
|
||||
+ try:
|
||||
+ self.assertEqual(b"", server.received_data)
|
||||
+ self.assertIsInstance(wrap_error, OSError) # All platforms.
|
||||
+ self.non_linux_skip_if_other_okay_error(wrap_error)
|
||||
+ self.assertIsInstance(wrap_error, ssl.SSLError)
|
||||
+ self.assertIn("before TLS handshake with data", wrap_error.args[1])
|
||||
+ self.assertIn("before TLS handshake with data", wrap_error.reason)
|
||||
+ self.assertNotEqual(0, wrap_error.args[0])
|
||||
+ self.assertIsNone(wrap_error.library, msg="attr must exist")
|
||||
+ finally:
|
||||
+ # gh-108342: Explicitly break the reference cycle
|
||||
+ wrap_error = None
|
||||
+ server = None
|
||||
|
||||
def test_preauth_data_to_tls_client(self):
|
||||
+ server_can_continue_with_wrap_socket = threading.Event()
|
||||
client_can_continue_with_wrap_socket = threading.Event()
|
||||
|
||||
def call_after_accept(conn_to_client):
|
||||
+ if not server_can_continue_with_wrap_socket.wait(support.SHORT_TIMEOUT):
|
||||
+ print("ERROR: test client took too long")
|
||||
+
|
||||
# This forces an immediate connection close via RST on .close().
|
||||
set_socket_so_linger_on_with_zero_timeout(conn_to_client)
|
||||
conn_to_client.send(
|
||||
@@ -4084,8 +4110,10 @@ class TestPreHandshakeClose(unittest.TestCase):
|
||||
|
||||
with socket.socket() as client:
|
||||
client.connect(server.listener.getsockname())
|
||||
- if not client_can_continue_with_wrap_socket.wait(2.0):
|
||||
- self.fail("test server took too long.")
|
||||
+ server_can_continue_with_wrap_socket.set()
|
||||
+
|
||||
+ if not client_can_continue_with_wrap_socket.wait(support.SHORT_TIMEOUT):
|
||||
+ self.fail("test server took too long")
|
||||
ssl_ctx = ssl.create_default_context()
|
||||
try:
|
||||
tls_client = ssl_ctx.wrap_socket(
|
||||
@@ -4099,24 +4127,31 @@ class TestPreHandshakeClose(unittest.TestCase):
|
||||
tls_client.close()
|
||||
|
||||
server.join()
|
||||
- self.assertEqual(b"", received_data)
|
||||
- self.assertIsInstance(wrap_error, OSError) # All platforms.
|
||||
- self.non_linux_skip_if_other_okay_error(wrap_error)
|
||||
- self.assertIsInstance(wrap_error, ssl.SSLError)
|
||||
- self.assertIn("before TLS handshake with data", wrap_error.args[1])
|
||||
- self.assertIn("before TLS handshake with data", wrap_error.reason)
|
||||
- self.assertNotEqual(0, wrap_error.args[0])
|
||||
- self.assertIsNone(wrap_error.library, msg="attr must exist")
|
||||
+ try:
|
||||
+ self.assertEqual(b"", received_data)
|
||||
+ self.assertIsInstance(wrap_error, OSError) # All platforms.
|
||||
+ self.non_linux_skip_if_other_okay_error(wrap_error)
|
||||
+ self.assertIsInstance(wrap_error, ssl.SSLError)
|
||||
+ self.assertIn("before TLS handshake with data", wrap_error.args[1])
|
||||
+ self.assertIn("before TLS handshake with data", wrap_error.reason)
|
||||
+ self.assertNotEqual(0, wrap_error.args[0])
|
||||
+ self.assertIsNone(wrap_error.library, msg="attr must exist")
|
||||
+ finally:
|
||||
+ # gh-108342: Explicitly break the reference cycle
|
||||
+ wrap_error = None
|
||||
+ server = None
|
||||
|
||||
def test_https_client_non_tls_response_ignored(self):
|
||||
-
|
||||
server_responding = threading.Event()
|
||||
|
||||
class SynchronizedHTTPSConnection(http.client.HTTPSConnection):
|
||||
def connect(self):
|
||||
+ # Call clear text HTTP connect(), not the encrypted HTTPS (TLS)
|
||||
+ # connect(): wrap_socket() is called manually below.
|
||||
http.client.HTTPConnection.connect(self)
|
||||
+
|
||||
# Wait for our fault injection server to have done its thing.
|
||||
- if not server_responding.wait(1.0) and support.verbose:
|
||||
+ if not server_responding.wait(support.SHORT_TIMEOUT) and support.verbose:
|
||||
sys.stdout.write("server_responding event never set.")
|
||||
self.sock = self._context.wrap_socket(
|
||||
self.sock, server_hostname=self.host)
|
||||
@@ -4131,29 +4166,34 @@ class TestPreHandshakeClose(unittest.TestCase):
|
||||
server_responding.set()
|
||||
return True # Tell the server to stop.
|
||||
|
||||
+ timeout = 2.0
|
||||
server = self.SingleConnectionTestServerThread(
|
||||
call_after_accept=call_after_accept,
|
||||
- name="non_tls_http_RST_responder")
|
||||
+ name="non_tls_http_RST_responder",
|
||||
+ timeout=timeout)
|
||||
server.__enter__() # starts it
|
||||
self.addCleanup(server.__exit__) # ... & unittest.TestCase stops it.
|
||||
# Redundant; call_after_accept sets SO_LINGER on the accepted conn.
|
||||
set_socket_so_linger_on_with_zero_timeout(server.listener)
|
||||
|
||||
connection = SynchronizedHTTPSConnection(
|
||||
- f"localhost",
|
||||
+ server.listener.getsockname()[0],
|
||||
port=server.port,
|
||||
context=ssl.create_default_context(),
|
||||
- timeout=2.0,
|
||||
+ timeout=timeout,
|
||||
)
|
||||
+
|
||||
# There are lots of reasons this raises as desired, long before this
|
||||
# test was added. Sending the request requires a successful TLS wrapped
|
||||
# socket; that fails if the connection is broken. It may seem pointless
|
||||
# to test this. It serves as an illustration of something that we never
|
||||
# want to happen... properly not happening.
|
||||
- with self.assertRaises(OSError) as err_ctx:
|
||||
+ with self.assertRaises(OSError):
|
||||
connection.request("HEAD", "/test", headers={"Host": "localhost"})
|
||||
response = connection.getresponse()
|
||||
|
||||
+ server.join()
|
||||
+
|
||||
|
||||
def test_main(verbose=False):
|
||||
if support.verbose:
|
||||
--
|
||||
2.41.0
|
||||
|
||||
|
||||
From 2e8776245e9fb8ede784077df26684b4b53df0bc Mon Sep 17 00:00:00 2001
|
||||
From: Charalampos Stratakis <cstratak@redhat.com>
|
||||
Date: Mon, 25 Sep 2023 21:55:29 +0200
|
||||
Subject: [PATCH 4/4] Downstream: Additional fixup for 3.6:
|
||||
|
||||
Use alternative for self.getblocking(), which was added in Python 3.7
|
||||
see: https://docs.python.org/3/library/socket.html#socket.socket.getblocking
|
||||
|
||||
Set self._sslobj early to avoid AttributeError
|
||||
---
|
||||
Lib/ssl.py | 3 ++-
|
||||
1 file changed, 2 insertions(+), 1 deletion(-)
|
||||
|
||||
diff --git a/Lib/ssl.py b/Lib/ssl.py
|
||||
index 67869c9..daedc82 100644
|
||||
--- a/Lib/ssl.py
|
||||
+++ b/Lib/ssl.py
|
||||
@@ -690,6 +690,7 @@ class SSLSocket(socket):
|
||||
suppress_ragged_eofs=True, npn_protocols=None, ciphers=None,
|
||||
server_hostname=None,
|
||||
_context=None, _session=None):
|
||||
+ self._sslobj = None
|
||||
|
||||
if _context:
|
||||
self._context = _context
|
||||
@@ -755,7 +756,7 @@ class SSLSocket(socket):
|
||||
if e.errno != errno.ENOTCONN:
|
||||
raise
|
||||
connected = False
|
||||
- blocking = self.getblocking()
|
||||
+ blocking = (self.gettimeout() != 0)
|
||||
self.setblocking(False)
|
||||
try:
|
||||
# We are not connected so this is not supposed to block, but
|
||||
--
|
||||
2.41.0
|
||||
|
|
@ -0,0 +1,143 @@
|
|||
From c563f409ea30bcb0623d785428c9257917371b76 Mon Sep 17 00:00:00 2001
|
||||
From: "Miss Islington (bot)"
|
||||
<31488909+miss-islington@users.noreply.github.com>
|
||||
Date: Thu, 23 Jan 2020 06:49:19 -0800
|
||||
Subject: [PATCH] bpo-39421: Fix posible crash in heapq with custom comparison
|
||||
operators (GH-18118) (GH-18146)
|
||||
|
||||
(cherry picked from commit 79f89e6e5a659846d1068e8b1bd8e491ccdef861)
|
||||
|
||||
Co-authored-by: Pablo Galindo <Pablogsal@gmail.com>
|
||||
---
|
||||
Lib/test/test_heapq.py | 31 ++++++++++++++++
|
||||
.../2020-01-22-15-53-37.bpo-39421.O3nG7u.rst | 2 ++
|
||||
Modules/_heapqmodule.c | 35 ++++++++++++++-----
|
||||
3 files changed, 59 insertions(+), 9 deletions(-)
|
||||
create mode 100644 Misc/NEWS.d/next/Core and Builtins/2020-01-22-15-53-37.bpo-39421.O3nG7u.rst
|
||||
|
||||
diff --git a/Lib/test/test_heapq.py b/Lib/test/test_heapq.py
|
||||
index 2f8c648d84a58..7c3fb0210f69b 100644
|
||||
--- a/Lib/test/test_heapq.py
|
||||
+++ b/Lib/test/test_heapq.py
|
||||
@@ -414,6 +414,37 @@ def test_heappop_mutating_heap(self):
|
||||
with self.assertRaises((IndexError, RuntimeError)):
|
||||
self.module.heappop(heap)
|
||||
|
||||
+ def test_comparison_operator_modifiying_heap(self):
|
||||
+ # See bpo-39421: Strong references need to be taken
|
||||
+ # when comparing objects as they can alter the heap
|
||||
+ class EvilClass(int):
|
||||
+ def __lt__(self, o):
|
||||
+ heap.clear()
|
||||
+ return NotImplemented
|
||||
+
|
||||
+ heap = []
|
||||
+ self.module.heappush(heap, EvilClass(0))
|
||||
+ self.assertRaises(IndexError, self.module.heappushpop, heap, 1)
|
||||
+
|
||||
+ def test_comparison_operator_modifiying_heap_two_heaps(self):
|
||||
+
|
||||
+ class h(int):
|
||||
+ def __lt__(self, o):
|
||||
+ list2.clear()
|
||||
+ return NotImplemented
|
||||
+
|
||||
+ class g(int):
|
||||
+ def __lt__(self, o):
|
||||
+ list1.clear()
|
||||
+ return NotImplemented
|
||||
+
|
||||
+ list1, list2 = [], []
|
||||
+
|
||||
+ self.module.heappush(list1, h(0))
|
||||
+ self.module.heappush(list2, g(0))
|
||||
+
|
||||
+ self.assertRaises((IndexError, RuntimeError), self.module.heappush, list1, g(1))
|
||||
+ self.assertRaises((IndexError, RuntimeError), self.module.heappush, list2, h(1))
|
||||
|
||||
class TestErrorHandlingPython(TestErrorHandling, TestCase):
|
||||
module = py_heapq
|
||||
diff --git a/Misc/NEWS.d/next/Core and Builtins/2020-01-22-15-53-37.bpo-39421.O3nG7u.rst b/Misc/NEWS.d/next/Core and Builtins/2020-01-22-15-53-37.bpo-39421.O3nG7u.rst
|
||||
new file mode 100644
|
||||
index 0000000000000..bae008150ee12
|
||||
--- /dev/null
|
||||
+++ b/Misc/NEWS.d/next/Core and Builtins/2020-01-22-15-53-37.bpo-39421.O3nG7u.rst
|
||||
@@ -0,0 +1,2 @@
|
||||
+Fix possible crashes when operating with the functions in the :mod:`heapq`
|
||||
+module and custom comparison operators.
|
||||
diff --git a/Modules/_heapqmodule.c b/Modules/_heapqmodule.c
|
||||
index b499e1f668aae..0fb35ffe5ec48 100644
|
||||
--- a/Modules/_heapqmodule.c
|
||||
+++ b/Modules/_heapqmodule.c
|
||||
@@ -29,7 +29,11 @@ siftdown(PyListObject *heap, Py_ssize_t startpos, Py_ssize_t pos)
|
||||
while (pos > startpos) {
|
||||
parentpos = (pos - 1) >> 1;
|
||||
parent = arr[parentpos];
|
||||
+ Py_INCREF(newitem);
|
||||
+ Py_INCREF(parent);
|
||||
cmp = PyObject_RichCompareBool(newitem, parent, Py_LT);
|
||||
+ Py_DECREF(parent);
|
||||
+ Py_DECREF(newitem);
|
||||
if (cmp < 0)
|
||||
return -1;
|
||||
if (size != PyList_GET_SIZE(heap)) {
|
||||
@@ -71,10 +75,13 @@ siftup(PyListObject *heap, Py_ssize_t pos)
|
||||
/* Set childpos to index of smaller child. */
|
||||
childpos = 2*pos + 1; /* leftmost child position */
|
||||
if (childpos + 1 < endpos) {
|
||||
- cmp = PyObject_RichCompareBool(
|
||||
- arr[childpos],
|
||||
- arr[childpos + 1],
|
||||
- Py_LT);
|
||||
+ PyObject* a = arr[childpos];
|
||||
+ PyObject* b = arr[childpos + 1];
|
||||
+ Py_INCREF(a);
|
||||
+ Py_INCREF(b);
|
||||
+ cmp = PyObject_RichCompareBool(a, b, Py_LT);
|
||||
+ Py_DECREF(a);
|
||||
+ Py_DECREF(b);
|
||||
if (cmp < 0)
|
||||
return -1;
|
||||
childpos += ((unsigned)cmp ^ 1); /* increment when cmp==0 */
|
||||
@@ -229,7 +236,10 @@ heappushpop(PyObject *self, PyObject *args)
|
||||
return item;
|
||||
}
|
||||
|
||||
- cmp = PyObject_RichCompareBool(PyList_GET_ITEM(heap, 0), item, Py_LT);
|
||||
+ PyObject* top = PyList_GET_ITEM(heap, 0);
|
||||
+ Py_INCREF(top);
|
||||
+ cmp = PyObject_RichCompareBool(top, item, Py_LT);
|
||||
+ Py_DECREF(top);
|
||||
if (cmp < 0)
|
||||
return NULL;
|
||||
if (cmp == 0) {
|
||||
@@ -383,7 +393,11 @@ siftdown_max(PyListObject *heap, Py_ssize_t startpos, Py_ssize_t pos)
|
||||
while (pos > startpos) {
|
||||
parentpos = (pos - 1) >> 1;
|
||||
parent = arr[parentpos];
|
||||
+ Py_INCREF(parent);
|
||||
+ Py_INCREF(newitem);
|
||||
cmp = PyObject_RichCompareBool(parent, newitem, Py_LT);
|
||||
+ Py_DECREF(parent);
|
||||
+ Py_DECREF(newitem);
|
||||
if (cmp < 0)
|
||||
return -1;
|
||||
if (size != PyList_GET_SIZE(heap)) {
|
||||
@@ -425,10 +439,13 @@ siftup_max(PyListObject *heap, Py_ssize_t pos)
|
||||
/* Set childpos to index of smaller child. */
|
||||
childpos = 2*pos + 1; /* leftmost child position */
|
||||
if (childpos + 1 < endpos) {
|
||||
- cmp = PyObject_RichCompareBool(
|
||||
- arr[childpos + 1],
|
||||
- arr[childpos],
|
||||
- Py_LT);
|
||||
+ PyObject* a = arr[childpos + 1];
|
||||
+ PyObject* b = arr[childpos];
|
||||
+ Py_INCREF(a);
|
||||
+ Py_INCREF(b);
|
||||
+ cmp = PyObject_RichCompareBool(a, b, Py_LT);
|
||||
+ Py_DECREF(a);
|
||||
+ Py_DECREF(b);
|
||||
if (cmp < 0)
|
||||
return -1;
|
||||
childpos += ((unsigned)cmp ^ 1); /* increment when cmp==0 */
|
|
@ -0,0 +1,560 @@
|
|||
From cec66eefbee76ee95f252a52de0f5abc1b677f5b Mon Sep 17 00:00:00 2001
|
||||
From: Lumir Balhar <lbalhar@redhat.com>
|
||||
Date: Tue, 12 Dec 2023 13:03:42 +0100
|
||||
Subject: [PATCH] [3.6] bpo-42103: Improve validation of Plist files.
|
||||
(GH-22882) (GH-23118)
|
||||
|
||||
* Prevent some possible DoS attacks via providing invalid Plist files
|
||||
with extremely large number of objects or collection sizes.
|
||||
* Raise InvalidFileException for too large bytes and string size instead of returning garbage.
|
||||
* Raise InvalidFileException instead of ValueError for specific invalid datetime (NaN).
|
||||
* Raise InvalidFileException instead of TypeError for non-hashable dict keys.
|
||||
* Add more tests for invalid Plist files..
|
||||
(cherry picked from commit 34637a0ce21e7261b952fbd9d006474cc29b681f)
|
||||
|
||||
Co-authored-by: Serhiy Storchaka <storchaka@gmail.com>
|
||||
---
|
||||
Lib/plistlib.py | 34 +-
|
||||
Lib/test/test_plistlib.py | 394 +++++++++++++++---
|
||||
.../2020-10-23-19-20-14.bpo-42103.C5obK2.rst | 3 +
|
||||
.../2020-10-23-19-19-30.bpo-42103.cILT66.rst | 2 +
|
||||
4 files changed, 366 insertions(+), 67 deletions(-)
|
||||
create mode 100644 Misc/NEWS.d/next/Library/2020-10-23-19-20-14.bpo-42103.C5obK2.rst
|
||||
create mode 100644 Misc/NEWS.d/next/Security/2020-10-23-19-19-30.bpo-42103.cILT66.rst
|
||||
|
||||
diff --git a/Lib/plistlib.py b/Lib/plistlib.py
|
||||
index a918643..df1f346 100644
|
||||
--- a/Lib/plistlib.py
|
||||
+++ b/Lib/plistlib.py
|
||||
@@ -626,7 +626,7 @@ class _BinaryPlistParser:
|
||||
return self._read_object(top_object)
|
||||
|
||||
except (OSError, IndexError, struct.error, OverflowError,
|
||||
- UnicodeDecodeError):
|
||||
+ ValueError):
|
||||
raise InvalidFileException()
|
||||
|
||||
def _get_size(self, tokenL):
|
||||
@@ -642,7 +642,7 @@ class _BinaryPlistParser:
|
||||
def _read_ints(self, n, size):
|
||||
data = self._fp.read(size * n)
|
||||
if size in _BINARY_FORMAT:
|
||||
- return struct.unpack('>' + _BINARY_FORMAT[size] * n, data)
|
||||
+ return struct.unpack(f'>{n}{_BINARY_FORMAT[size]}', data)
|
||||
else:
|
||||
if not size or len(data) != size * n:
|
||||
raise InvalidFileException()
|
||||
@@ -701,19 +701,25 @@ class _BinaryPlistParser:
|
||||
|
||||
elif tokenH == 0x40: # data
|
||||
s = self._get_size(tokenL)
|
||||
- if self._use_builtin_types:
|
||||
- result = self._fp.read(s)
|
||||
- else:
|
||||
- result = Data(self._fp.read(s))
|
||||
+ result = self._fp.read(s)
|
||||
+ if len(result) != s:
|
||||
+ raise InvalidFileException()
|
||||
+ if not self._use_builtin_types:
|
||||
+ result = Data(result)
|
||||
|
||||
elif tokenH == 0x50: # ascii string
|
||||
s = self._get_size(tokenL)
|
||||
- result = self._fp.read(s).decode('ascii')
|
||||
- result = result
|
||||
+ data = self._fp.read(s)
|
||||
+ if len(data) != s:
|
||||
+ raise InvalidFileException()
|
||||
+ result = data.decode('ascii')
|
||||
|
||||
elif tokenH == 0x60: # unicode string
|
||||
- s = self._get_size(tokenL)
|
||||
- result = self._fp.read(s * 2).decode('utf-16be')
|
||||
+ s = self._get_size(tokenL) * 2
|
||||
+ data = self._fp.read(s)
|
||||
+ if len(data) != s:
|
||||
+ raise InvalidFileException()
|
||||
+ result = data.decode('utf-16be')
|
||||
|
||||
# tokenH == 0x80 is documented as 'UID' and appears to be used for
|
||||
# keyed-archiving, not in plists.
|
||||
@@ -737,9 +743,11 @@ class _BinaryPlistParser:
|
||||
obj_refs = self._read_refs(s)
|
||||
result = self._dict_type()
|
||||
self._objects[ref] = result
|
||||
- for k, o in zip(key_refs, obj_refs):
|
||||
- result[self._read_object(k)] = self._read_object(o)
|
||||
-
|
||||
+ try:
|
||||
+ for k, o in zip(key_refs, obj_refs):
|
||||
+ result[self._read_object(k)] = self._read_object(o)
|
||||
+ except TypeError:
|
||||
+ raise InvalidFileException()
|
||||
else:
|
||||
raise InvalidFileException()
|
||||
|
||||
diff --git a/Lib/test/test_plistlib.py b/Lib/test/test_plistlib.py
|
||||
index d47c607..f71245d 100644
|
||||
--- a/Lib/test/test_plistlib.py
|
||||
+++ b/Lib/test/test_plistlib.py
|
||||
@@ -1,5 +1,6 @@
|
||||
# Copyright (C) 2003-2013 Python Software Foundation
|
||||
|
||||
+import struct
|
||||
import unittest
|
||||
import plistlib
|
||||
import os
|
||||
@@ -90,6 +91,284 @@ TESTDATA={
|
||||
xQHHAsQC0gAAAAAAAAIBAAAAAAAAADkAAAAAAAAAAAAAAAAAAALs'''),
|
||||
}
|
||||
|
||||
+INVALID_BINARY_PLISTS = [
|
||||
+ ('too short data',
|
||||
+ b''
|
||||
+ ),
|
||||
+ ('too large offset_table_offset and offset_size = 1',
|
||||
+ b'\x00\x08'
|
||||
+ b'\x00\x00\x00\x00\x00\x00\x01\x01'
|
||||
+ b'\x00\x00\x00\x00\x00\x00\x00\x01'
|
||||
+ b'\x00\x00\x00\x00\x00\x00\x00\x00'
|
||||
+ b'\x00\x00\x00\x00\x00\x00\x00\x2a'
|
||||
+ ),
|
||||
+ ('too large offset_table_offset and nonstandard offset_size',
|
||||
+ b'\x00\x00\x00\x08'
|
||||
+ b'\x00\x00\x00\x00\x00\x00\x03\x01'
|
||||
+ b'\x00\x00\x00\x00\x00\x00\x00\x01'
|
||||
+ b'\x00\x00\x00\x00\x00\x00\x00\x00'
|
||||
+ b'\x00\x00\x00\x00\x00\x00\x00\x2c'
|
||||
+ ),
|
||||
+ ('integer overflow in offset_table_offset',
|
||||
+ b'\x00\x08'
|
||||
+ b'\x00\x00\x00\x00\x00\x00\x01\x01'
|
||||
+ b'\x00\x00\x00\x00\x00\x00\x00\x01'
|
||||
+ b'\x00\x00\x00\x00\x00\x00\x00\x00'
|
||||
+ b'\xff\xff\xff\xff\xff\xff\xff\xff'
|
||||
+ ),
|
||||
+ ('too large top_object',
|
||||
+ b'\x00\x08'
|
||||
+ b'\x00\x00\x00\x00\x00\x00\x01\x01'
|
||||
+ b'\x00\x00\x00\x00\x00\x00\x00\x01'
|
||||
+ b'\x00\x00\x00\x00\x00\x00\x00\x01'
|
||||
+ b'\x00\x00\x00\x00\x00\x00\x00\x09'
|
||||
+ ),
|
||||
+ ('integer overflow in top_object',
|
||||
+ b'\x00\x08'
|
||||
+ b'\x00\x00\x00\x00\x00\x00\x01\x01'
|
||||
+ b'\x00\x00\x00\x00\x00\x00\x00\x01'
|
||||
+ b'\xff\xff\xff\xff\xff\xff\xff\xff'
|
||||
+ b'\x00\x00\x00\x00\x00\x00\x00\x09'
|
||||
+ ),
|
||||
+ ('too large num_objects and offset_size = 1',
|
||||
+ b'\x00\x08'
|
||||
+ b'\x00\x00\x00\x00\x00\x00\x01\x01'
|
||||
+ b'\x00\x00\x00\x00\x00\x00\x00\xff'
|
||||
+ b'\x00\x00\x00\x00\x00\x00\x00\x00'
|
||||
+ b'\x00\x00\x00\x00\x00\x00\x00\x09'
|
||||
+ ),
|
||||
+ ('too large num_objects and nonstandard offset_size',
|
||||
+ b'\x00\x00\x00\x08'
|
||||
+ b'\x00\x00\x00\x00\x00\x00\x03\x01'
|
||||
+ b'\x00\x00\x00\x00\x00\x00\x00\xff'
|
||||
+ b'\x00\x00\x00\x00\x00\x00\x00\x00'
|
||||
+ b'\x00\x00\x00\x00\x00\x00\x00\x09'
|
||||
+ ),
|
||||
+ ('extremally large num_objects (32 bit)',
|
||||
+ b'\x00\x08'
|
||||
+ b'\x00\x00\x00\x00\x00\x00\x01\x01'
|
||||
+ b'\x00\x00\x00\x00\x7f\xff\xff\xff'
|
||||
+ b'\x00\x00\x00\x00\x00\x00\x00\x00'
|
||||
+ b'\x00\x00\x00\x00\x00\x00\x00\x09'
|
||||
+ ),
|
||||
+ ('extremally large num_objects (64 bit)',
|
||||
+ b'\x00\x08'
|
||||
+ b'\x00\x00\x00\x00\x00\x00\x01\x01'
|
||||
+ b'\x00\x00\x00\xff\xff\xff\xff\xff'
|
||||
+ b'\x00\x00\x00\x00\x00\x00\x00\x00'
|
||||
+ b'\x00\x00\x00\x00\x00\x00\x00\x09'
|
||||
+ ),
|
||||
+ ('integer overflow in num_objects',
|
||||
+ b'\x00\x08'
|
||||
+ b'\x00\x00\x00\x00\x00\x00\x01\x01'
|
||||
+ b'\xff\xff\xff\xff\xff\xff\xff\xff'
|
||||
+ b'\x00\x00\x00\x00\x00\x00\x00\x00'
|
||||
+ b'\x00\x00\x00\x00\x00\x00\x00\x09'
|
||||
+ ),
|
||||
+ ('offset_size = 0',
|
||||
+ b'\x00\x08'
|
||||
+ b'\x00\x00\x00\x00\x00\x00\x00\x01'
|
||||
+ b'\x00\x00\x00\x00\x00\x00\x00\x01'
|
||||
+ b'\x00\x00\x00\x00\x00\x00\x00\x00'
|
||||
+ b'\x00\x00\x00\x00\x00\x00\x00\x09'
|
||||
+ ),
|
||||
+ ('ref_size = 0',
|
||||
+ b'\xa1\x01\x00\x08\x0a'
|
||||
+ b'\x00\x00\x00\x00\x00\x00\x01\x00'
|
||||
+ b'\x00\x00\x00\x00\x00\x00\x00\x02'
|
||||
+ b'\x00\x00\x00\x00\x00\x00\x00\x00'
|
||||
+ b'\x00\x00\x00\x00\x00\x00\x00\x0b'
|
||||
+ ),
|
||||
+ ('too large offset',
|
||||
+ b'\x00\x2a'
|
||||
+ b'\x00\x00\x00\x00\x00\x00\x01\x01'
|
||||
+ b'\x00\x00\x00\x00\x00\x00\x00\x01'
|
||||
+ b'\x00\x00\x00\x00\x00\x00\x00\x00'
|
||||
+ b'\x00\x00\x00\x00\x00\x00\x00\x09'
|
||||
+ ),
|
||||
+ ('integer overflow in offset',
|
||||
+ b'\x00\xff\xff\xff\xff\xff\xff\xff\xff'
|
||||
+ b'\x00\x00\x00\x00\x00\x00\x08\x01'
|
||||
+ b'\x00\x00\x00\x00\x00\x00\x00\x01'
|
||||
+ b'\x00\x00\x00\x00\x00\x00\x00\x00'
|
||||
+ b'\x00\x00\x00\x00\x00\x00\x00\x09'
|
||||
+ ),
|
||||
+ ('too large array size',
|
||||
+ b'\xaf\x00\x01\xff\x00\x08\x0c'
|
||||
+ b'\x00\x00\x00\x00\x00\x00\x01\x01'
|
||||
+ b'\x00\x00\x00\x00\x00\x00\x00\x02'
|
||||
+ b'\x00\x00\x00\x00\x00\x00\x00\x00'
|
||||
+ b'\x00\x00\x00\x00\x00\x00\x00\x0d'
|
||||
+ ),
|
||||
+ ('extremally large array size (32-bit)',
|
||||
+ b'\xaf\x02\x7f\xff\xff\xff\x01\x00\x08\x0f'
|
||||
+ b'\x00\x00\x00\x00\x00\x00\x01\x01'
|
||||
+ b'\x00\x00\x00\x00\x00\x00\x00\x02'
|
||||
+ b'\x00\x00\x00\x00\x00\x00\x00\x00'
|
||||
+ b'\x00\x00\x00\x00\x00\x00\x00\x10'
|
||||
+ ),
|
||||
+ ('extremally large array size (64-bit)',
|
||||
+ b'\xaf\x03\x00\x00\x00\xff\xff\xff\xff\xff\x01\x00\x08\x13'
|
||||
+ b'\x00\x00\x00\x00\x00\x00\x01\x01'
|
||||
+ b'\x00\x00\x00\x00\x00\x00\x00\x02'
|
||||
+ b'\x00\x00\x00\x00\x00\x00\x00\x00'
|
||||
+ b'\x00\x00\x00\x00\x00\x00\x00\x14'
|
||||
+ ),
|
||||
+ ('integer overflow in array size',
|
||||
+ b'\xaf\x03\xff\xff\xff\xff\xff\xff\xff\xff\x01\x00\x08\x13'
|
||||
+ b'\x00\x00\x00\x00\x00\x00\x01\x01'
|
||||
+ b'\x00\x00\x00\x00\x00\x00\x00\x02'
|
||||
+ b'\x00\x00\x00\x00\x00\x00\x00\x00'
|
||||
+ b'\x00\x00\x00\x00\x00\x00\x00\x14'
|
||||
+ ),
|
||||
+ ('too large reference index',
|
||||
+ b'\xa1\x02\x00\x08\x0a'
|
||||
+ b'\x00\x00\x00\x00\x00\x00\x01\x01'
|
||||
+ b'\x00\x00\x00\x00\x00\x00\x00\x02'
|
||||
+ b'\x00\x00\x00\x00\x00\x00\x00\x00'
|
||||
+ b'\x00\x00\x00\x00\x00\x00\x00\x0b'
|
||||
+ ),
|
||||
+ ('integer overflow in reference index',
|
||||
+ b'\xa1\xff\xff\xff\xff\xff\xff\xff\xff\x00\x08\x11'
|
||||
+ b'\x00\x00\x00\x00\x00\x00\x01\x08'
|
||||
+ b'\x00\x00\x00\x00\x00\x00\x00\x02'
|
||||
+ b'\x00\x00\x00\x00\x00\x00\x00\x00'
|
||||
+ b'\x00\x00\x00\x00\x00\x00\x00\x12'
|
||||
+ ),
|
||||
+ ('too large bytes size',
|
||||
+ b'\x4f\x00\x23\x41\x08'
|
||||
+ b'\x00\x00\x00\x00\x00\x00\x01\x01'
|
||||
+ b'\x00\x00\x00\x00\x00\x00\x00\x01'
|
||||
+ b'\x00\x00\x00\x00\x00\x00\x00\x00'
|
||||
+ b'\x00\x00\x00\x00\x00\x00\x00\x0c'
|
||||
+ ),
|
||||
+ ('extremally large bytes size (32-bit)',
|
||||
+ b'\x4f\x02\x7f\xff\xff\xff\x41\x08'
|
||||
+ b'\x00\x00\x00\x00\x00\x00\x01\x01'
|
||||
+ b'\x00\x00\x00\x00\x00\x00\x00\x01'
|
||||
+ b'\x00\x00\x00\x00\x00\x00\x00\x00'
|
||||
+ b'\x00\x00\x00\x00\x00\x00\x00\x0f'
|
||||
+ ),
|
||||
+ ('extremally large bytes size (64-bit)',
|
||||
+ b'\x4f\x03\x00\x00\x00\xff\xff\xff\xff\xff\x41\x08'
|
||||
+ b'\x00\x00\x00\x00\x00\x00\x01\x01'
|
||||
+ b'\x00\x00\x00\x00\x00\x00\x00\x01'
|
||||
+ b'\x00\x00\x00\x00\x00\x00\x00\x00'
|
||||
+ b'\x00\x00\x00\x00\x00\x00\x00\x13'
|
||||
+ ),
|
||||
+ ('integer overflow in bytes size',
|
||||
+ b'\x4f\x03\xff\xff\xff\xff\xff\xff\xff\xff\x41\x08'
|
||||
+ b'\x00\x00\x00\x00\x00\x00\x01\x01'
|
||||
+ b'\x00\x00\x00\x00\x00\x00\x00\x01'
|
||||
+ b'\x00\x00\x00\x00\x00\x00\x00\x00'
|
||||
+ b'\x00\x00\x00\x00\x00\x00\x00\x13'
|
||||
+ ),
|
||||
+ ('too large ASCII size',
|
||||
+ b'\x5f\x00\x23\x41\x08'
|
||||
+ b'\x00\x00\x00\x00\x00\x00\x01\x01'
|
||||
+ b'\x00\x00\x00\x00\x00\x00\x00\x01'
|
||||
+ b'\x00\x00\x00\x00\x00\x00\x00\x00'
|
||||
+ b'\x00\x00\x00\x00\x00\x00\x00\x0c'
|
||||
+ ),
|
||||
+ ('extremally large ASCII size (32-bit)',
|
||||
+ b'\x5f\x02\x7f\xff\xff\xff\x41\x08'
|
||||
+ b'\x00\x00\x00\x00\x00\x00\x01\x01'
|
||||
+ b'\x00\x00\x00\x00\x00\x00\x00\x01'
|
||||
+ b'\x00\x00\x00\x00\x00\x00\x00\x00'
|
||||
+ b'\x00\x00\x00\x00\x00\x00\x00\x0f'
|
||||
+ ),
|
||||
+ ('extremally large ASCII size (64-bit)',
|
||||
+ b'\x5f\x03\x00\x00\x00\xff\xff\xff\xff\xff\x41\x08'
|
||||
+ b'\x00\x00\x00\x00\x00\x00\x01\x01'
|
||||
+ b'\x00\x00\x00\x00\x00\x00\x00\x01'
|
||||
+ b'\x00\x00\x00\x00\x00\x00\x00\x00'
|
||||
+ b'\x00\x00\x00\x00\x00\x00\x00\x13'
|
||||
+ ),
|
||||
+ ('integer overflow in ASCII size',
|
||||
+ b'\x5f\x03\xff\xff\xff\xff\xff\xff\xff\xff\x41\x08'
|
||||
+ b'\x00\x00\x00\x00\x00\x00\x01\x01'
|
||||
+ b'\x00\x00\x00\x00\x00\x00\x00\x01'
|
||||
+ b'\x00\x00\x00\x00\x00\x00\x00\x00'
|
||||
+ b'\x00\x00\x00\x00\x00\x00\x00\x13'
|
||||
+ ),
|
||||
+ ('invalid ASCII',
|
||||
+ b'\x51\xff\x08'
|
||||
+ b'\x00\x00\x00\x00\x00\x00\x01\x01'
|
||||
+ b'\x00\x00\x00\x00\x00\x00\x00\x01'
|
||||
+ b'\x00\x00\x00\x00\x00\x00\x00\x00'
|
||||
+ b'\x00\x00\x00\x00\x00\x00\x00\x0a'
|
||||
+ ),
|
||||
+ ('too large UTF-16 size',
|
||||
+ b'\x6f\x00\x13\x20\xac\x00\x08'
|
||||
+ b'\x00\x00\x00\x00\x00\x00\x01\x01'
|
||||
+ b'\x00\x00\x00\x00\x00\x00\x00\x01'
|
||||
+ b'\x00\x00\x00\x00\x00\x00\x00\x00'
|
||||
+ b'\x00\x00\x00\x00\x00\x00\x00\x0e'
|
||||
+ ),
|
||||
+ ('extremally large UTF-16 size (32-bit)',
|
||||
+ b'\x6f\x02\x4f\xff\xff\xff\x20\xac\x00\x08'
|
||||
+ b'\x00\x00\x00\x00\x00\x00\x01\x01'
|
||||
+ b'\x00\x00\x00\x00\x00\x00\x00\x01'
|
||||
+ b'\x00\x00\x00\x00\x00\x00\x00\x00'
|
||||
+ b'\x00\x00\x00\x00\x00\x00\x00\x11'
|
||||
+ ),
|
||||
+ ('extremally large UTF-16 size (64-bit)',
|
||||
+ b'\x6f\x03\x00\x00\x00\xff\xff\xff\xff\xff\x20\xac\x00\x08'
|
||||
+ b'\x00\x00\x00\x00\x00\x00\x01\x01'
|
||||
+ b'\x00\x00\x00\x00\x00\x00\x00\x01'
|
||||
+ b'\x00\x00\x00\x00\x00\x00\x00\x00'
|
||||
+ b'\x00\x00\x00\x00\x00\x00\x00\x15'
|
||||
+ ),
|
||||
+ ('integer overflow in UTF-16 size',
|
||||
+ b'\x6f\x03\xff\xff\xff\xff\xff\xff\xff\xff\x20\xac\x00\x08'
|
||||
+ b'\x00\x00\x00\x00\x00\x00\x01\x01'
|
||||
+ b'\x00\x00\x00\x00\x00\x00\x00\x01'
|
||||
+ b'\x00\x00\x00\x00\x00\x00\x00\x00'
|
||||
+ b'\x00\x00\x00\x00\x00\x00\x00\x15'
|
||||
+ ),
|
||||
+ ('invalid UTF-16',
|
||||
+ b'\x61\xd8\x00\x08'
|
||||
+ b'\x00\x00\x00\x00\x00\x00\x01\x01'
|
||||
+ b'\x00\x00\x00\x00\x00\x00\x00\x01'
|
||||
+ b'\x00\x00\x00\x00\x00\x00\x00\x00'
|
||||
+ b'\x00\x00\x00\x00\x00\x00\x00\x0b'
|
||||
+ ),
|
||||
+ ('non-hashable key',
|
||||
+ b'\xd1\x01\x01\xa0\x08\x0b'
|
||||
+ b'\x00\x00\x00\x00\x00\x00\x01\x01'
|
||||
+ b'\x00\x00\x00\x00\x00\x00\x00\x02'
|
||||
+ b'\x00\x00\x00\x00\x00\x00\x00\x00'
|
||||
+ b'\x00\x00\x00\x00\x00\x00\x00\x0c'
|
||||
+ ),
|
||||
+ ('too large datetime (datetime overflow)',
|
||||
+ b'\x33\x42\x50\x00\x00\x00\x00\x00\x00\x08'
|
||||
+ b'\x00\x00\x00\x00\x00\x00\x01\x01'
|
||||
+ b'\x00\x00\x00\x00\x00\x00\x00\x01'
|
||||
+ b'\x00\x00\x00\x00\x00\x00\x00\x00'
|
||||
+ b'\x00\x00\x00\x00\x00\x00\x00\x11'
|
||||
+ ),
|
||||
+ ('too large datetime (timedelta overflow)',
|
||||
+ b'\x33\x42\xe0\x00\x00\x00\x00\x00\x00\x08'
|
||||
+ b'\x00\x00\x00\x00\x00\x00\x01\x01'
|
||||
+ b'\x00\x00\x00\x00\x00\x00\x00\x01'
|
||||
+ b'\x00\x00\x00\x00\x00\x00\x00\x00'
|
||||
+ b'\x00\x00\x00\x00\x00\x00\x00\x11'
|
||||
+ ),
|
||||
+ ('invalid datetime (Infinity)',
|
||||
+ b'\x33\x7f\xf0\x00\x00\x00\x00\x00\x00\x08'
|
||||
+ b'\x00\x00\x00\x00\x00\x00\x01\x01'
|
||||
+ b'\x00\x00\x00\x00\x00\x00\x00\x01'
|
||||
+ b'\x00\x00\x00\x00\x00\x00\x00\x00'
|
||||
+ b'\x00\x00\x00\x00\x00\x00\x00\x11'
|
||||
+ ),
|
||||
+ ('invalid datetime (NaN)',
|
||||
+ b'\x33\x7f\xf8\x00\x00\x00\x00\x00\x00\x08'
|
||||
+ b'\x00\x00\x00\x00\x00\x00\x01\x01'
|
||||
+ b'\x00\x00\x00\x00\x00\x00\x00\x01'
|
||||
+ b'\x00\x00\x00\x00\x00\x00\x00\x00'
|
||||
+ b'\x00\x00\x00\x00\x00\x00\x00\x11'
|
||||
+ ),
|
||||
+]
|
||||
|
||||
class TestPlistlib(unittest.TestCase):
|
||||
|
||||
@@ -447,6 +726,21 @@ class TestPlistlib(unittest.TestCase):
|
||||
|
||||
class TestBinaryPlistlib(unittest.TestCase):
|
||||
|
||||
+ @staticmethod
|
||||
+ def decode(*objects, offset_size=1, ref_size=1):
|
||||
+ data = [b'bplist00']
|
||||
+ offset = 8
|
||||
+ offsets = []
|
||||
+ for x in objects:
|
||||
+ offsets.append(offset.to_bytes(offset_size, 'big'))
|
||||
+ data.append(x)
|
||||
+ offset += len(x)
|
||||
+ tail = struct.pack('>6xBBQQQ', offset_size, ref_size,
|
||||
+ len(objects), 0, offset)
|
||||
+ data.extend(offsets)
|
||||
+ data.append(tail)
|
||||
+ return plistlib.loads(b''.join(data), fmt=plistlib.FMT_BINARY)
|
||||
+
|
||||
def test_nonstandard_refs_size(self):
|
||||
# Issue #21538: Refs and offsets are 24-bit integers
|
||||
data = (b'bplist00'
|
||||
@@ -461,7 +755,7 @@ class TestBinaryPlistlib(unittest.TestCase):
|
||||
|
||||
def test_dump_duplicates(self):
|
||||
# Test effectiveness of saving duplicated objects
|
||||
- for x in (None, False, True, 12345, 123.45, 'abcde', b'abcde',
|
||||
+ for x in (None, False, True, 12345, 123.45, 'abcde', 'абвгд', b'abcde',
|
||||
datetime.datetime(2004, 10, 26, 10, 33, 33),
|
||||
plistlib.Data(b'abcde'), bytearray(b'abcde'),
|
||||
[12, 345], (12, 345), {'12': 345}):
|
||||
@@ -500,6 +794,20 @@ class TestBinaryPlistlib(unittest.TestCase):
|
||||
b = plistlib.loads(plistlib.dumps(a, fmt=plistlib.FMT_BINARY))
|
||||
self.assertIs(b['x'], b)
|
||||
|
||||
+ def test_deep_nesting(self):
|
||||
+ for N in [300, 100000]:
|
||||
+ chunks = [b'\xa1' + (i + 1).to_bytes(4, 'big') for i in range(N)]
|
||||
+ try:
|
||||
+ result = self.decode(*chunks, b'\x54seed', offset_size=4, ref_size=4)
|
||||
+ except RecursionError:
|
||||
+ pass
|
||||
+ else:
|
||||
+ for i in range(N):
|
||||
+ self.assertIsInstance(result, list)
|
||||
+ self.assertEqual(len(result), 1)
|
||||
+ result = result[0]
|
||||
+ self.assertEqual(result, 'seed')
|
||||
+
|
||||
def test_large_timestamp(self):
|
||||
# Issue #26709: 32-bit timestamp out of range
|
||||
for ts in -2**31-1, 2**31:
|
||||
@@ -509,55 +817,37 @@ class TestBinaryPlistlib(unittest.TestCase):
|
||||
data = plistlib.dumps(d, fmt=plistlib.FMT_BINARY)
|
||||
self.assertEqual(plistlib.loads(data), d)
|
||||
|
||||
+ def test_load_singletons(self):
|
||||
+ self.assertIs(self.decode(b'\x00'), None)
|
||||
+ self.assertIs(self.decode(b'\x08'), False)
|
||||
+ self.assertIs(self.decode(b'\x09'), True)
|
||||
+ self.assertEqual(self.decode(b'\x0f'), b'')
|
||||
+
|
||||
+ def test_load_int(self):
|
||||
+ self.assertEqual(self.decode(b'\x10\x00'), 0)
|
||||
+ self.assertEqual(self.decode(b'\x10\xfe'), 0xfe)
|
||||
+ self.assertEqual(self.decode(b'\x11\xfe\xdc'), 0xfedc)
|
||||
+ self.assertEqual(self.decode(b'\x12\xfe\xdc\xba\x98'), 0xfedcba98)
|
||||
+ self.assertEqual(self.decode(b'\x13\x01\x23\x45\x67\x89\xab\xcd\xef'),
|
||||
+ 0x0123456789abcdef)
|
||||
+ self.assertEqual(self.decode(b'\x13\xfe\xdc\xba\x98\x76\x54\x32\x10'),
|
||||
+ -0x123456789abcdf0)
|
||||
+
|
||||
+ def test_unsupported(self):
|
||||
+ unsupported = [*range(1, 8), *range(10, 15),
|
||||
+ 0x20, 0x21, *range(0x24, 0x33), *range(0x34, 0x40)]
|
||||
+ for i in [0x70, 0x90, 0xb0, 0xc0, 0xe0, 0xf0]:
|
||||
+ unsupported.extend(i + j for j in range(16))
|
||||
+ for token in unsupported:
|
||||
+ with self.subTest(f'token {token:02x}'):
|
||||
+ with self.assertRaises(plistlib.InvalidFileException):
|
||||
+ self.decode(bytes([token]) + b'\x00'*16)
|
||||
+
|
||||
def test_invalid_binary(self):
|
||||
- for data in [
|
||||
- # too short data
|
||||
- b'',
|
||||
- # too large offset_table_offset and nonstandard offset_size
|
||||
- b'\x00\x08'
|
||||
- b'\x00\x00\x00\x00\x00\x00\x03\x01'
|
||||
- b'\x00\x00\x00\x00\x00\x00\x00\x01'
|
||||
- b'\x00\x00\x00\x00\x00\x00\x00\x00'
|
||||
- b'\x00\x00\x00\x00\x00\x00\x00\x2a',
|
||||
- # integer overflow in offset_table_offset
|
||||
- b'\x00\x08'
|
||||
- b'\x00\x00\x00\x00\x00\x00\x01\x01'
|
||||
- b'\x00\x00\x00\x00\x00\x00\x00\x01'
|
||||
- b'\x00\x00\x00\x00\x00\x00\x00\x00'
|
||||
- b'\xff\xff\xff\xff\xff\xff\xff\xff',
|
||||
- # offset_size = 0
|
||||
- b'\x00\x08'
|
||||
- b'\x00\x00\x00\x00\x00\x00\x00\x01'
|
||||
- b'\x00\x00\x00\x00\x00\x00\x00\x01'
|
||||
- b'\x00\x00\x00\x00\x00\x00\x00\x00'
|
||||
- b'\x00\x00\x00\x00\x00\x00\x00\x09',
|
||||
- # ref_size = 0
|
||||
- b'\xa1\x01\x00\x08\x0a'
|
||||
- b'\x00\x00\x00\x00\x00\x00\x01\x00'
|
||||
- b'\x00\x00\x00\x00\x00\x00\x00\x02'
|
||||
- b'\x00\x00\x00\x00\x00\x00\x00\x00'
|
||||
- b'\x00\x00\x00\x00\x00\x00\x00\x0b',
|
||||
- # integer overflow in offset
|
||||
- b'\x00\xff\xff\xff\xff\xff\xff\xff\xff'
|
||||
- b'\x00\x00\x00\x00\x00\x00\x08\x01'
|
||||
- b'\x00\x00\x00\x00\x00\x00\x00\x01'
|
||||
- b'\x00\x00\x00\x00\x00\x00\x00\x00'
|
||||
- b'\x00\x00\x00\x00\x00\x00\x00\x09',
|
||||
- # invalid ASCII
|
||||
- b'\x51\xff\x08'
|
||||
- b'\x00\x00\x00\x00\x00\x00\x01\x01'
|
||||
- b'\x00\x00\x00\x00\x00\x00\x00\x01'
|
||||
- b'\x00\x00\x00\x00\x00\x00\x00\x00'
|
||||
- b'\x00\x00\x00\x00\x00\x00\x00\x0a',
|
||||
- # invalid UTF-16
|
||||
- b'\x61\xd8\x00\x08'
|
||||
- b'\x00\x00\x00\x00\x00\x00\x01\x01'
|
||||
- b'\x00\x00\x00\x00\x00\x00\x00\x01'
|
||||
- b'\x00\x00\x00\x00\x00\x00\x00\x00'
|
||||
- b'\x00\x00\x00\x00\x00\x00\x00\x0b',
|
||||
- ]:
|
||||
- with self.assertRaises(plistlib.InvalidFileException):
|
||||
- plistlib.loads(b'bplist00' + data, fmt=plistlib.FMT_BINARY)
|
||||
+ for name, data in INVALID_BINARY_PLISTS:
|
||||
+ with self.subTest(name):
|
||||
+ with self.assertRaises(plistlib.InvalidFileException):
|
||||
+ plistlib.loads(b'bplist00' + data, fmt=plistlib.FMT_BINARY)
|
||||
|
||||
|
||||
class TestPlistlibDeprecated(unittest.TestCase):
|
||||
@@ -655,9 +945,5 @@ class MiscTestCase(unittest.TestCase):
|
||||
support.check__all__(self, plistlib, blacklist=blacklist)
|
||||
|
||||
|
||||
-def test_main():
|
||||
- support.run_unittest(TestPlistlib, TestPlistlibDeprecated, MiscTestCase)
|
||||
-
|
||||
-
|
||||
if __name__ == '__main__':
|
||||
- test_main()
|
||||
+ unittest.main()
|
||||
diff --git a/Misc/NEWS.d/next/Library/2020-10-23-19-20-14.bpo-42103.C5obK2.rst b/Misc/NEWS.d/next/Library/2020-10-23-19-20-14.bpo-42103.C5obK2.rst
|
||||
new file mode 100644
|
||||
index 0000000..4eb694c
|
||||
--- /dev/null
|
||||
+++ b/Misc/NEWS.d/next/Library/2020-10-23-19-20-14.bpo-42103.C5obK2.rst
|
||||
@@ -0,0 +1,3 @@
|
||||
+:exc:`~plistlib.InvalidFileException` and :exc:`RecursionError` are now
|
||||
+the only errors caused by loading malformed binary Plist file (previously
|
||||
+ValueError and TypeError could be raised in some specific cases).
|
||||
diff --git a/Misc/NEWS.d/next/Security/2020-10-23-19-19-30.bpo-42103.cILT66.rst b/Misc/NEWS.d/next/Security/2020-10-23-19-19-30.bpo-42103.cILT66.rst
|
||||
new file mode 100644
|
||||
index 0000000..15d7b65
|
||||
--- /dev/null
|
||||
+++ b/Misc/NEWS.d/next/Security/2020-10-23-19-19-30.bpo-42103.cILT66.rst
|
||||
@@ -0,0 +1,2 @@
|
||||
+Prevented potential DoS attack via CPU and RAM exhaustion when processing
|
||||
+malformed Apple Property List files in binary format.
|
||||
--
|
||||
2.43.0
|
||||
|
|
@ -0,0 +1,88 @@
|
|||
From 0d02ff99721f7650e39ba4c7d8fe06f412bbb591 Mon Sep 17 00:00:00 2001
|
||||
From: Victor Stinner <vstinner@python.org>
|
||||
Date: Wed, 13 Dec 2023 11:50:26 +0100
|
||||
Subject: [PATCH] bpo-46623: Skip two test_zlib tests on s390x (GH-31096)
|
||||
|
||||
Skip test_pair() and test_speech128() of test_zlib on s390x since
|
||||
they fail if zlib uses the s390x hardware accelerator.
|
||||
---
|
||||
Lib/test/test_zlib.py | 32 +++++++++++++++++++
|
||||
.../2022-02-03-09-45-26.bpo-46623.vxzuhV.rst | 2 ++
|
||||
2 files changed, 34 insertions(+)
|
||||
create mode 100644 Misc/NEWS.d/next/Tests/2022-02-03-09-45-26.bpo-46623.vxzuhV.rst
|
||||
|
||||
diff --git a/Lib/test/test_zlib.py b/Lib/test/test_zlib.py
|
||||
index b7170b4..770a425 100644
|
||||
--- a/Lib/test/test_zlib.py
|
||||
+++ b/Lib/test/test_zlib.py
|
||||
@@ -1,6 +1,7 @@
|
||||
import unittest
|
||||
from test import support
|
||||
import binascii
|
||||
+import os
|
||||
import pickle
|
||||
import random
|
||||
import sys
|
||||
@@ -15,6 +16,35 @@ requires_Decompress_copy = unittest.skipUnless(
|
||||
hasattr(zlib.decompressobj(), "copy"),
|
||||
'requires Decompress.copy()')
|
||||
|
||||
+# bpo-46623: On s390x, when a hardware accelerator is used, using different
|
||||
+# ways to compress data with zlib can produce different compressed data.
|
||||
+# Simplified test_pair() code:
|
||||
+#
|
||||
+# def func1(data):
|
||||
+# return zlib.compress(data)
|
||||
+#
|
||||
+# def func2(data)
|
||||
+# co = zlib.compressobj()
|
||||
+# x1 = co.compress(data)
|
||||
+# x2 = co.flush()
|
||||
+# return x1 + x2
|
||||
+#
|
||||
+# On s390x if zlib uses a hardware accelerator, func1() creates a single
|
||||
+# "final" compressed block whereas func2() produces 3 compressed blocks (the
|
||||
+# last one is a final block). On other platforms with no accelerator, func1()
|
||||
+# and func2() produce the same compressed data made of a single (final)
|
||||
+# compressed block.
|
||||
+#
|
||||
+# Only the compressed data is different, the decompression returns the original
|
||||
+# data:
|
||||
+#
|
||||
+# zlib.decompress(func1(data)) == zlib.decompress(func2(data)) == data
|
||||
+#
|
||||
+# Make the assumption that s390x always has an accelerator to simplify the skip
|
||||
+# condition. Windows doesn't have os.uname() but it doesn't support s390x.
|
||||
+skip_on_s390x = unittest.skipIf(hasattr(os, 'uname') and os.uname().machine == 's390x',
|
||||
+ 'skipped on s390x')
|
||||
+
|
||||
|
||||
class VersionTestCase(unittest.TestCase):
|
||||
|
||||
@@ -174,6 +204,7 @@ class CompressTestCase(BaseCompressTestCase, unittest.TestCase):
|
||||
bufsize=zlib.DEF_BUF_SIZE),
|
||||
HAMLET_SCENE)
|
||||
|
||||
+ @skip_on_s390x
|
||||
def test_speech128(self):
|
||||
# compress more data
|
||||
data = HAMLET_SCENE * 128
|
||||
@@ -225,6 +256,7 @@ class CompressTestCase(BaseCompressTestCase, unittest.TestCase):
|
||||
|
||||
class CompressObjectTestCase(BaseCompressTestCase, unittest.TestCase):
|
||||
# Test compression object
|
||||
+ @skip_on_s390x
|
||||
def test_pair(self):
|
||||
# straightforward compress/decompress objects
|
||||
datasrc = HAMLET_SCENE * 128
|
||||
diff --git a/Misc/NEWS.d/next/Tests/2022-02-03-09-45-26.bpo-46623.vxzuhV.rst b/Misc/NEWS.d/next/Tests/2022-02-03-09-45-26.bpo-46623.vxzuhV.rst
|
||||
new file mode 100644
|
||||
index 0000000..be085c0
|
||||
--- /dev/null
|
||||
+++ b/Misc/NEWS.d/next/Tests/2022-02-03-09-45-26.bpo-46623.vxzuhV.rst
|
||||
@@ -0,0 +1,2 @@
|
||||
+Skip test_pair() and test_speech128() of test_zlib on s390x since they fail
|
||||
+if zlib uses the s390x hardware accelerator. Patch by Victor Stinner.
|
||||
--
|
||||
2.43.0
|
||||
|
|
@ -0,0 +1,750 @@
|
|||
From 0000000000000000000000000000000000000000 Mon Sep 17 00:00:00 2001
|
||||
From: Victor Stinner <vstinner@python.org>
|
||||
Date: Fri, 15 Dec 2023 16:10:40 +0100
|
||||
Subject: [PATCH] 00415: [CVE-2023-27043] gh-102988: Reject malformed addresses
|
||||
in email.parseaddr() (#111116)
|
||||
|
||||
Detect email address parsing errors and return empty tuple to
|
||||
indicate the parsing error (old API). Add an optional 'strict'
|
||||
parameter to getaddresses() and parseaddr() functions. Patch by
|
||||
Thomas Dwyer.
|
||||
|
||||
Co-Authored-By: Thomas Dwyer <github@tomd.tel>
|
||||
---
|
||||
Doc/library/email.utils.rst | 19 +-
|
||||
Lib/email/utils.py | 151 ++++++++++++-
|
||||
Lib/test/test_email/test_email.py | 204 +++++++++++++++++-
|
||||
...-10-20-15-28-08.gh-issue-102988.dStNO7.rst | 8 +
|
||||
4 files changed, 361 insertions(+), 21 deletions(-)
|
||||
create mode 100644 Misc/NEWS.d/next/Library/2023-10-20-15-28-08.gh-issue-102988.dStNO7.rst
|
||||
|
||||
diff --git a/Doc/library/email.utils.rst b/Doc/library/email.utils.rst
|
||||
index 63fae2ab84..d1e1898591 100644
|
||||
--- a/Doc/library/email.utils.rst
|
||||
+++ b/Doc/library/email.utils.rst
|
||||
@@ -60,13 +60,18 @@ of the new API.
|
||||
begins with angle brackets, they are stripped off.
|
||||
|
||||
|
||||
-.. function:: parseaddr(address)
|
||||
+.. function:: parseaddr(address, *, strict=True)
|
||||
|
||||
Parse address -- which should be the value of some address-containing field such
|
||||
as :mailheader:`To` or :mailheader:`Cc` -- into its constituent *realname* and
|
||||
*email address* parts. Returns a tuple of that information, unless the parse
|
||||
fails, in which case a 2-tuple of ``('', '')`` is returned.
|
||||
|
||||
+ If *strict* is true, use a strict parser which rejects malformed inputs.
|
||||
+
|
||||
+ .. versionchanged:: 3.13
|
||||
+ Add *strict* optional parameter and reject malformed inputs by default.
|
||||
+
|
||||
|
||||
.. function:: formataddr(pair, charset='utf-8')
|
||||
|
||||
@@ -84,12 +89,15 @@ of the new API.
|
||||
Added the *charset* option.
|
||||
|
||||
|
||||
-.. function:: getaddresses(fieldvalues)
|
||||
+.. function:: getaddresses(fieldvalues, *, strict=True)
|
||||
|
||||
This method returns a list of 2-tuples of the form returned by ``parseaddr()``.
|
||||
*fieldvalues* is a sequence of header field values as might be returned by
|
||||
- :meth:`Message.get_all <email.message.Message.get_all>`. Here's a simple
|
||||
- example that gets all the recipients of a message::
|
||||
+ :meth:`Message.get_all <email.message.Message.get_all>`.
|
||||
+
|
||||
+ If *strict* is true, use a strict parser which rejects malformed inputs.
|
||||
+
|
||||
+ Here's a simple example that gets all the recipients of a message::
|
||||
|
||||
from email.utils import getaddresses
|
||||
|
||||
@@ -99,6 +107,9 @@ of the new API.
|
||||
resent_ccs = msg.get_all('resent-cc', [])
|
||||
all_recipients = getaddresses(tos + ccs + resent_tos + resent_ccs)
|
||||
|
||||
+ .. versionchanged:: 3.13
|
||||
+ Add *strict* optional parameter and reject malformed inputs by default.
|
||||
+
|
||||
|
||||
.. function:: parsedate(date)
|
||||
|
||||
diff --git a/Lib/email/utils.py b/Lib/email/utils.py
|
||||
index 39c2240607..f83b7e5d7e 100644
|
||||
--- a/Lib/email/utils.py
|
||||
+++ b/Lib/email/utils.py
|
||||
@@ -48,6 +48,7 @@ TICK = "'"
|
||||
specialsre = re.compile(r'[][\\()<>@,:;".]')
|
||||
escapesre = re.compile(r'[\\"]')
|
||||
|
||||
+
|
||||
def _has_surrogates(s):
|
||||
"""Return True if s contains surrogate-escaped binary data."""
|
||||
# This check is based on the fact that unless there are surrogates, utf8
|
||||
@@ -106,12 +107,127 @@ def formataddr(pair, charset='utf-8'):
|
||||
return address
|
||||
|
||||
|
||||
+def _iter_escaped_chars(addr):
|
||||
+ pos = 0
|
||||
+ escape = False
|
||||
+ for pos, ch in enumerate(addr):
|
||||
+ if escape:
|
||||
+ yield (pos, '\\' + ch)
|
||||
+ escape = False
|
||||
+ elif ch == '\\':
|
||||
+ escape = True
|
||||
+ else:
|
||||
+ yield (pos, ch)
|
||||
+ if escape:
|
||||
+ yield (pos, '\\')
|
||||
|
||||
-def getaddresses(fieldvalues):
|
||||
- """Return a list of (REALNAME, EMAIL) for each fieldvalue."""
|
||||
- all = COMMASPACE.join(fieldvalues)
|
||||
- a = _AddressList(all)
|
||||
- return a.addresslist
|
||||
+
|
||||
+def _strip_quoted_realnames(addr):
|
||||
+ """Strip real names between quotes."""
|
||||
+ if '"' not in addr:
|
||||
+ # Fast path
|
||||
+ return addr
|
||||
+
|
||||
+ start = 0
|
||||
+ open_pos = None
|
||||
+ result = []
|
||||
+ for pos, ch in _iter_escaped_chars(addr):
|
||||
+ if ch == '"':
|
||||
+ if open_pos is None:
|
||||
+ open_pos = pos
|
||||
+ else:
|
||||
+ if start != open_pos:
|
||||
+ result.append(addr[start:open_pos])
|
||||
+ start = pos + 1
|
||||
+ open_pos = None
|
||||
+
|
||||
+ if start < len(addr):
|
||||
+ result.append(addr[start:])
|
||||
+
|
||||
+ return ''.join(result)
|
||||
+
|
||||
+
|
||||
+supports_strict_parsing = True
|
||||
+
|
||||
+def getaddresses(fieldvalues, *, strict=True):
|
||||
+ """Return a list of (REALNAME, EMAIL) or ('','') for each fieldvalue.
|
||||
+
|
||||
+ When parsing fails for a fieldvalue, a 2-tuple of ('', '') is returned in
|
||||
+ its place.
|
||||
+
|
||||
+ If strict is true, use a strict parser which rejects malformed inputs.
|
||||
+ """
|
||||
+
|
||||
+ # If strict is true, if the resulting list of parsed addresses is greater
|
||||
+ # than the number of fieldvalues in the input list, a parsing error has
|
||||
+ # occurred and consequently a list containing a single empty 2-tuple [('',
|
||||
+ # '')] is returned in its place. This is done to avoid invalid output.
|
||||
+ #
|
||||
+ # Malformed input: getaddresses(['alice@example.com <bob@example.com>'])
|
||||
+ # Invalid output: [('', 'alice@example.com'), ('', 'bob@example.com')]
|
||||
+ # Safe output: [('', '')]
|
||||
+
|
||||
+ if not strict:
|
||||
+ all = COMMASPACE.join(str(v) for v in fieldvalues)
|
||||
+ a = _AddressList(all)
|
||||
+ return a.addresslist
|
||||
+
|
||||
+ fieldvalues = [str(v) for v in fieldvalues]
|
||||
+ fieldvalues = _pre_parse_validation(fieldvalues)
|
||||
+ addr = COMMASPACE.join(fieldvalues)
|
||||
+ a = _AddressList(addr)
|
||||
+ result = _post_parse_validation(a.addresslist)
|
||||
+
|
||||
+ # Treat output as invalid if the number of addresses is not equal to the
|
||||
+ # expected number of addresses.
|
||||
+ n = 0
|
||||
+ for v in fieldvalues:
|
||||
+ # When a comma is used in the Real Name part it is not a deliminator.
|
||||
+ # So strip those out before counting the commas.
|
||||
+ v = _strip_quoted_realnames(v)
|
||||
+ # Expected number of addresses: 1 + number of commas
|
||||
+ n += 1 + v.count(',')
|
||||
+ if len(result) != n:
|
||||
+ return [('', '')]
|
||||
+
|
||||
+ return result
|
||||
+
|
||||
+
|
||||
+def _check_parenthesis(addr):
|
||||
+ # Ignore parenthesis in quoted real names.
|
||||
+ addr = _strip_quoted_realnames(addr)
|
||||
+
|
||||
+ opens = 0
|
||||
+ for pos, ch in _iter_escaped_chars(addr):
|
||||
+ if ch == '(':
|
||||
+ opens += 1
|
||||
+ elif ch == ')':
|
||||
+ opens -= 1
|
||||
+ if opens < 0:
|
||||
+ return False
|
||||
+ return (opens == 0)
|
||||
+
|
||||
+
|
||||
+def _pre_parse_validation(email_header_fields):
|
||||
+ accepted_values = []
|
||||
+ for v in email_header_fields:
|
||||
+ if not _check_parenthesis(v):
|
||||
+ v = "('', '')"
|
||||
+ accepted_values.append(v)
|
||||
+
|
||||
+ return accepted_values
|
||||
+
|
||||
+
|
||||
+def _post_parse_validation(parsed_email_header_tuples):
|
||||
+ accepted_values = []
|
||||
+ # The parser would have parsed a correctly formatted domain-literal
|
||||
+ # The existence of an [ after parsing indicates a parsing failure
|
||||
+ for v in parsed_email_header_tuples:
|
||||
+ if '[' in v[1]:
|
||||
+ v = ('', '')
|
||||
+ accepted_values.append(v)
|
||||
+
|
||||
+ return accepted_values
|
||||
|
||||
|
||||
|
||||
@@ -214,16 +330,33 @@ def parsedate_to_datetime(data):
|
||||
tzinfo=datetime.timezone(datetime.timedelta(seconds=tz)))
|
||||
|
||||
|
||||
-def parseaddr(addr):
|
||||
+def parseaddr(addr, *, strict=True):
|
||||
"""
|
||||
Parse addr into its constituent realname and email address parts.
|
||||
|
||||
Return a tuple of realname and email address, unless the parse fails, in
|
||||
which case return a 2-tuple of ('', '').
|
||||
+
|
||||
+ If strict is True, use a strict parser which rejects malformed inputs.
|
||||
"""
|
||||
- addrs = _AddressList(addr).addresslist
|
||||
- if not addrs:
|
||||
- return '', ''
|
||||
+ if not strict:
|
||||
+ addrs = _AddressList(addr).addresslist
|
||||
+ if not addrs:
|
||||
+ return ('', '')
|
||||
+ return addrs[0]
|
||||
+
|
||||
+ if isinstance(addr, list):
|
||||
+ addr = addr[0]
|
||||
+
|
||||
+ if not isinstance(addr, str):
|
||||
+ return ('', '')
|
||||
+
|
||||
+ addr = _pre_parse_validation([addr])[0]
|
||||
+ addrs = _post_parse_validation(_AddressList(addr).addresslist)
|
||||
+
|
||||
+ if not addrs or len(addrs) > 1:
|
||||
+ return ('', '')
|
||||
+
|
||||
return addrs[0]
|
||||
|
||||
|
||||
diff --git a/Lib/test/test_email/test_email.py b/Lib/test/test_email/test_email.py
|
||||
index e4e40b612f..ce36efc1b1 100644
|
||||
--- a/Lib/test/test_email/test_email.py
|
||||
+++ b/Lib/test/test_email/test_email.py
|
||||
@@ -19,6 +19,7 @@ except ImportError:
|
||||
|
||||
import email
|
||||
import email.policy
|
||||
+import email.utils
|
||||
|
||||
from email.charset import Charset
|
||||
from email.header import Header, decode_header, make_header
|
||||
@@ -3207,15 +3208,154 @@ Foo
|
||||
[('Al Person', 'aperson@dom.ain'),
|
||||
('Bud Person', 'bperson@dom.ain')])
|
||||
|
||||
+ def test_getaddresses_comma_in_name(self):
|
||||
+ """GH-106669 regression test."""
|
||||
+ self.assertEqual(
|
||||
+ utils.getaddresses(
|
||||
+ [
|
||||
+ '"Bud, Person" <bperson@dom.ain>',
|
||||
+ 'aperson@dom.ain (Al Person)',
|
||||
+ '"Mariusz Felisiak" <to@example.com>',
|
||||
+ ]
|
||||
+ ),
|
||||
+ [
|
||||
+ ('Bud, Person', 'bperson@dom.ain'),
|
||||
+ ('Al Person', 'aperson@dom.ain'),
|
||||
+ ('Mariusz Felisiak', 'to@example.com'),
|
||||
+ ],
|
||||
+ )
|
||||
+
|
||||
+ def test_parsing_errors(self):
|
||||
+ """Test for parsing errors from CVE-2023-27043 and CVE-2019-16056"""
|
||||
+ alice = 'alice@example.org'
|
||||
+ bob = 'bob@example.com'
|
||||
+ empty = ('', '')
|
||||
+
|
||||
+ # Test utils.getaddresses() and utils.parseaddr() on malformed email
|
||||
+ # addresses: default behavior (strict=True) rejects malformed address,
|
||||
+ # and strict=False which tolerates malformed address.
|
||||
+ for invalid_separator, expected_non_strict in (
|
||||
+ ('(', [(f'<{bob}>', alice)]),
|
||||
+ (')', [('', alice), empty, ('', bob)]),
|
||||
+ ('<', [('', alice), empty, ('', bob), empty]),
|
||||
+ ('>', [('', alice), empty, ('', bob)]),
|
||||
+ ('[', [('', f'{alice}[<{bob}>]')]),
|
||||
+ (']', [('', alice), empty, ('', bob)]),
|
||||
+ ('@', [empty, empty, ('', bob)]),
|
||||
+ (';', [('', alice), empty, ('', bob)]),
|
||||
+ (':', [('', alice), ('', bob)]),
|
||||
+ ('.', [('', alice + '.'), ('', bob)]),
|
||||
+ ('"', [('', alice), ('', f'<{bob}>')]),
|
||||
+ ):
|
||||
+ address = f'{alice}{invalid_separator}<{bob}>'
|
||||
+ with self.subTest(address=address):
|
||||
+ self.assertEqual(utils.getaddresses([address]),
|
||||
+ [empty])
|
||||
+ self.assertEqual(utils.getaddresses([address], strict=False),
|
||||
+ expected_non_strict)
|
||||
+
|
||||
+ self.assertEqual(utils.parseaddr([address]),
|
||||
+ empty)
|
||||
+ self.assertEqual(utils.parseaddr([address], strict=False),
|
||||
+ ('', address))
|
||||
+
|
||||
+ # Comma (',') is treated differently depending on strict parameter.
|
||||
+ # Comma without quotes.
|
||||
+ address = f'{alice},<{bob}>'
|
||||
+ self.assertEqual(utils.getaddresses([address]),
|
||||
+ [('', alice), ('', bob)])
|
||||
+ self.assertEqual(utils.getaddresses([address], strict=False),
|
||||
+ [('', alice), ('', bob)])
|
||||
+ self.assertEqual(utils.parseaddr([address]),
|
||||
+ empty)
|
||||
+ self.assertEqual(utils.parseaddr([address], strict=False),
|
||||
+ ('', address))
|
||||
+
|
||||
+ # Real name between quotes containing comma.
|
||||
+ address = '"Alice, alice@example.org" <bob@example.com>'
|
||||
+ expected_strict = ('Alice, alice@example.org', 'bob@example.com')
|
||||
+ self.assertEqual(utils.getaddresses([address]), [expected_strict])
|
||||
+ self.assertEqual(utils.getaddresses([address], strict=False), [expected_strict])
|
||||
+ self.assertEqual(utils.parseaddr([address]), expected_strict)
|
||||
+ self.assertEqual(utils.parseaddr([address], strict=False),
|
||||
+ ('', address))
|
||||
+
|
||||
+ # Valid parenthesis in comments.
|
||||
+ address = 'alice@example.org (Alice)'
|
||||
+ expected_strict = ('Alice', 'alice@example.org')
|
||||
+ self.assertEqual(utils.getaddresses([address]), [expected_strict])
|
||||
+ self.assertEqual(utils.getaddresses([address], strict=False), [expected_strict])
|
||||
+ self.assertEqual(utils.parseaddr([address]), expected_strict)
|
||||
+ self.assertEqual(utils.parseaddr([address], strict=False),
|
||||
+ ('', address))
|
||||
+
|
||||
+ # Invalid parenthesis in comments.
|
||||
+ address = 'alice@example.org )Alice('
|
||||
+ self.assertEqual(utils.getaddresses([address]), [empty])
|
||||
+ self.assertEqual(utils.getaddresses([address], strict=False),
|
||||
+ [('', 'alice@example.org'), ('', ''), ('', 'Alice')])
|
||||
+ self.assertEqual(utils.parseaddr([address]), empty)
|
||||
+ self.assertEqual(utils.parseaddr([address], strict=False),
|
||||
+ ('', address))
|
||||
+
|
||||
+ # Two addresses with quotes separated by comma.
|
||||
+ address = '"Jane Doe" <jane@example.net>, "John Doe" <john@example.net>'
|
||||
+ self.assertEqual(utils.getaddresses([address]),
|
||||
+ [('Jane Doe', 'jane@example.net'),
|
||||
+ ('John Doe', 'john@example.net')])
|
||||
+ self.assertEqual(utils.getaddresses([address], strict=False),
|
||||
+ [('Jane Doe', 'jane@example.net'),
|
||||
+ ('John Doe', 'john@example.net')])
|
||||
+ self.assertEqual(utils.parseaddr([address]), empty)
|
||||
+ self.assertEqual(utils.parseaddr([address], strict=False),
|
||||
+ ('', address))
|
||||
+
|
||||
+ # Test email.utils.supports_strict_parsing attribute
|
||||
+ self.assertEqual(email.utils.supports_strict_parsing, True)
|
||||
+
|
||||
def test_getaddresses_nasty(self):
|
||||
- eq = self.assertEqual
|
||||
- eq(utils.getaddresses(['foo: ;']), [('', '')])
|
||||
- eq(utils.getaddresses(
|
||||
- ['[]*-- =~$']),
|
||||
- [('', ''), ('', ''), ('', '*--')])
|
||||
- eq(utils.getaddresses(
|
||||
- ['foo: ;', '"Jason R. Mastaler" <jason@dom.ain>']),
|
||||
- [('', ''), ('Jason R. Mastaler', 'jason@dom.ain')])
|
||||
+ for addresses, expected in (
|
||||
+ (['"Sürname, Firstname" <to@example.com>'],
|
||||
+ [('Sürname, Firstname', 'to@example.com')]),
|
||||
+
|
||||
+ (['foo: ;'],
|
||||
+ [('', '')]),
|
||||
+
|
||||
+ (['foo: ;', '"Jason R. Mastaler" <jason@dom.ain>'],
|
||||
+ [('', ''), ('Jason R. Mastaler', 'jason@dom.ain')]),
|
||||
+
|
||||
+ ([r'Pete(A nice \) chap) <pete(his account)@silly.test(his host)>'],
|
||||
+ [('Pete (A nice ) chap his account his host)', 'pete@silly.test')]),
|
||||
+
|
||||
+ (['(Empty list)(start)Undisclosed recipients :(nobody(I know))'],
|
||||
+ [('', '')]),
|
||||
+
|
||||
+ (['Mary <@machine.tld:mary@example.net>, , jdoe@test . example'],
|
||||
+ [('Mary', 'mary@example.net'), ('', ''), ('', 'jdoe@test.example')]),
|
||||
+
|
||||
+ (['John Doe <jdoe@machine(comment). example>'],
|
||||
+ [('John Doe (comment)', 'jdoe@machine.example')]),
|
||||
+
|
||||
+ (['"Mary Smith: Personal Account" <smith@home.example>'],
|
||||
+ [('Mary Smith: Personal Account', 'smith@home.example')]),
|
||||
+
|
||||
+ (['Undisclosed recipients:;'],
|
||||
+ [('', '')]),
|
||||
+
|
||||
+ ([r'<boss@nil.test>, "Giant; \"Big\" Box" <bob@example.net>'],
|
||||
+ [('', 'boss@nil.test'), ('Giant; "Big" Box', 'bob@example.net')]),
|
||||
+ ):
|
||||
+ with self.subTest(addresses=addresses):
|
||||
+ self.assertEqual(utils.getaddresses(addresses),
|
||||
+ expected)
|
||||
+ self.assertEqual(utils.getaddresses(addresses, strict=False),
|
||||
+ expected)
|
||||
+
|
||||
+ addresses = ['[]*-- =~$']
|
||||
+ self.assertEqual(utils.getaddresses(addresses),
|
||||
+ [('', '')])
|
||||
+ self.assertEqual(utils.getaddresses(addresses, strict=False),
|
||||
+ [('', ''), ('', ''), ('', '*--')])
|
||||
|
||||
def test_getaddresses_embedded_comment(self):
|
||||
"""Test proper handling of a nested comment"""
|
||||
@@ -3397,6 +3537,54 @@ multipart/report
|
||||
m = cls(*constructor, policy=email.policy.default)
|
||||
self.assertIs(m.policy, email.policy.default)
|
||||
|
||||
+ def test_iter_escaped_chars(self):
|
||||
+ self.assertEqual(list(utils._iter_escaped_chars(r'a\\b\"c\\"d')),
|
||||
+ [(0, 'a'),
|
||||
+ (2, '\\\\'),
|
||||
+ (3, 'b'),
|
||||
+ (5, '\\"'),
|
||||
+ (6, 'c'),
|
||||
+ (8, '\\\\'),
|
||||
+ (9, '"'),
|
||||
+ (10, 'd')])
|
||||
+ self.assertEqual(list(utils._iter_escaped_chars('a\\')),
|
||||
+ [(0, 'a'), (1, '\\')])
|
||||
+
|
||||
+ def test_strip_quoted_realnames(self):
|
||||
+ def check(addr, expected):
|
||||
+ self.assertEqual(utils._strip_quoted_realnames(addr), expected)
|
||||
+
|
||||
+ check('"Jane Doe" <jane@example.net>, "John Doe" <john@example.net>',
|
||||
+ ' <jane@example.net>, <john@example.net>')
|
||||
+ check(r'"Jane \"Doe\"." <jane@example.net>',
|
||||
+ ' <jane@example.net>')
|
||||
+
|
||||
+ # special cases
|
||||
+ check(r'before"name"after', 'beforeafter')
|
||||
+ check(r'before"name"', 'before')
|
||||
+ check(r'b"name"', 'b') # single char
|
||||
+ check(r'"name"after', 'after')
|
||||
+ check(r'"name"a', 'a') # single char
|
||||
+ check(r'"name"', '')
|
||||
+
|
||||
+ # no change
|
||||
+ for addr in (
|
||||
+ 'Jane Doe <jane@example.net>, John Doe <john@example.net>',
|
||||
+ 'lone " quote',
|
||||
+ ):
|
||||
+ self.assertEqual(utils._strip_quoted_realnames(addr), addr)
|
||||
+
|
||||
+
|
||||
+ def test_check_parenthesis(self):
|
||||
+ addr = 'alice@example.net'
|
||||
+ self.assertTrue(utils._check_parenthesis(f'{addr} (Alice)'))
|
||||
+ self.assertFalse(utils._check_parenthesis(f'{addr} )Alice('))
|
||||
+ self.assertFalse(utils._check_parenthesis(f'{addr} (Alice))'))
|
||||
+ self.assertFalse(utils._check_parenthesis(f'{addr} ((Alice)'))
|
||||
+
|
||||
+ # Ignore real name between quotes
|
||||
+ self.assertTrue(utils._check_parenthesis(f'")Alice((" {addr}'))
|
||||
+
|
||||
|
||||
# Test the iterator/generators
|
||||
class TestIterators(TestEmailBase):
|
||||
diff --git a/Misc/NEWS.d/next/Library/2023-10-20-15-28-08.gh-issue-102988.dStNO7.rst b/Misc/NEWS.d/next/Library/2023-10-20-15-28-08.gh-issue-102988.dStNO7.rst
|
||||
new file mode 100644
|
||||
index 0000000000..3d0e9e4078
|
||||
--- /dev/null
|
||||
+++ b/Misc/NEWS.d/next/Library/2023-10-20-15-28-08.gh-issue-102988.dStNO7.rst
|
||||
@@ -0,0 +1,8 @@
|
||||
+:func:`email.utils.getaddresses` and :func:`email.utils.parseaddr` now
|
||||
+return ``('', '')`` 2-tuples in more situations where invalid email
|
||||
+addresses are encountered instead of potentially inaccurate values. Add
|
||||
+optional *strict* parameter to these two functions: use ``strict=False`` to
|
||||
+get the old behavior, accept malformed inputs.
|
||||
+``getattr(email.utils, 'supports_strict_parsing', False)`` can be use to check
|
||||
+if the *strict* paramater is available. Patch by Thomas Dwyer and Victor
|
||||
+Stinner to improve the CVE-2023-27043 fix.
|
||||
|
||||
|
||||
From 4df4fad359c280f2328b98ea9b4414f244624a58 Mon Sep 17 00:00:00 2001
|
||||
From: Lumir Balhar <lbalhar@redhat.com>
|
||||
Date: Mon, 18 Dec 2023 20:15:33 +0100
|
||||
Subject: [PATCH] Make it possible to disable strict parsing in email module
|
||||
|
||||
---
|
||||
Doc/library/email.utils.rst | 26 +++++++++++
|
||||
Lib/email/utils.py | 54 ++++++++++++++++++++++-
|
||||
Lib/test/test_email/test_email.py | 72 ++++++++++++++++++++++++++++++-
|
||||
3 files changed, 149 insertions(+), 3 deletions(-)
|
||||
|
||||
diff --git a/Doc/library/email.utils.rst b/Doc/library/email.utils.rst
|
||||
index d1e1898591..7aef773b5f 100644
|
||||
--- a/Doc/library/email.utils.rst
|
||||
+++ b/Doc/library/email.utils.rst
|
||||
@@ -69,6 +69,19 @@ of the new API.
|
||||
|
||||
If *strict* is true, use a strict parser which rejects malformed inputs.
|
||||
|
||||
+ The default setting for *strict* is set to ``True``, but you can override
|
||||
+ it by setting the environment variable ``PYTHON_EMAIL_DISABLE_STRICT_ADDR_PARSING``
|
||||
+ to non-empty string.
|
||||
+
|
||||
+ Additionally, you can permanently set the default value for *strict* to
|
||||
+ ``False`` by creating the configuration file ``/etc/python/email.cfg``
|
||||
+ with the following content:
|
||||
+
|
||||
+ .. code-block:: ini
|
||||
+
|
||||
+ [email_addr_parsing]
|
||||
+ PYTHON_EMAIL_DISABLE_STRICT_ADDR_PARSING = true
|
||||
+
|
||||
.. versionchanged:: 3.13
|
||||
Add *strict* optional parameter and reject malformed inputs by default.
|
||||
|
||||
@@ -97,6 +110,19 @@ of the new API.
|
||||
|
||||
If *strict* is true, use a strict parser which rejects malformed inputs.
|
||||
|
||||
+ The default setting for *strict* is set to ``True``, but you can override
|
||||
+ it by setting the environment variable ``PYTHON_EMAIL_DISABLE_STRICT_ADDR_PARSING``
|
||||
+ to non-empty string.
|
||||
+
|
||||
+ Additionally, you can permanently set the default value for *strict* to
|
||||
+ ``False`` by creating the configuration file ``/etc/python/email.cfg``
|
||||
+ with the following content:
|
||||
+
|
||||
+ .. code-block:: ini
|
||||
+
|
||||
+ [email_addr_parsing]
|
||||
+ PYTHON_EMAIL_DISABLE_STRICT_ADDR_PARSING = true
|
||||
+
|
||||
Here's a simple example that gets all the recipients of a message::
|
||||
|
||||
from email.utils import getaddresses
|
||||
diff --git a/Lib/email/utils.py b/Lib/email/utils.py
|
||||
index f83b7e5d7e..b8e90ceb8e 100644
|
||||
--- a/Lib/email/utils.py
|
||||
+++ b/Lib/email/utils.py
|
||||
@@ -48,6 +48,46 @@ TICK = "'"
|
||||
specialsre = re.compile(r'[][\\()<>@,:;".]')
|
||||
escapesre = re.compile(r'[\\"]')
|
||||
|
||||
+_EMAIL_CONFIG_FILE = "/etc/python/email.cfg"
|
||||
+_cached_strict_addr_parsing = None
|
||||
+
|
||||
+
|
||||
+def _use_strict_email_parsing():
|
||||
+ """"Cache implementation for _cached_strict_addr_parsing"""
|
||||
+ global _cached_strict_addr_parsing
|
||||
+ if _cached_strict_addr_parsing is None:
|
||||
+ _cached_strict_addr_parsing = _use_strict_email_parsing_impl()
|
||||
+ return _cached_strict_addr_parsing
|
||||
+
|
||||
+
|
||||
+def _use_strict_email_parsing_impl():
|
||||
+ """Returns True if strict email parsing is not disabled by
|
||||
+ config file or env variable.
|
||||
+ """
|
||||
+ disabled = bool(os.environ.get("PYTHON_EMAIL_DISABLE_STRICT_ADDR_PARSING"))
|
||||
+ if disabled:
|
||||
+ return False
|
||||
+
|
||||
+ try:
|
||||
+ file = open(_EMAIL_CONFIG_FILE)
|
||||
+ except FileNotFoundError:
|
||||
+ pass
|
||||
+ else:
|
||||
+ with file:
|
||||
+ import configparser
|
||||
+ config = configparser.ConfigParser(
|
||||
+ interpolation=None,
|
||||
+ comment_prefixes=('#', ),
|
||||
+
|
||||
+ )
|
||||
+ config.read_file(file)
|
||||
+ disabled = config.getboolean('email_addr_parsing', "PYTHON_EMAIL_DISABLE_STRICT_ADDR_PARSING", fallback=None)
|
||||
+
|
||||
+ if disabled:
|
||||
+ return False
|
||||
+
|
||||
+ return True
|
||||
+
|
||||
|
||||
def _has_surrogates(s):
|
||||
"""Return True if s contains surrogate-escaped binary data."""
|
||||
@@ -149,7 +189,7 @@ def _strip_quoted_realnames(addr):
|
||||
|
||||
supports_strict_parsing = True
|
||||
|
||||
-def getaddresses(fieldvalues, *, strict=True):
|
||||
+def getaddresses(fieldvalues, *, strict=None):
|
||||
"""Return a list of (REALNAME, EMAIL) or ('','') for each fieldvalue.
|
||||
|
||||
When parsing fails for a fieldvalue, a 2-tuple of ('', '') is returned in
|
||||
@@ -158,6 +198,11 @@ def getaddresses(fieldvalues, *, strict=True):
|
||||
If strict is true, use a strict parser which rejects malformed inputs.
|
||||
"""
|
||||
|
||||
+ # If default is used, it's True unless disabled
|
||||
+ # by env variable or config file.
|
||||
+ if strict == None:
|
||||
+ strict = _use_strict_email_parsing()
|
||||
+
|
||||
# If strict is true, if the resulting list of parsed addresses is greater
|
||||
# than the number of fieldvalues in the input list, a parsing error has
|
||||
# occurred and consequently a list containing a single empty 2-tuple [('',
|
||||
@@ -330,7 +375,7 @@ def parsedate_to_datetime(data):
|
||||
tzinfo=datetime.timezone(datetime.timedelta(seconds=tz)))
|
||||
|
||||
|
||||
-def parseaddr(addr, *, strict=True):
|
||||
+def parseaddr(addr, *, strict=None):
|
||||
"""
|
||||
Parse addr into its constituent realname and email address parts.
|
||||
|
||||
@@ -339,6 +384,11 @@ def parseaddr(addr, *, strict=True):
|
||||
|
||||
If strict is True, use a strict parser which rejects malformed inputs.
|
||||
"""
|
||||
+ # If default is used, it's True unless disabled
|
||||
+ # by env variable or config file.
|
||||
+ if strict == None:
|
||||
+ strict = _use_strict_email_parsing()
|
||||
+
|
||||
if not strict:
|
||||
addrs = _AddressList(addr).addresslist
|
||||
if not addrs:
|
||||
diff --git a/Lib/test/test_email/test_email.py b/Lib/test/test_email/test_email.py
|
||||
index ce36efc1b1..05ea201b68 100644
|
||||
--- a/Lib/test/test_email/test_email.py
|
||||
+++ b/Lib/test/test_email/test_email.py
|
||||
@@ -7,6 +7,9 @@ import time
|
||||
import base64
|
||||
import unittest
|
||||
import textwrap
|
||||
+import contextlib
|
||||
+import tempfile
|
||||
+import os
|
||||
|
||||
from io import StringIO, BytesIO
|
||||
from itertools import chain
|
||||
@@ -41,7 +44,7 @@ from email import iterators
|
||||
from email import base64mime
|
||||
from email import quoprimime
|
||||
|
||||
-from test.support import unlink, start_threads
|
||||
+from test.support import unlink, start_threads, EnvironmentVarGuard, swap_attr
|
||||
from test.test_email import openfile, TestEmailBase
|
||||
|
||||
# These imports are documented to work, but we are testing them using a
|
||||
@@ -3313,6 +3316,73 @@ Foo
|
||||
# Test email.utils.supports_strict_parsing attribute
|
||||
self.assertEqual(email.utils.supports_strict_parsing, True)
|
||||
|
||||
+ def test_parsing_errors_strict_set_via_env_var(self):
|
||||
+ address = 'alice@example.org )Alice('
|
||||
+ empty = ('', '')
|
||||
+
|
||||
+ # Reset cached default value to make the function
|
||||
+ # reload the config file provided below.
|
||||
+ utils._cached_strict_addr_parsing = None
|
||||
+
|
||||
+ # Strict disabled via env variable, old behavior expected
|
||||
+ with EnvironmentVarGuard() as environ:
|
||||
+ environ["PYTHON_EMAIL_DISABLE_STRICT_ADDR_PARSING"] = "1"
|
||||
+
|
||||
+ self.assertEqual(utils.getaddresses([address]),
|
||||
+ [('', 'alice@example.org'), ('', ''), ('', 'Alice')])
|
||||
+ self.assertEqual(utils.parseaddr([address]), ('', address))
|
||||
+
|
||||
+ # Clear cache again
|
||||
+ utils._cached_strict_addr_parsing = None
|
||||
+
|
||||
+ # Default strict=True, empty result expected
|
||||
+ self.assertEqual(utils.getaddresses([address]), [empty])
|
||||
+ self.assertEqual(utils.parseaddr([address]), empty)
|
||||
+
|
||||
+ # Clear cache again
|
||||
+ utils._cached_strict_addr_parsing = None
|
||||
+
|
||||
+ # Empty string in env variable = strict parsing enabled (default)
|
||||
+ with EnvironmentVarGuard() as environ:
|
||||
+ environ["PYTHON_EMAIL_DISABLE_STRICT_ADDR_PARSING"] = ""
|
||||
+
|
||||
+ # Default strict=True, empty result expected
|
||||
+ self.assertEqual(utils.getaddresses([address]), [empty])
|
||||
+ self.assertEqual(utils.parseaddr([address]), empty)
|
||||
+
|
||||
+ @contextlib.contextmanager
|
||||
+ def _email_strict_parsing_conf(self):
|
||||
+ """Context for the given email strict parsing configured in config file"""
|
||||
+ with tempfile.TemporaryDirectory() as tmpdirname:
|
||||
+ filename = os.path.join(tmpdirname, 'conf.cfg')
|
||||
+ with swap_attr(utils, "_EMAIL_CONFIG_FILE", filename):
|
||||
+ with open(filename, 'w') as file:
|
||||
+ file.write('[email_addr_parsing]\n')
|
||||
+ file.write('PYTHON_EMAIL_DISABLE_STRICT_ADDR_PARSING = true')
|
||||
+ utils._EMAIL_CONFIG_FILE = filename
|
||||
+ yield
|
||||
+
|
||||
+ def test_parsing_errors_strict_disabled_via_config_file(self):
|
||||
+ address = 'alice@example.org )Alice('
|
||||
+ empty = ('', '')
|
||||
+
|
||||
+ # Reset cached default value to make the function
|
||||
+ # reload the config file provided below.
|
||||
+ utils._cached_strict_addr_parsing = None
|
||||
+
|
||||
+ # Strict disabled via config file, old results expected
|
||||
+ with self._email_strict_parsing_conf():
|
||||
+ self.assertEqual(utils.getaddresses([address]),
|
||||
+ [('', 'alice@example.org'), ('', ''), ('', 'Alice')])
|
||||
+ self.assertEqual(utils.parseaddr([address]), ('', address))
|
||||
+
|
||||
+ # Clear cache again
|
||||
+ utils._cached_strict_addr_parsing = None
|
||||
+
|
||||
+ # Default strict=True, empty result expected
|
||||
+ self.assertEqual(utils.getaddresses([address]), [empty])
|
||||
+ self.assertEqual(utils.parseaddr([address]), empty)
|
||||
+
|
||||
def test_getaddresses_nasty(self):
|
||||
for addresses, expected in (
|
||||
(['"Sürname, Firstname" <to@example.com>'],
|
||||
--
|
||||
2.43.0
|
||||
|
|
@ -0,0 +1,90 @@
|
|||
From 87acab66e124912549fbc3151f27ca7fae76386c Mon Sep 17 00:00:00 2001
|
||||
From: Serhiy Storchaka <storchaka@gmail.com>
|
||||
Date: Tue, 23 Apr 2024 19:54:00 +0200
|
||||
Subject: [PATCH] gh-115133: Fix tests for XMLPullParser with Expat 2.6.0
|
||||
|
||||
Feeding the parser by too small chunks defers parsing to prevent
|
||||
CVE-2023-52425. Future versions of Expat may be more reactive.
|
||||
|
||||
(cherry picked from commit 4a08e7b3431cd32a0daf22a33421cd3035343dc4)
|
||||
---
|
||||
Lib/test/test_xml_etree.py | 53 +++++++++++--------
|
||||
...-02-08-14-21-28.gh-issue-115133.ycl4ko.rst | 2 +
|
||||
2 files changed, 33 insertions(+), 22 deletions(-)
|
||||
create mode 100644 Misc/NEWS.d/next/Library/2024-02-08-14-21-28.gh-issue-115133.ycl4ko.rst
|
||||
|
||||
diff --git a/Lib/test/test_xml_etree.py b/Lib/test/test_xml_etree.py
|
||||
index acaa519..c01af47 100644
|
||||
--- a/Lib/test/test_xml_etree.py
|
||||
+++ b/Lib/test/test_xml_etree.py
|
||||
@@ -1044,28 +1044,37 @@ class XMLPullParserTest(unittest.TestCase):
|
||||
self.assertEqual([(action, elem.tag) for action, elem in events],
|
||||
expected)
|
||||
|
||||
- def test_simple_xml(self):
|
||||
- for chunk_size in (None, 1, 5):
|
||||
- with self.subTest(chunk_size=chunk_size):
|
||||
- parser = ET.XMLPullParser()
|
||||
- self.assert_event_tags(parser, [])
|
||||
- self._feed(parser, "<!-- comment -->\n", chunk_size)
|
||||
- self.assert_event_tags(parser, [])
|
||||
- self._feed(parser,
|
||||
- "<root>\n <element key='value'>text</element",
|
||||
- chunk_size)
|
||||
- self.assert_event_tags(parser, [])
|
||||
- self._feed(parser, ">\n", chunk_size)
|
||||
- self.assert_event_tags(parser, [('end', 'element')])
|
||||
- self._feed(parser, "<element>text</element>tail\n", chunk_size)
|
||||
- self._feed(parser, "<empty-element/>\n", chunk_size)
|
||||
- self.assert_event_tags(parser, [
|
||||
- ('end', 'element'),
|
||||
- ('end', 'empty-element'),
|
||||
- ])
|
||||
- self._feed(parser, "</root>\n", chunk_size)
|
||||
- self.assert_event_tags(parser, [('end', 'root')])
|
||||
- self.assertIsNone(parser.close())
|
||||
+ def test_simple_xml(self, chunk_size=None):
|
||||
+ parser = ET.XMLPullParser()
|
||||
+ self.assert_event_tags(parser, [])
|
||||
+ self._feed(parser, "<!-- comment -->\n", chunk_size)
|
||||
+ self.assert_event_tags(parser, [])
|
||||
+ self._feed(parser,
|
||||
+ "<root>\n <element key='value'>text</element",
|
||||
+ chunk_size)
|
||||
+ self.assert_event_tags(parser, [])
|
||||
+ self._feed(parser, ">\n", chunk_size)
|
||||
+ self.assert_event_tags(parser, [('end', 'element')])
|
||||
+ self._feed(parser, "<element>text</element>tail\n", chunk_size)
|
||||
+ self._feed(parser, "<empty-element/>\n", chunk_size)
|
||||
+ self.assert_event_tags(parser, [
|
||||
+ ('end', 'element'),
|
||||
+ ('end', 'empty-element'),
|
||||
+ ])
|
||||
+ self._feed(parser, "</root>\n", chunk_size)
|
||||
+ self.assert_event_tags(parser, [('end', 'root')])
|
||||
+ self.assertIsNone(parser.close())
|
||||
+
|
||||
+ @unittest.expectedFailure
|
||||
+ def test_simple_xml_chunk_1(self):
|
||||
+ self.test_simple_xml(chunk_size=1)
|
||||
+
|
||||
+ @unittest.expectedFailure
|
||||
+ def test_simple_xml_chunk_5(self):
|
||||
+ self.test_simple_xml(chunk_size=5)
|
||||
+
|
||||
+ def test_simple_xml_chunk_22(self):
|
||||
+ self.test_simple_xml(chunk_size=22)
|
||||
|
||||
def test_feed_while_iterating(self):
|
||||
parser = ET.XMLPullParser()
|
||||
diff --git a/Misc/NEWS.d/next/Library/2024-02-08-14-21-28.gh-issue-115133.ycl4ko.rst b/Misc/NEWS.d/next/Library/2024-02-08-14-21-28.gh-issue-115133.ycl4ko.rst
|
||||
new file mode 100644
|
||||
index 0000000..6f10152
|
||||
--- /dev/null
|
||||
+++ b/Misc/NEWS.d/next/Library/2024-02-08-14-21-28.gh-issue-115133.ycl4ko.rst
|
||||
@@ -0,0 +1,2 @@
|
||||
+Fix tests for :class:`~xml.etree.ElementTree.XMLPullParser` with Expat
|
||||
+2.6.0.
|
||||
--
|
||||
2.44.0
|
||||
|
|
@ -0,0 +1,291 @@
|
|||
From 82f1ea4b72be40f58fd0a9a37f8d8d2f7d16f9e0 Mon Sep 17 00:00:00 2001
|
||||
From: Lumir Balhar <lbalhar@redhat.com>
|
||||
Date: Wed, 24 Apr 2024 00:19:23 +0200
|
||||
Subject: [PATCH] CVE-2023-6597
|
||||
|
||||
Co-authored-by: Søren Løvborg <sorenl@unity3d.com>
|
||||
Co-authored-by: Serhiy Storchaka <storchaka@gmail.com>
|
||||
---
|
||||
Lib/tempfile.py | 44 +++++++++-
|
||||
Lib/test/test_tempfile.py | 166 +++++++++++++++++++++++++++++++++++---
|
||||
2 files changed, 199 insertions(+), 11 deletions(-)
|
||||
|
||||
diff --git a/Lib/tempfile.py b/Lib/tempfile.py
|
||||
index 2cb5434..d79b70c 100644
|
||||
--- a/Lib/tempfile.py
|
||||
+++ b/Lib/tempfile.py
|
||||
@@ -276,6 +276,23 @@ def _mkstemp_inner(dir, pre, suf, flags, output_type):
|
||||
"No usable temporary file name found")
|
||||
|
||||
|
||||
+def _dont_follow_symlinks(func, path, *args):
|
||||
+ # Pass follow_symlinks=False, unless not supported on this platform.
|
||||
+ if func in _os.supports_follow_symlinks:
|
||||
+ func(path, *args, follow_symlinks=False)
|
||||
+ elif _os.name == 'nt' or not _os.path.islink(path):
|
||||
+ func(path, *args)
|
||||
+
|
||||
+
|
||||
+def _resetperms(path):
|
||||
+ try:
|
||||
+ chflags = _os.chflags
|
||||
+ except AttributeError:
|
||||
+ pass
|
||||
+ else:
|
||||
+ _dont_follow_symlinks(chflags, path, 0)
|
||||
+ _dont_follow_symlinks(_os.chmod, path, 0o700)
|
||||
+
|
||||
# User visible interfaces.
|
||||
|
||||
def gettempprefix():
|
||||
@@ -794,9 +811,32 @@ class TemporaryDirectory(object):
|
||||
self, self._cleanup, self.name,
|
||||
warn_message="Implicitly cleaning up {!r}".format(self))
|
||||
|
||||
+ @classmethod
|
||||
+ def _rmtree(cls, name):
|
||||
+ def onerror(func, path, exc_info):
|
||||
+ if issubclass(exc_info[0], PermissionError):
|
||||
+ try:
|
||||
+ if path != name:
|
||||
+ _resetperms(_os.path.dirname(path))
|
||||
+ _resetperms(path)
|
||||
+
|
||||
+ try:
|
||||
+ _os.unlink(path)
|
||||
+ # PermissionError is raised on FreeBSD for directories
|
||||
+ except (IsADirectoryError, PermissionError):
|
||||
+ cls._rmtree(path)
|
||||
+ except FileNotFoundError:
|
||||
+ pass
|
||||
+ elif issubclass(exc_info[0], FileNotFoundError):
|
||||
+ pass
|
||||
+ else:
|
||||
+ raise
|
||||
+
|
||||
+ _shutil.rmtree(name, onerror=onerror)
|
||||
+
|
||||
@classmethod
|
||||
def _cleanup(cls, name, warn_message):
|
||||
- _shutil.rmtree(name)
|
||||
+ cls._rmtree(name)
|
||||
_warnings.warn(warn_message, ResourceWarning)
|
||||
|
||||
def __repr__(self):
|
||||
@@ -810,4 +850,4 @@ class TemporaryDirectory(object):
|
||||
|
||||
def cleanup(self):
|
||||
if self._finalizer.detach():
|
||||
- _shutil.rmtree(self.name)
|
||||
+ self._rmtree(self.name)
|
||||
diff --git a/Lib/test/test_tempfile.py b/Lib/test/test_tempfile.py
|
||||
index 710756b..c5560e1 100644
|
||||
--- a/Lib/test/test_tempfile.py
|
||||
+++ b/Lib/test/test_tempfile.py
|
||||
@@ -1298,19 +1298,25 @@ class NulledModules:
|
||||
class TestTemporaryDirectory(BaseTestCase):
|
||||
"""Test TemporaryDirectory()."""
|
||||
|
||||
- def do_create(self, dir=None, pre="", suf="", recurse=1):
|
||||
+ def do_create(self, dir=None, pre="", suf="", recurse=1, dirs=1, files=1):
|
||||
if dir is None:
|
||||
dir = tempfile.gettempdir()
|
||||
tmp = tempfile.TemporaryDirectory(dir=dir, prefix=pre, suffix=suf)
|
||||
self.nameCheck(tmp.name, dir, pre, suf)
|
||||
- # Create a subdirectory and some files
|
||||
- if recurse:
|
||||
- d1 = self.do_create(tmp.name, pre, suf, recurse-1)
|
||||
- d1.name = None
|
||||
- with open(os.path.join(tmp.name, "test.txt"), "wb") as f:
|
||||
- f.write(b"Hello world!")
|
||||
+ self.do_create2(tmp.name, recurse, dirs, files)
|
||||
return tmp
|
||||
|
||||
+ def do_create2(self, path, recurse=1, dirs=1, files=1):
|
||||
+ # Create subdirectories and some files
|
||||
+ if recurse:
|
||||
+ for i in range(dirs):
|
||||
+ name = os.path.join(path, "dir%d" % i)
|
||||
+ os.mkdir(name)
|
||||
+ self.do_create2(name, recurse-1, dirs, files)
|
||||
+ for i in range(files):
|
||||
+ with open(os.path.join(path, "test%d.txt" % i), "wb") as f:
|
||||
+ f.write(b"Hello world!")
|
||||
+
|
||||
def test_mkdtemp_failure(self):
|
||||
# Check no additional exception if mkdtemp fails
|
||||
# Previously would raise AttributeError instead
|
||||
@@ -1350,11 +1356,108 @@ class TestTemporaryDirectory(BaseTestCase):
|
||||
"TemporaryDirectory %s exists after cleanup" % d1.name)
|
||||
self.assertTrue(os.path.exists(d2.name),
|
||||
"Directory pointed to by a symlink was deleted")
|
||||
- self.assertEqual(os.listdir(d2.name), ['test.txt'],
|
||||
+ self.assertEqual(os.listdir(d2.name), ['test0.txt'],
|
||||
"Contents of the directory pointed to by a symlink "
|
||||
"were deleted")
|
||||
d2.cleanup()
|
||||
|
||||
+ @support.skip_unless_symlink
|
||||
+ def test_cleanup_with_symlink_modes(self):
|
||||
+ # cleanup() should not follow symlinks when fixing mode bits (#91133)
|
||||
+ with self.do_create(recurse=0) as d2:
|
||||
+ file1 = os.path.join(d2, 'file1')
|
||||
+ open(file1, 'wb').close()
|
||||
+ dir1 = os.path.join(d2, 'dir1')
|
||||
+ os.mkdir(dir1)
|
||||
+ for mode in range(8):
|
||||
+ mode <<= 6
|
||||
+ with self.subTest(mode=format(mode, '03o')):
|
||||
+ def test(target, target_is_directory):
|
||||
+ d1 = self.do_create(recurse=0)
|
||||
+ symlink = os.path.join(d1.name, 'symlink')
|
||||
+ os.symlink(target, symlink,
|
||||
+ target_is_directory=target_is_directory)
|
||||
+ try:
|
||||
+ os.chmod(symlink, mode, follow_symlinks=False)
|
||||
+ except NotImplementedError:
|
||||
+ pass
|
||||
+ try:
|
||||
+ os.chmod(symlink, mode)
|
||||
+ except FileNotFoundError:
|
||||
+ pass
|
||||
+ os.chmod(d1.name, mode)
|
||||
+ d1.cleanup()
|
||||
+ self.assertFalse(os.path.exists(d1.name))
|
||||
+
|
||||
+ with self.subTest('nonexisting file'):
|
||||
+ test('nonexisting', target_is_directory=False)
|
||||
+ with self.subTest('nonexisting dir'):
|
||||
+ test('nonexisting', target_is_directory=True)
|
||||
+
|
||||
+ with self.subTest('existing file'):
|
||||
+ os.chmod(file1, mode)
|
||||
+ old_mode = os.stat(file1).st_mode
|
||||
+ test(file1, target_is_directory=False)
|
||||
+ new_mode = os.stat(file1).st_mode
|
||||
+ self.assertEqual(new_mode, old_mode,
|
||||
+ '%03o != %03o' % (new_mode, old_mode))
|
||||
+
|
||||
+ with self.subTest('existing dir'):
|
||||
+ os.chmod(dir1, mode)
|
||||
+ old_mode = os.stat(dir1).st_mode
|
||||
+ test(dir1, target_is_directory=True)
|
||||
+ new_mode = os.stat(dir1).st_mode
|
||||
+ self.assertEqual(new_mode, old_mode,
|
||||
+ '%03o != %03o' % (new_mode, old_mode))
|
||||
+
|
||||
+ @unittest.skipUnless(hasattr(os, 'chflags'), 'requires os.chflags')
|
||||
+ @support.skip_unless_symlink
|
||||
+ def test_cleanup_with_symlink_flags(self):
|
||||
+ # cleanup() should not follow symlinks when fixing flags (#91133)
|
||||
+ flags = stat.UF_IMMUTABLE | stat.UF_NOUNLINK
|
||||
+ self.check_flags(flags)
|
||||
+
|
||||
+ with self.do_create(recurse=0) as d2:
|
||||
+ file1 = os.path.join(d2, 'file1')
|
||||
+ open(file1, 'wb').close()
|
||||
+ dir1 = os.path.join(d2, 'dir1')
|
||||
+ os.mkdir(dir1)
|
||||
+ def test(target, target_is_directory):
|
||||
+ d1 = self.do_create(recurse=0)
|
||||
+ symlink = os.path.join(d1.name, 'symlink')
|
||||
+ os.symlink(target, symlink,
|
||||
+ target_is_directory=target_is_directory)
|
||||
+ try:
|
||||
+ os.chflags(symlink, flags, follow_symlinks=False)
|
||||
+ except NotImplementedError:
|
||||
+ pass
|
||||
+ try:
|
||||
+ os.chflags(symlink, flags)
|
||||
+ except FileNotFoundError:
|
||||
+ pass
|
||||
+ os.chflags(d1.name, flags)
|
||||
+ d1.cleanup()
|
||||
+ self.assertFalse(os.path.exists(d1.name))
|
||||
+
|
||||
+ with self.subTest('nonexisting file'):
|
||||
+ test('nonexisting', target_is_directory=False)
|
||||
+ with self.subTest('nonexisting dir'):
|
||||
+ test('nonexisting', target_is_directory=True)
|
||||
+
|
||||
+ with self.subTest('existing file'):
|
||||
+ os.chflags(file1, flags)
|
||||
+ old_flags = os.stat(file1).st_flags
|
||||
+ test(file1, target_is_directory=False)
|
||||
+ new_flags = os.stat(file1).st_flags
|
||||
+ self.assertEqual(new_flags, old_flags)
|
||||
+
|
||||
+ with self.subTest('existing dir'):
|
||||
+ os.chflags(dir1, flags)
|
||||
+ old_flags = os.stat(dir1).st_flags
|
||||
+ test(dir1, target_is_directory=True)
|
||||
+ new_flags = os.stat(dir1).st_flags
|
||||
+ self.assertEqual(new_flags, old_flags)
|
||||
+
|
||||
@support.cpython_only
|
||||
def test_del_on_collection(self):
|
||||
# A TemporaryDirectory is deleted when garbage collected
|
||||
@@ -1385,7 +1488,7 @@ class TestTemporaryDirectory(BaseTestCase):
|
||||
|
||||
tmp2 = os.path.join(tmp.name, 'test_dir')
|
||||
os.mkdir(tmp2)
|
||||
- with open(os.path.join(tmp2, "test.txt"), "w") as f:
|
||||
+ with open(os.path.join(tmp2, "test0.txt"), "w") as f:
|
||||
f.write("Hello world!")
|
||||
|
||||
{mod}.tmp = tmp
|
||||
@@ -1453,6 +1556,51 @@ class TestTemporaryDirectory(BaseTestCase):
|
||||
self.assertEqual(name, d.name)
|
||||
self.assertFalse(os.path.exists(name))
|
||||
|
||||
+ def test_modes(self):
|
||||
+ for mode in range(8):
|
||||
+ mode <<= 6
|
||||
+ with self.subTest(mode=format(mode, '03o')):
|
||||
+ d = self.do_create(recurse=3, dirs=2, files=2)
|
||||
+ with d:
|
||||
+ # Change files and directories mode recursively.
|
||||
+ for root, dirs, files in os.walk(d.name, topdown=False):
|
||||
+ for name in files:
|
||||
+ os.chmod(os.path.join(root, name), mode)
|
||||
+ os.chmod(root, mode)
|
||||
+ d.cleanup()
|
||||
+ self.assertFalse(os.path.exists(d.name))
|
||||
+
|
||||
+ def check_flags(self, flags):
|
||||
+ # skip the test if these flags are not supported (ex: FreeBSD 13)
|
||||
+ filename = support.TESTFN
|
||||
+ try:
|
||||
+ open(filename, "w").close()
|
||||
+ try:
|
||||
+ os.chflags(filename, flags)
|
||||
+ except OSError as exc:
|
||||
+ # "OSError: [Errno 45] Operation not supported"
|
||||
+ self.skipTest(f"chflags() doesn't support flags "
|
||||
+ f"{flags:#b}: {exc}")
|
||||
+ else:
|
||||
+ os.chflags(filename, 0)
|
||||
+ finally:
|
||||
+ support.unlink(filename)
|
||||
+
|
||||
+ @unittest.skipUnless(hasattr(os, 'chflags'), 'requires os.lchflags')
|
||||
+ def test_flags(self):
|
||||
+ flags = stat.UF_IMMUTABLE | stat.UF_NOUNLINK
|
||||
+ self.check_flags(flags)
|
||||
+
|
||||
+ d = self.do_create(recurse=3, dirs=2, files=2)
|
||||
+ with d:
|
||||
+ # Change files and directories flags recursively.
|
||||
+ for root, dirs, files in os.walk(d.name, topdown=False):
|
||||
+ for name in files:
|
||||
+ os.chflags(os.path.join(root, name), flags)
|
||||
+ os.chflags(root, flags)
|
||||
+ d.cleanup()
|
||||
+ self.assertFalse(os.path.exists(d.name))
|
||||
+
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
--
|
||||
2.44.0
|
||||
|
|
@ -0,0 +1,346 @@
|
|||
From 066df4fd454d6ff9be66e80b2a65995b10af174f Mon Sep 17 00:00:00 2001
|
||||
From: John Jolly <john.jolly@gmail.com>
|
||||
Date: Tue, 30 Jan 2018 01:51:35 -0700
|
||||
Subject: [PATCH] bpo-22908: Add seek and tell functionality to ZipExtFile
|
||||
(GH-4966)
|
||||
|
||||
This allows for nested zip files, tar files within zip files, zip files within tar files, etc.
|
||||
|
||||
Contributed by: John Jolly
|
||||
---
|
||||
Doc/library/zipfile.rst | 6 +-
|
||||
Lib/test/test_zipfile.py | 34 ++++++++
|
||||
Lib/zipfile.py | 82 +++++++++++++++++++
|
||||
.../2017-12-21-22-00-11.bpo-22908.cVm89I.rst | 2 +
|
||||
4 files changed, 121 insertions(+), 3 deletions(-)
|
||||
create mode 100644 Misc/NEWS.d/next/Library/2017-12-21-22-00-11.bpo-22908.cVm89I.rst
|
||||
|
||||
diff --git a/Doc/library/zipfile.rst b/Doc/library/zipfile.rst
|
||||
index d58efe0b417516..7c9a8c80225491 100644
|
||||
--- a/Doc/library/zipfile.rst
|
||||
+++ b/Doc/library/zipfile.rst
|
||||
@@ -246,9 +246,9 @@ ZipFile Objects
|
||||
With *mode* ``'r'`` the file-like object
|
||||
(``ZipExtFile``) is read-only and provides the following methods:
|
||||
:meth:`~io.BufferedIOBase.read`, :meth:`~io.IOBase.readline`,
|
||||
- :meth:`~io.IOBase.readlines`, :meth:`__iter__`,
|
||||
- :meth:`~iterator.__next__`. These objects can operate independently of
|
||||
- the ZipFile.
|
||||
+ :meth:`~io.IOBase.readlines`, :meth:`~io.IOBase.seek`,
|
||||
+ :meth:`~io.IOBase.tell`, :meth:`__iter__`, :meth:`~iterator.__next__`.
|
||||
+ These objects can operate independently of the ZipFile.
|
||||
|
||||
With ``mode='w'``, a writable file handle is returned, which supports the
|
||||
:meth:`~io.BufferedIOBase.write` method. While a writable file handle is open,
|
||||
diff --git a/Lib/test/test_zipfile.py b/Lib/test/test_zipfile.py
|
||||
index 94db858a1517c4..61c3e349a69ef4 100644
|
||||
--- a/Lib/test/test_zipfile.py
|
||||
+++ b/Lib/test/test_zipfile.py
|
||||
@@ -1628,6 +1628,40 @@ def test_open_conflicting_handles(self):
|
||||
self.assertEqual(zipf.read('baz'), msg3)
|
||||
self.assertEqual(zipf.namelist(), ['foo', 'bar', 'baz'])
|
||||
|
||||
+ def test_seek_tell(self):
|
||||
+ # Test seek functionality
|
||||
+ txt = b"Where's Bruce?"
|
||||
+ bloc = txt.find(b"Bruce")
|
||||
+ # Check seek on a file
|
||||
+ with zipfile.ZipFile(TESTFN, "w") as zipf:
|
||||
+ zipf.writestr("foo.txt", txt)
|
||||
+ with zipfile.ZipFile(TESTFN, "r") as zipf:
|
||||
+ with zipf.open("foo.txt", "r") as fp:
|
||||
+ fp.seek(bloc, os.SEEK_SET)
|
||||
+ self.assertEqual(fp.tell(), bloc)
|
||||
+ fp.seek(-bloc, os.SEEK_CUR)
|
||||
+ self.assertEqual(fp.tell(), 0)
|
||||
+ fp.seek(bloc, os.SEEK_CUR)
|
||||
+ self.assertEqual(fp.tell(), bloc)
|
||||
+ self.assertEqual(fp.read(5), txt[bloc:bloc+5])
|
||||
+ fp.seek(0, os.SEEK_END)
|
||||
+ self.assertEqual(fp.tell(), len(txt))
|
||||
+ # Check seek on memory file
|
||||
+ data = io.BytesIO()
|
||||
+ with zipfile.ZipFile(data, mode="w") as zipf:
|
||||
+ zipf.writestr("foo.txt", txt)
|
||||
+ with zipfile.ZipFile(data, mode="r") as zipf:
|
||||
+ with zipf.open("foo.txt", "r") as fp:
|
||||
+ fp.seek(bloc, os.SEEK_SET)
|
||||
+ self.assertEqual(fp.tell(), bloc)
|
||||
+ fp.seek(-bloc, os.SEEK_CUR)
|
||||
+ self.assertEqual(fp.tell(), 0)
|
||||
+ fp.seek(bloc, os.SEEK_CUR)
|
||||
+ self.assertEqual(fp.tell(), bloc)
|
||||
+ self.assertEqual(fp.read(5), txt[bloc:bloc+5])
|
||||
+ fp.seek(0, os.SEEK_END)
|
||||
+ self.assertEqual(fp.tell(), len(txt))
|
||||
+
|
||||
def tearDown(self):
|
||||
unlink(TESTFN)
|
||||
unlink(TESTFN2)
|
||||
diff --git a/Lib/zipfile.py b/Lib/zipfile.py
|
||||
index f9db45f58a2bde..5df7b1bf75b9d9 100644
|
||||
--- a/Lib/zipfile.py
|
||||
+++ b/Lib/zipfile.py
|
||||
@@ -696,6 +696,18 @@ def __init__(self, file, pos, close, lock, writing):
|
||||
self._close = close
|
||||
self._lock = lock
|
||||
self._writing = writing
|
||||
+ self.seekable = file.seekable
|
||||
+ self.tell = file.tell
|
||||
+
|
||||
+ def seek(self, offset, whence=0):
|
||||
+ with self._lock:
|
||||
+ if self.writing():
|
||||
+ raise ValueError("Can't reposition in the ZIP file while "
|
||||
+ "there is an open writing handle on it. "
|
||||
+ "Close the writing handle before trying to read.")
|
||||
+ self._file.seek(self._pos)
|
||||
+ self._pos = self._file.tell()
|
||||
+ return self._pos
|
||||
|
||||
def read(self, n=-1):
|
||||
with self._lock:
|
||||
@@ -746,6 +758,9 @@ class ZipExtFile(io.BufferedIOBase):
|
||||
# Read from compressed files in 4k blocks.
|
||||
MIN_READ_SIZE = 4096
|
||||
|
||||
+ # Chunk size to read during seek
|
||||
+ MAX_SEEK_READ = 1 << 24
|
||||
+
|
||||
def __init__(self, fileobj, mode, zipinfo, decrypter=None,
|
||||
close_fileobj=False):
|
||||
self._fileobj = fileobj
|
||||
@@ -778,6 +793,17 @@ def __init__(self, fileobj, mode, zipinfo, decrypter=None,
|
||||
else:
|
||||
self._expected_crc = None
|
||||
|
||||
+ self._seekable = False
|
||||
+ try:
|
||||
+ if fileobj.seekable():
|
||||
+ self._orig_compress_start = fileobj.tell()
|
||||
+ self._orig_compress_size = zipinfo.compress_size
|
||||
+ self._orig_file_size = zipinfo.file_size
|
||||
+ self._orig_start_crc = self._running_crc
|
||||
+ self._seekable = True
|
||||
+ except AttributeError:
|
||||
+ pass
|
||||
+
|
||||
def __repr__(self):
|
||||
result = ['<%s.%s' % (self.__class__.__module__,
|
||||
self.__class__.__qualname__)]
|
||||
@@ -963,6 +989,62 @@ def close(self):
|
||||
finally:
|
||||
super().close()
|
||||
|
||||
+ def seekable(self):
|
||||
+ return self._seekable
|
||||
+
|
||||
+ def seek(self, offset, whence=0):
|
||||
+ if not self._seekable:
|
||||
+ raise io.UnsupportedOperation("underlying stream is not seekable")
|
||||
+ curr_pos = self.tell()
|
||||
+ if whence == 0: # Seek from start of file
|
||||
+ new_pos = offset
|
||||
+ elif whence == 1: # Seek from current position
|
||||
+ new_pos = curr_pos + offset
|
||||
+ elif whence == 2: # Seek from EOF
|
||||
+ new_pos = self._orig_file_size + offset
|
||||
+ else:
|
||||
+ raise ValueError("whence must be os.SEEK_SET (0), "
|
||||
+ "os.SEEK_CUR (1), or os.SEEK_END (2)")
|
||||
+
|
||||
+ if new_pos > self._orig_file_size:
|
||||
+ new_pos = self._orig_file_size
|
||||
+
|
||||
+ if new_pos < 0:
|
||||
+ new_pos = 0
|
||||
+
|
||||
+ read_offset = new_pos - curr_pos
|
||||
+ buff_offset = read_offset + self._offset
|
||||
+
|
||||
+ if buff_offset >= 0 and buff_offset < len(self._readbuffer):
|
||||
+ # Just move the _offset index if the new position is in the _readbuffer
|
||||
+ self._offset = buff_offset
|
||||
+ read_offset = 0
|
||||
+ elif read_offset < 0:
|
||||
+ # Position is before the current position. Reset the ZipExtFile
|
||||
+
|
||||
+ self._fileobj.seek(self._orig_compress_start)
|
||||
+ self._running_crc = self._orig_start_crc
|
||||
+ self._compress_left = self._orig_compress_size
|
||||
+ self._left = self._orig_file_size
|
||||
+ self._readbuffer = b''
|
||||
+ self._offset = 0
|
||||
+ self._decompressor = zipfile._get_decompressor(self._compress_type)
|
||||
+ self._eof = False
|
||||
+ read_offset = new_pos
|
||||
+
|
||||
+ while read_offset > 0:
|
||||
+ read_len = min(self.MAX_SEEK_READ, read_offset)
|
||||
+ self.read(read_len)
|
||||
+ read_offset -= read_len
|
||||
+
|
||||
+ return self.tell()
|
||||
+
|
||||
+ def tell(self):
|
||||
+ if not self._seekable:
|
||||
+ raise io.UnsupportedOperation("underlying stream is not seekable")
|
||||
+ filepos = self._orig_file_size - self._left - len(self._readbuffer) + self._offset
|
||||
+ return filepos
|
||||
+
|
||||
|
||||
class _ZipWriteFile(io.BufferedIOBase):
|
||||
def __init__(self, zf, zinfo, zip64):
|
||||
diff --git a/Misc/NEWS.d/next/Library/2017-12-21-22-00-11.bpo-22908.cVm89I.rst b/Misc/NEWS.d/next/Library/2017-12-21-22-00-11.bpo-22908.cVm89I.rst
|
||||
new file mode 100644
|
||||
index 00000000000000..4f3cc0166019f1
|
||||
--- /dev/null
|
||||
+++ b/Misc/NEWS.d/next/Library/2017-12-21-22-00-11.bpo-22908.cVm89I.rst
|
||||
@@ -0,0 +1,2 @@
|
||||
+Added seek and tell to the ZipExtFile class. This only works if the file
|
||||
+object used to open the zipfile is seekable.
|
||||
|
||||
|
||||
From 55beb125db2942b5362454e05542e9661e964a65 Mon Sep 17 00:00:00 2001
|
||||
From: Serhiy Storchaka <storchaka@gmail.com>
|
||||
Date: Tue, 23 Apr 2024 14:29:31 +0200
|
||||
Subject: [PATCH] gh-109858: Protect zipfile from "quoted-overlap" zipbomb
|
||||
(GH-110016) (GH-113916)
|
||||
|
||||
Raise BadZipFile when try to read an entry that overlaps with other entry or
|
||||
central directory.
|
||||
(cherry picked from commit 66363b9a7b9fe7c99eba3a185b74c5fdbf842eba)
|
||||
---
|
||||
Lib/test/test_zipfile.py | 60 +++++++++++++++++++
|
||||
Lib/zipfile.py | 12 ++++
|
||||
...-09-28-13-15-51.gh-issue-109858.43e2dg.rst | 3 +
|
||||
3 files changed, 75 insertions(+)
|
||||
create mode 100644 Misc/NEWS.d/next/Library/2023-09-28-13-15-51.gh-issue-109858.43e2dg.rst
|
||||
|
||||
diff --git a/Lib/test/test_zipfile.py b/Lib/test/test_zipfile.py
|
||||
index 7f82586..0379909 100644
|
||||
--- a/Lib/test/test_zipfile.py
|
||||
+++ b/Lib/test/test_zipfile.py
|
||||
@@ -1644,6 +1644,66 @@ class OtherTests(unittest.TestCase):
|
||||
fp.seek(0, os.SEEK_END)
|
||||
self.assertEqual(fp.tell(), len(txt))
|
||||
|
||||
+ @requires_zlib
|
||||
+ def test_full_overlap(self):
|
||||
+ data = (
|
||||
+ b'PK\x03\x04\x14\x00\x00\x00\x08\x00\xa0lH\x05\xe2\x1e'
|
||||
+ b'8\xbb\x10\x00\x00\x00\t\x04\x00\x00\x01\x00\x00\x00a\xed'
|
||||
+ b'\xc0\x81\x08\x00\x00\x00\xc00\xd6\xfbK\\d\x0b`P'
|
||||
+ b'K\x01\x02\x14\x00\x14\x00\x00\x00\x08\x00\xa0lH\x05\xe2'
|
||||
+ b'\x1e8\xbb\x10\x00\x00\x00\t\x04\x00\x00\x01\x00\x00\x00\x00'
|
||||
+ b'\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00aPK'
|
||||
+ b'\x01\x02\x14\x00\x14\x00\x00\x00\x08\x00\xa0lH\x05\xe2\x1e'
|
||||
+ b'8\xbb\x10\x00\x00\x00\t\x04\x00\x00\x01\x00\x00\x00\x00\x00'
|
||||
+ b'\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00bPK\x05'
|
||||
+ b'\x06\x00\x00\x00\x00\x02\x00\x02\x00^\x00\x00\x00/\x00\x00'
|
||||
+ b'\x00\x00\x00'
|
||||
+ )
|
||||
+ with zipfile.ZipFile(io.BytesIO(data), 'r') as zipf:
|
||||
+ self.assertEqual(zipf.namelist(), ['a', 'b'])
|
||||
+ zi = zipf.getinfo('a')
|
||||
+ self.assertEqual(zi.header_offset, 0)
|
||||
+ self.assertEqual(zi.compress_size, 16)
|
||||
+ self.assertEqual(zi.file_size, 1033)
|
||||
+ zi = zipf.getinfo('b')
|
||||
+ self.assertEqual(zi.header_offset, 0)
|
||||
+ self.assertEqual(zi.compress_size, 16)
|
||||
+ self.assertEqual(zi.file_size, 1033)
|
||||
+ self.assertEqual(len(zipf.read('a')), 1033)
|
||||
+ with self.assertRaisesRegex(zipfile.BadZipFile, 'File name.*differ'):
|
||||
+ zipf.read('b')
|
||||
+
|
||||
+ @requires_zlib
|
||||
+ def test_quoted_overlap(self):
|
||||
+ data = (
|
||||
+ b'PK\x03\x04\x14\x00\x00\x00\x08\x00\xa0lH\x05Y\xfc'
|
||||
+ b'8\x044\x00\x00\x00(\x04\x00\x00\x01\x00\x00\x00a\x00'
|
||||
+ b'\x1f\x00\xe0\xffPK\x03\x04\x14\x00\x00\x00\x08\x00\xa0l'
|
||||
+ b'H\x05\xe2\x1e8\xbb\x10\x00\x00\x00\t\x04\x00\x00\x01\x00'
|
||||
+ b'\x00\x00b\xed\xc0\x81\x08\x00\x00\x00\xc00\xd6\xfbK\\'
|
||||
+ b'd\x0b`PK\x01\x02\x14\x00\x14\x00\x00\x00\x08\x00\xa0'
|
||||
+ b'lH\x05Y\xfc8\x044\x00\x00\x00(\x04\x00\x00\x01'
|
||||
+ b'\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00'
|
||||
+ b'\x00aPK\x01\x02\x14\x00\x14\x00\x00\x00\x08\x00\xa0l'
|
||||
+ b'H\x05\xe2\x1e8\xbb\x10\x00\x00\x00\t\x04\x00\x00\x01\x00'
|
||||
+ b'\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00$\x00\x00\x00'
|
||||
+ b'bPK\x05\x06\x00\x00\x00\x00\x02\x00\x02\x00^\x00\x00'
|
||||
+ b'\x00S\x00\x00\x00\x00\x00'
|
||||
+ )
|
||||
+ with zipfile.ZipFile(io.BytesIO(data), 'r') as zipf:
|
||||
+ self.assertEqual(zipf.namelist(), ['a', 'b'])
|
||||
+ zi = zipf.getinfo('a')
|
||||
+ self.assertEqual(zi.header_offset, 0)
|
||||
+ self.assertEqual(zi.compress_size, 52)
|
||||
+ self.assertEqual(zi.file_size, 1064)
|
||||
+ zi = zipf.getinfo('b')
|
||||
+ self.assertEqual(zi.header_offset, 36)
|
||||
+ self.assertEqual(zi.compress_size, 16)
|
||||
+ self.assertEqual(zi.file_size, 1033)
|
||||
+ with self.assertRaisesRegex(zipfile.BadZipFile, 'Overlapped entries'):
|
||||
+ zipf.read('a')
|
||||
+ self.assertEqual(len(zipf.read('b')), 1033)
|
||||
+
|
||||
def tearDown(self):
|
||||
unlink(TESTFN)
|
||||
unlink(TESTFN2)
|
||||
diff --git a/Lib/zipfile.py b/Lib/zipfile.py
|
||||
index 0ab9fac..e6d7676 100644
|
||||
--- a/Lib/zipfile.py
|
||||
+++ b/Lib/zipfile.py
|
||||
@@ -338,6 +338,7 @@ class ZipInfo (object):
|
||||
'compress_size',
|
||||
'file_size',
|
||||
'_raw_time',
|
||||
+ '_end_offset',
|
||||
)
|
||||
|
||||
def __init__(self, filename="NoName", date_time=(1980,1,1,0,0,0)):
|
||||
@@ -376,6 +377,7 @@ class ZipInfo (object):
|
||||
self.volume = 0 # Volume number of file header
|
||||
self.internal_attr = 0 # Internal attributes
|
||||
self.external_attr = 0 # External file attributes
|
||||
+ self._end_offset = None # Start of the next local header or central directory
|
||||
# Other attributes are set by class ZipFile:
|
||||
# header_offset Byte offset to the file header
|
||||
# CRC CRC-32 of the uncompressed file
|
||||
@@ -1346,6 +1348,12 @@ class ZipFile:
|
||||
if self.debug > 2:
|
||||
print("total", total)
|
||||
|
||||
+ end_offset = self.start_dir
|
||||
+ for zinfo in sorted(self.filelist,
|
||||
+ key=lambda zinfo: zinfo.header_offset,
|
||||
+ reverse=True):
|
||||
+ zinfo._end_offset = end_offset
|
||||
+ end_offset = zinfo.header_offset
|
||||
|
||||
def namelist(self):
|
||||
"""Return a list of file names in the archive."""
|
||||
@@ -1500,6 +1508,10 @@ class ZipFile:
|
||||
'File name in directory %r and header %r differ.'
|
||||
% (zinfo.orig_filename, fname))
|
||||
|
||||
+ if (zinfo._end_offset is not None and
|
||||
+ zef_file.tell() + zinfo.compress_size > zinfo._end_offset):
|
||||
+ raise BadZipFile(f"Overlapped entries: {zinfo.orig_filename!r} (possible zip bomb)")
|
||||
+
|
||||
# check for encrypted flag & handle password
|
||||
is_encrypted = zinfo.flag_bits & 0x1
|
||||
zd = None
|
||||
diff --git a/Misc/NEWS.d/next/Library/2023-09-28-13-15-51.gh-issue-109858.43e2dg.rst b/Misc/NEWS.d/next/Library/2023-09-28-13-15-51.gh-issue-109858.43e2dg.rst
|
||||
new file mode 100644
|
||||
index 0000000..be279ca
|
||||
--- /dev/null
|
||||
+++ b/Misc/NEWS.d/next/Library/2023-09-28-13-15-51.gh-issue-109858.43e2dg.rst
|
||||
@@ -0,0 +1,3 @@
|
||||
+Protect :mod:`zipfile` from "quoted-overlap" zipbomb. It now raises
|
||||
+BadZipFile when try to read an entry that overlaps with other entry or
|
||||
+central directory.
|
||||
--
|
||||
2.44.0
|
||||
|
|
@ -14,7 +14,7 @@ URL: https://www.python.org/
|
|||
# WARNING When rebasing to a new Python version,
|
||||
# remember to update the python3-docs package as well
|
||||
Version: %{pybasever}.8
|
||||
Release: 23%{?dist}
|
||||
Release: 62%{?dist}
|
||||
License: Python
|
||||
|
||||
|
||||
|
@ -56,6 +56,15 @@ License: Python
|
|||
%bcond_with valgrind
|
||||
%endif
|
||||
|
||||
# https://fedoraproject.org/wiki/Changes/Python_Upstream_Architecture_Names
|
||||
# For a very long time we have converted "upstream architecture names" to "Fedora names".
|
||||
# This made sense at the time, see https://github.com/pypa/manylinux/issues/687#issuecomment-666362947
|
||||
# However, with manylinux wheels popularity growth, this is now a problem.
|
||||
# Wheels built on a Linux that doesn't do this were not compatible with ours and vice versa.
|
||||
# We now have a compatibility layer to workaround a problem,
|
||||
# but we also no longer use the legacy arch names in Fedora 34+.
|
||||
# This bcond controls the behavior. The defaults should be good for anybody.
|
||||
%bcond_without legacy_archnames
|
||||
|
||||
# ==================================
|
||||
# Notes from bootstraping Python 3.6
|
||||
|
@ -111,8 +120,21 @@ License: Python
|
|||
%global LDVERSION_optimized %{pybasever}%{ABIFLAGS_optimized}
|
||||
%global LDVERSION_debug %{pybasever}%{ABIFLAGS_debug}
|
||||
|
||||
%global SOABI_optimized cpython-%{pyshortver}%{ABIFLAGS_optimized}-%{_arch}-linux%{_gnu}
|
||||
%global SOABI_debug cpython-%{pyshortver}%{ABIFLAGS_debug}-%{_arch}-linux%{_gnu}
|
||||
# When we use the upstream arch triplets, we convert them from the legacy ones
|
||||
# This is reversed in prep when %%with legacy_archnames, so we keep both macros
|
||||
%global platform_triplet_legacy %{_arch}-linux%{_gnu}
|
||||
%global platform_triplet_upstream %{expand:%(echo %{platform_triplet_legacy} | sed -E \\
|
||||
-e 's/^arm(eb)?-linux-gnueabi$/arm\\1-linux-gnueabihf/' \\
|
||||
-e 's/^mips64(el)?-linux-gnu$/mips64\\1-linux-gnuabi64/' \\
|
||||
-e 's/^ppc(64)?(le)?-linux-gnu$/powerpc\\1\\2-linux-gnu/')}
|
||||
%if %{with legacy_archnames}
|
||||
%global platform_triplet %{platform_triplet_legacy}
|
||||
%else
|
||||
%global platform_triplet %{platform_triplet_upstream}
|
||||
%endif
|
||||
|
||||
%global SOABI_optimized cpython-%{pyshortver}%{ABIFLAGS_optimized}-%{platform_triplet}
|
||||
%global SOABI_debug cpython-%{pyshortver}%{ABIFLAGS_debug}-%{platform_triplet}
|
||||
|
||||
# All bytecode files are in a __pycache__ subdirectory, with a name
|
||||
# reflecting the version of the bytecode.
|
||||
|
@ -170,6 +192,7 @@ BuildRequires: gcc-c++
|
|||
%if %{with gdbm}
|
||||
BuildRequires: gdbm-devel >= 1:1.13
|
||||
%endif
|
||||
BuildRequires: git-core
|
||||
BuildRequires: glibc-devel
|
||||
BuildRequires: gmp-devel
|
||||
BuildRequires: libappstream-glib
|
||||
|
@ -309,22 +332,11 @@ Patch163: 00163-disable-parts-of-test_socket-in-rpm-build.patch
|
|||
# See https://bugzilla.redhat.com/show_bug.cgi?id=614680
|
||||
Patch170: 00170-gc-assertions.patch
|
||||
|
||||
# 00178 #
|
||||
# Don't duplicate various FLAGS in sysconfig values
|
||||
# http://bugs.python.org/issue17679
|
||||
# Does not affect python2 AFAICS (different sysconfig values initialization)
|
||||
Patch178: 00178-dont-duplicate-flags-in-sysconfig.patch
|
||||
|
||||
# 00189 #
|
||||
# Instead of bundled wheels, use our RPM packaged wheels from
|
||||
# /usr/share/python3-wheels
|
||||
Patch189: 00189-use-rpm-wheels.patch
|
||||
|
||||
# 00205 #
|
||||
# LIBPL variable in makefile takes LIBPL from configure.ac
|
||||
# but the LIBPL variable defined there doesn't respect libdir macro
|
||||
Patch205: 00205-make-libpl-respect-lib64.patch
|
||||
|
||||
# 00251
|
||||
# Set values of prefix and exec_prefix in distutils install command
|
||||
# to /usr/local if executable is /usr/bin/python* and RPM build
|
||||
|
@ -332,6 +344,16 @@ Patch205: 00205-make-libpl-respect-lib64.patch
|
|||
# Fedora Change: https://fedoraproject.org/wiki/Changes/Making_sudo_pip_safe
|
||||
Patch251: 00251-change-user-install-location.patch
|
||||
|
||||
# 00257 #
|
||||
# Use the monotonic clock for threading.Condition.wait() as to not be affected by
|
||||
# the system clock changes.
|
||||
# This patch works around the issue.
|
||||
# Implemented by backporting the respective python2 code:
|
||||
# https://github.com/python/cpython/blob/v2.7.18/Lib/threading.py#L331
|
||||
# along with our downstream patch for python2 fixing the same issue.
|
||||
# Downstream only.
|
||||
Patch257: 00257-threading-condition-wait.patch
|
||||
|
||||
# 00262 #
|
||||
# Backport of PEP 538: Coercing the legacy C locale to a UTF-8 based locale
|
||||
# https://www.python.org/dev/peps/pep-0538/
|
||||
|
@ -339,10 +361,6 @@ Patch251: 00251-change-user-install-location.patch
|
|||
# Original proposal: https://bugzilla.redhat.com/show_bug.cgi?id=1404918
|
||||
Patch262: 00262-pep538_coerce_legacy_c_locale.patch
|
||||
|
||||
# 00274 #
|
||||
# Upstream uses Debian-style architecture naming. Change to match Fedora.
|
||||
Patch274: 00274-fix-arch-names.patch
|
||||
|
||||
# 00294 #
|
||||
# Define TLS cipher suite on build time depending
|
||||
# on the OpenSSL default cipher suite selection.
|
||||
|
@ -488,6 +506,374 @@ Patch332: 00332-CVE-2019-16056.patch
|
|||
# Resolves: https://bugzilla.redhat.com/show_bug.cgi?id=1749576
|
||||
Patch333: 00333-reduce-pgo-tests.patch
|
||||
|
||||
# 00338 #
|
||||
# Fix test_gdb for when compiling python with Link Time Optimizations
|
||||
# Fixed upstream: https://bugs.python.org/issue38239
|
||||
Patch338: 00338-fix-test_gdb-for-LTO.patch
|
||||
|
||||
# 00344 #
|
||||
# Fix CVE-2019-16935: XSS vulnerability in the ocumentation XML-RPC server in server_title field
|
||||
# Fixed upstream: https://bugs.python.org/issue38243
|
||||
# Resolves: https://bugzilla.redhat.com/how_bug.cgi?id=1798001
|
||||
Patch344: 00344-CVE-2019-16935.patch
|
||||
|
||||
# 00345 #
|
||||
# Skip from test_site the test_startup_imports case if a path of sys.path
|
||||
# contains a .pth file
|
||||
# Fixed upstream: https://bugs.python.org/issue27807
|
||||
# Resolves: https://bugzilla.redhat.com/show_bug.cgi?id=1814392
|
||||
Patch345: 00345-fix-test_site-with-extra-pth-files.patch
|
||||
|
||||
# 00346 #
|
||||
# Fix CVE-2020-8492: wrong backtracking in urllib.request.AbstractBasicAuthHandler allows for a ReDoS
|
||||
# Fixed upstream: https://bugs.python.org/issue39503
|
||||
# Resolves: https://bugzilla.redhat.com/show_bug.cgi?id=1810618
|
||||
Patch346: 00346-CVE-2020-8492.patch
|
||||
|
||||
# 00351 #
|
||||
# Avoid infinite loop when reading specially crafted TAR files using the tarfile module
|
||||
# (CVE-2019-20907).
|
||||
# See: https://bugs.python.org/issue39017
|
||||
Patch351: 00351-avoid-infinite-loop-in-the-tarfile-module.patch
|
||||
|
||||
# 00352 # 5253c417a23b3658fa115d2c72fa54b20293a31c
|
||||
# Resolve hash collisions for IPv4Interface and IPv6Interface
|
||||
#
|
||||
# CVE-2020-14422
|
||||
# The hash() methods of classes IPv4Interface and IPv6Interface had issue
|
||||
# of generating constant hash values of 32 and 128 respectively causing hash collisions.
|
||||
# The fix uses the hash() function to generate hash values for the objects
|
||||
# instead of XOR operation.
|
||||
# Fixed upstream: https://bugs.python.org/issue41004
|
||||
Patch352: 00352-resolve-hash-collisions-for-ipv4interface-and-ipv6interface.patch
|
||||
|
||||
# 00353 #
|
||||
# Original names for architectures with different names downstream
|
||||
#
|
||||
# https://fedoraproject.org/wiki/Changes/Python_Upstream_Architecture_Names
|
||||
#
|
||||
# Pythons in RHEL/Fedora used different names for some architectures
|
||||
# than upstream and other distros (for example ppc64 vs. powerpc64).
|
||||
# This was patched in patch 274, now it is sedded if %%with legacy_archnames.
|
||||
#
|
||||
# That meant that an extension built with the default upstream settings
|
||||
# (on other distro or as an manylinux wheel) could not been found by Python
|
||||
# on RHEL/Fedora because it had a different suffix.
|
||||
# This patch adds the legacy names to importlib so Python is able
|
||||
# to import extensions with a legacy architecture name in its
|
||||
# file name.
|
||||
# It work both ways, so it support both %%with and %%without legacy_archnames.
|
||||
#
|
||||
# WARNING: This patch has no effect on Python built with bootstrap
|
||||
# enabled because Python/importlib_external.h is not regenerated
|
||||
# and therefore Python during bootstrap contains importlib from
|
||||
# upstream without this feature. It's possible to include
|
||||
# Python/importlib_external.h to this patch but it'd make rebasing
|
||||
# a nightmare because it's basically a binary file.
|
||||
Patch353: 00353-architecture-names-upstream-downstream.patch
|
||||
|
||||
# 00354 #
|
||||
# Reject control chars in HTTP method in http.client to prevent
|
||||
# HTTP header injection
|
||||
# Fixed ustream: https://bugs.python.org/issue39603
|
||||
Patch354: 00354-cve-2020-26116-http-request-method-crlf-injection-in-httplib.patch
|
||||
|
||||
# 00355 #
|
||||
# No longer call eval() on content received via HTTP in the CJK codec tests
|
||||
# Fixed upstream: https://bugs.python.org/issue41944
|
||||
Patch355: 00355-CVE-2020-27619.patch
|
||||
|
||||
# 00356 #
|
||||
# options -a and -k for pathfix.py used in %%py3_shebang_fix
|
||||
# Upstream: https://github.com/python/cpython/commit/c71c54c62600fd721baed3c96709e3d6e9c33817
|
||||
Patch356: 00356-k_and_a_options_for_pathfix.patch
|
||||
|
||||
# 00357 #
|
||||
# CVE-2021-3177 stack-based buffer overflow in PyCArg_repr in _ctypes/callproc.c
|
||||
# Upstream: https://bugs.python.org/issue42938
|
||||
# Main BZ: https://bugzilla.redhat.com/show_bug.cgi?id=1918168
|
||||
Patch357: 00357-CVE-2021-3177.patch
|
||||
|
||||
# 00359 #
|
||||
# CVE-2021-23336 python: Web Cache Poisoning via urllib.parse.parse_qsl and
|
||||
# urllib.parse.parse_qs by using a semicolon in query parameters
|
||||
# Upstream: https://bugs.python.org/issue42967
|
||||
# Main BZ: https://bugzilla.redhat.com/show_bug.cgi?id=1928904
|
||||
Patch359: 00359-CVE-2021-23336.patch
|
||||
|
||||
# 00360 #
|
||||
# CVE-2021-3426: information disclosure via pydoc
|
||||
# Upstream: https://bugs.python.org/issue42988
|
||||
# Main BZ: https://bugzilla.redhat.com/show_bug.cgi?id=1935913
|
||||
Patch360: 00360-CVE-2021-3426.patch
|
||||
|
||||
# 00362 #
|
||||
# The threading.enumerate() function now uses a reentrant lock to
|
||||
# prevent a hang on reentrant call.
|
||||
# Upstream: https://bugs.python.org/issue44422
|
||||
# Main BZ: https://bugzilla.redhat.com/show_bug.cgi?id=1959459
|
||||
Patch362: 00362-threading-enumerate-rlock.patch
|
||||
|
||||
# 00364 #
|
||||
# Don't call PyThread_exit_thread() explicitly.
|
||||
# Upstream: https://bugs.python.org/issue44434
|
||||
# Main BZ: https://bugzilla.redhat.com/show_bug.cgi?id=1972293
|
||||
Patch364: 00364-thread-exit.patch
|
||||
|
||||
# 00366 #
|
||||
# CVE-2021-3733: Denial of service when identifying crafted invalid RFCs
|
||||
# Upstream: https://bugs.python.org/issue43075
|
||||
# Tracking bug: https://bugzilla.redhat.com/show_bug.cgi?id=1995234
|
||||
Patch366: 00366-CVE-2021-3733.patch
|
||||
|
||||
# 00368 #
|
||||
# CVE-2021-3737: client can enter an infinite loop on a 100 Continue response from the server
|
||||
# Upstream: https://bugs.python.org/issue44022
|
||||
# Tracking bug: https://bugzilla.redhat.com/show_bug.cgi?id=1995162
|
||||
Patch368: 00368-CVE-2021-3737.patch
|
||||
|
||||
# 00369 #
|
||||
# Change shouldRollover() methods of logging.handlers to only rollover regular files and not devices
|
||||
# Upstream: https://bugs.python.org/issue45401
|
||||
# Bugzilla: https://bugzilla.redhat.com/show_bug.cgi?id=2009200
|
||||
Patch369: 00369-rollover-only-regular-files-in-logging-handlers.patch
|
||||
|
||||
# 00370 #
|
||||
# Utilize the monotonic clock for the global interpreter lock instead of the real-time clock
|
||||
# to avoid issues when the system time changes
|
||||
# Upstream: https://bugs.python.org/issue12822
|
||||
# Bugzilla: https://bugzilla.redhat.com/show_bug.cgi?id=2003758
|
||||
Patch370: 00370-GIL-monotonic-clock.patch
|
||||
|
||||
# 00372 #
|
||||
# CVE-2021-4189: ftplib should not use the host from the PASV response
|
||||
# Upstream: https://bugs.python.org/issue43285
|
||||
# Tracking bug: https://bugzilla.redhat.com/show_bug.cgi?id=2036020
|
||||
Patch372: 00372-CVE-2021-4189.patch
|
||||
|
||||
# 00377 #
|
||||
# CVE-2022-0391: urlparse does not sanitize URLs containing ASCII newline and tabs
|
||||
#
|
||||
# ASCII newline and tab characters are stripped from the URL.
|
||||
#
|
||||
# Upstream: https://bugs.python.org/issue43882
|
||||
# Tracking bug: https://bugzilla.redhat.com/show_bug.cgi?id=2047376
|
||||
Patch377: 00377-CVE-2022-0391.patch
|
||||
|
||||
# 00378 #
|
||||
# Support expat 2.4.5
|
||||
#
|
||||
# Curly brackets were never allowed in namespace URIs
|
||||
# according to RFC 3986, and so-called namespace-validating
|
||||
# XML parsers have the right to reject them a invalid URIs.
|
||||
#
|
||||
# libexpat >=2.4.5 has become strcter in that regard due to
|
||||
# related security issues; with ET.XML instantiating a
|
||||
# namespace-aware parser under the hood, this test has no
|
||||
# future in CPython.
|
||||
#
|
||||
# References:
|
||||
# - https://datatracker.ietf.org/doc/html/rfc3968
|
||||
# - https://www.w3.org/TR/xml-names/
|
||||
#
|
||||
# Also, test_minidom.py: Support Expat >=2.4.5
|
||||
#
|
||||
# The patch has diverged from upstream as the python test
|
||||
# suite was relying on checking the expat version, whereas
|
||||
# in RHEL fixes get backported instead of rebasing packages.
|
||||
#
|
||||
# Upstream: https://bugs.python.org/issue46811
|
||||
Patch378: 00378-support-expat-2-4-5.patch
|
||||
|
||||
# 00382 #
|
||||
# CVE-2015-20107
|
||||
#
|
||||
# Make mailcap refuse to match unsafe filenames/types/params (GH-91993)
|
||||
#
|
||||
# Upstream: https://github.com/python/cpython/issues/68966
|
||||
#
|
||||
# Tracker bug: https://bugzilla.redhat.com/show_bug.cgi?id=2075390
|
||||
Patch382: 00382-cve-2015-20107.patch
|
||||
|
||||
# 00386 #
|
||||
# CVE-2021-28861
|
||||
#
|
||||
# Fix an open redirection vulnerability in the `http.server` module when
|
||||
# an URI path starts with `//` that could produce a 301 Location header
|
||||
# with a misleading target. Vulnerability discovered, and logic fix
|
||||
# proposed, by Hamza Avvan (@hamzaavvan).
|
||||
#
|
||||
# Test and comments authored by Gregory P. Smith [Google].
|
||||
#
|
||||
# Upstream: https://github.com/python/cpython/pull/93879
|
||||
# Tracking bugzilla: https://bugzilla.redhat.com/show_bug.cgi?id=2120642
|
||||
Patch386: 00386-cve-2021-28861.patch
|
||||
|
||||
# 00387 #
|
||||
# CVE-2020-10735: Prevent DoS by very large int()
|
||||
#
|
||||
# gh-95778: CVE-2020-10735: Prevent DoS by very large int() (GH-96504)
|
||||
#
|
||||
# Converting between `int` and `str` in bases other than 2
|
||||
# (binary), 4, 8 (octal), 16 (hexadecimal), or 32 such as base 10 (decimal) now
|
||||
# raises a `ValueError` if the number of digits in string form is above a
|
||||
# limit to avoid potential denial of service attacks due to the algorithmic
|
||||
# complexity. This is a mitigation for CVE-2020-10735
|
||||
# (https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2020-10735).
|
||||
#
|
||||
# This new limit can be configured or disabled by environment variable, command
|
||||
# line flag, or :mod:`sys` APIs. See the `Integer String Conversion Length
|
||||
# Limitation` documentation. The default limit is 4300
|
||||
# digits in string form.
|
||||
#
|
||||
# Patch by Gregory P. Smith [Google] and Christian Heimes [Red Hat] with feedback
|
||||
# from Victor Stinner, Thomas Wouters, Steve Dower, Ned Deily, and Mark Dickinson.
|
||||
#
|
||||
# Notes on the backport to Python 3.6 in RHEL:
|
||||
#
|
||||
# * Use "Python 3.6.8-48" version in the documentation, whereas this
|
||||
# version will never be released
|
||||
# * Only add _Py_global_config_int_max_str_digits global variable:
|
||||
# Python 3.6 doesn't have PyConfig API (PEP 597) nor _PyRuntime.
|
||||
# * sys.flags.int_max_str_digits cannot be -1 on Python 3.6: it is
|
||||
# set to the default limit. Adapt test_int_max_str_digits() for that.
|
||||
# * Declare _PY_LONG_DEFAULT_MAX_STR_DIGITS and
|
||||
# _PY_LONG_MAX_STR_DIGITS_THRESHOLD macros in longobject.h but only
|
||||
# if the Py_BUILD_CORE macro is defined.
|
||||
# * Declare _Py_global_config_int_max_str_digits in pydebug.h.
|
||||
#
|
||||
#
|
||||
# gh-95778: Mention sys.set_int_max_str_digits() in error message (#96874)
|
||||
#
|
||||
# When ValueError is raised if an integer is larger than the limit,
|
||||
# mention sys.set_int_max_str_digits() in the error message.
|
||||
#
|
||||
#
|
||||
# gh-96848: Fix -X int_max_str_digits option parsing (#96988)
|
||||
#
|
||||
# Fix command line parsing: reject "-X int_max_str_digits" option with
|
||||
# no value (invalid) when the PYTHONINTMAXSTRDIGITS environment
|
||||
# variable is set to a valid limit.
|
||||
Patch387: 00387-cve-2020-10735-prevent-dos-by-very-large-int.patch
|
||||
|
||||
# 00394 #
|
||||
# CVE-2022-45061: CPU denial of service via inefficient IDNA decoder
|
||||
#
|
||||
# gh-98433: Fix quadratic time idna decoding.
|
||||
#
|
||||
# There was an unnecessary quadratic loop in idna decoding. This restores
|
||||
# the behavior to linear.
|
||||
Patch394: 00394-cve-2022-45061-cpu-denial-of-service-via-inefficient-idna-decoder.patch
|
||||
|
||||
# 00397 #
|
||||
# Add filters for tarfile extraction (CVE-2007-4559, PEP-706)
|
||||
# The first patches in the file backport the upstream fix:
|
||||
# - https://github.com/python/cpython/pull/104583
|
||||
# (see the linked issue for merged backports)
|
||||
# Next-to-last patch fixes determination of symlink targets, which were treated
|
||||
# as relative to the root of the archive,
|
||||
# rather than the directory containing the symlink.
|
||||
# Not yet upstream as of this writing.
|
||||
# The last patch is Red Hat configuration, see KB for documentation:
|
||||
# - https://access.redhat.com/articles/7004769
|
||||
Patch397: 00397-tarfile-filter.patch
|
||||
|
||||
# 00399 #
|
||||
# CVE-2023-24329
|
||||
#
|
||||
# gh-102153: Start stripping C0 control and space chars in `urlsplit` (GH-102508)
|
||||
#
|
||||
# `urllib.parse.urlsplit` has already been respecting the WHATWG spec a bit GH-25595.
|
||||
#
|
||||
# This adds more sanitizing to respect the "Remove any leading C0 control or space from input" [rule](https://url.spec.whatwg.org/GH-url-parsing:~:text=Remove%%20any%%20leading%%20and%%20trailing%%20C0%%20control%%20or%%20space%%20from%%20input.) in response to [CVE-2023-24329](https://nvd.nist.gov/vuln/detail/CVE-2023-24329).
|
||||
#
|
||||
# Backported from Python 3.12
|
||||
Patch399: 00399-cve-2023-24329.patch
|
||||
|
||||
# 00404 #
|
||||
# CVE-2023-40217
|
||||
#
|
||||
# Security fix for CVE-2023-40217: Bypass TLS handshake on closed sockets
|
||||
# Resolved upstream: https://github.com/python/cpython/issues/108310
|
||||
# Fixups added on top:
|
||||
# https://github.com/python/cpython/pull/108352
|
||||
# https://github.com/python/cpython/pull/108408
|
||||
#
|
||||
# Backported from Python 3.8
|
||||
Patch404: 00404-cve-2023-40217.patch
|
||||
|
||||
# 00408 #
|
||||
# CVE-2022-48560
|
||||
#
|
||||
# Security fix for CVE-2022-48560: python3: use after free in heappushpop()
|
||||
# of heapq module
|
||||
# Resolved upstream: https://github.com/python/cpython/issues/83602
|
||||
Patch408: 00408-CVE-2022-48560.patch
|
||||
|
||||
# 00413 #
|
||||
# CVE-2022-48564
|
||||
#
|
||||
# DoS when processing malformed Apple Property List files in binary format
|
||||
# Resolved upstream: https://github.com/python/cpython/commit/a63234c49b2fbfb6f0aca32525e525ce3d43b2b4
|
||||
Patch413: 00413-CVE-2022-48564.patch
|
||||
|
||||
# 00414 #
|
||||
#
|
||||
# Skip test_pair() and test_speech128() of test_zlib on s390x since
|
||||
# they fail if zlib uses the s390x hardware accelerator.
|
||||
Patch414: 00414-skip_test_zlib_s390x.patch
|
||||
|
||||
# 00415 #
|
||||
# [CVE-2023-27043] gh-102988: Reject malformed addresses in email.parseaddr() (#111116)
|
||||
#
|
||||
# Detect email address parsing errors and return empty tuple to
|
||||
# indicate the parsing error (old API). Add an optional 'strict'
|
||||
# parameter to getaddresses() and parseaddr() functions. Patch by
|
||||
# Thomas Dwyer.
|
||||
#
|
||||
# Upstream PR: https://github.com/python/cpython/pull/111116
|
||||
#
|
||||
# Second patch implmenets the possibility to restore the old behavior via
|
||||
# config file or environment variable.
|
||||
Patch415: 00415-cve-2023-27043-gh-102988-reject-malformed-addresses-in-email-parseaddr-111116.patch
|
||||
|
||||
# 00422 #
|
||||
# gh-115133: Fix tests for XMLPullParser with Expat 2.6.0
|
||||
#
|
||||
# Feeding the parser by too small chunks defers parsing to prevent
|
||||
# CVE-2023-52425. Future versions of Expat may be more reactive.
|
||||
#
|
||||
# Patch rebased because the CVE fix is backported to older expat in RHEL.
|
||||
Patch422: 00422-gh-115133-fix-tests-for-xmlpullparser-with-expat-2-6-0.patch
|
||||
|
||||
# 426 #
|
||||
# CVE-2023-6597
|
||||
#
|
||||
# Path traversal on tempfile.TemporaryDirectory
|
||||
#
|
||||
# Upstream: https://github.com/python/cpython/issues/91133
|
||||
# Tracking bug: https://bugzilla.redhat.com/show_bug.cgi?id=CVE-2023-6597
|
||||
#
|
||||
# To backport the fix cleanly the patch contains also this rebased commit:
|
||||
# Fix permission errors in TemporaryDirectory cleanup
|
||||
# https://github.com/python/cpython/commit/e9b51c0ad81da1da11ae65840ac8b50a8521373c
|
||||
Patch426: 00426-CVE-2023-6597.patch
|
||||
|
||||
# 427 #
|
||||
# CVE-2024-0450
|
||||
#
|
||||
# The zipfile module is vulnerable to zip-bombs leading to denial of service.
|
||||
#
|
||||
# Upstream: https://github.com/python/cpython/issues/109858
|
||||
# Tracking bug: https://bugzilla.redhat.com/show_bug.cgi?id=CVE-2024-0450
|
||||
#
|
||||
# To backport the fix cleanly also this change is backported:
|
||||
# Add seek and tell functionality to ZipExtFile
|
||||
# https://github.com/python/cpython/commit/066df4fd454d6ff9be66e80b2a65995b10af174f
|
||||
#
|
||||
# Patch rebased from 3.8.
|
||||
Patch427: 00427-CVE-2024-0450.patch
|
||||
|
||||
# (New patches go here ^^^)
|
||||
#
|
||||
# When adding new patches to "python" and "python3" in Fedora, EL, etc.,
|
||||
|
@ -534,10 +920,10 @@ Requires: python3-setuptools-wheel
|
|||
Requires: python3-pip-wheel
|
||||
%endif
|
||||
|
||||
# Runtime require alternatives
|
||||
Requires: %{_sbindir}/alternatives
|
||||
Requires(post): %{_sbindir}/alternatives
|
||||
Requires(postun): %{_sbindir}/alternatives
|
||||
# Require alternatives version that implements the --keep-foreign flag
|
||||
Requires: alternatives >= 1.19.1-1
|
||||
Requires(post): alternatives >= 1.19.1-1
|
||||
Requires(postun): alternatives >= 1.19.1-1
|
||||
|
||||
# This prevents ALL subpackages built from this spec to require
|
||||
# /usr/bin/python3*. Granularity per subpackage is impossible.
|
||||
|
@ -678,6 +1064,9 @@ Provides: %{name}-tools = %{version}-%{release}
|
|||
Provides: %{name}-tools%{?_isa} = %{version}-%{release}
|
||||
Obsoletes: %{name}-tools < %{version}-%{release}
|
||||
|
||||
|
||||
# Require alternatives version that implements the --keep-foreign flag
|
||||
Requires(postun): alternatives >= 1.19.1-1
|
||||
# python36 installs the alternatives master symlink to which we attach a slave
|
||||
Requires: python36
|
||||
Requires(post): python36
|
||||
|
@ -699,6 +1088,7 @@ configuration, browsers, and other dialogs.
|
|||
%package tkinter
|
||||
Summary: A GUI toolkit for Python
|
||||
Requires: platform-python = %{version}-%{release}
|
||||
Requires: %{name}-libs%{?_isa} = %{version}-%{release}
|
||||
|
||||
%description tkinter
|
||||
The Tkinter (Tk interface) library is a graphical user interface toolkit for
|
||||
|
@ -778,17 +1168,15 @@ rm -r Modules/zlib
|
|||
%patch160 -p1
|
||||
%patch163 -p1
|
||||
%patch170 -p1
|
||||
%patch178 -p1
|
||||
|
||||
%if %{with rpmwheels}
|
||||
%patch189 -p1
|
||||
rm Lib/ensurepip/_bundled/*.whl
|
||||
%endif
|
||||
|
||||
%patch205 -p1
|
||||
%patch251 -p1
|
||||
%patch257 -p1
|
||||
%patch262 -p1
|
||||
%patch274 -p1
|
||||
%patch294 -p1
|
||||
%patch316 -p1
|
||||
%patch317 -p1
|
||||
|
@ -803,12 +1191,56 @@ rm Lib/ensurepip/_bundled/*.whl
|
|||
%patch330 -p1
|
||||
%patch332 -p1
|
||||
%patch333 -p1
|
||||
%patch338 -p1
|
||||
%patch344 -p1
|
||||
%patch345 -p1
|
||||
%patch346 -p1
|
||||
|
||||
# Patch 351 adds binary file for testing. We need to apply it using Git.
|
||||
git apply %{PATCH351}
|
||||
|
||||
%patch352 -p1
|
||||
%patch353 -p1
|
||||
%patch354 -p1
|
||||
%patch355 -p1
|
||||
%patch356 -p1
|
||||
%patch357 -p1
|
||||
%patch359 -p1
|
||||
%patch360 -p1
|
||||
%patch362 -p1
|
||||
%patch364 -p1
|
||||
%patch366 -p1
|
||||
%patch368 -p1
|
||||
%patch369 -p1
|
||||
%patch370 -p1
|
||||
%patch372 -p1
|
||||
%patch377 -p1
|
||||
%patch378 -p1
|
||||
%patch382 -p1
|
||||
%patch386 -p1
|
||||
%patch387 -p1
|
||||
%patch394 -p1
|
||||
%patch397 -p1
|
||||
%patch399 -p1
|
||||
%patch404 -p1
|
||||
%patch408 -p1
|
||||
%patch413 -p1
|
||||
%patch414 -p1
|
||||
%patch415 -p1
|
||||
%patch422 -p1
|
||||
%patch426 -p1
|
||||
%patch427 -p1
|
||||
|
||||
# Remove files that should be generated by the build
|
||||
# (This is after patching, so that we can use patches directly from upstream)
|
||||
rm configure pyconfig.h.in
|
||||
|
||||
# When we use the legacy arch names, we need to change them in configure.ac
|
||||
%if %{with legacy_archnames}
|
||||
sed -i configure.ac \
|
||||
-e 's/\b%{platform_triplet_upstream}\b/%{platform_triplet_legacy}/'
|
||||
%endif
|
||||
|
||||
|
||||
# ======================================================
|
||||
# Configuring and building the code:
|
||||
|
@ -845,14 +1277,14 @@ topdir=$(pwd)
|
|||
# RHEL packages utilizing %%py3_build will use them as well
|
||||
# https://fedoraproject.org/wiki/Changes/Python_Extension_Flags
|
||||
export CFLAGS="%{extension_cflags} -D_GNU_SOURCE -fPIC -fwrapv"
|
||||
export CFLAGS_NODIST="%{build_cflags} -D_GNU_SOURCE -fPIC -fwrapv"
|
||||
export CFLAGS_NODIST="%{build_cflags} -D_GNU_SOURCE -fPIC -fwrapv -fno-semantic-interposition"
|
||||
export CXXFLAGS="%{extension_cxxflags} -D_GNU_SOURCE -fPIC -fwrapv"
|
||||
export CPPFLAGS="$(pkg-config --cflags-only-I libffi)"
|
||||
export OPT="%{extension_cflags} -D_GNU_SOURCE -fPIC -fwrapv"
|
||||
export LINKCC="gcc"
|
||||
export CFLAGS="$CFLAGS $(pkg-config --cflags openssl)"
|
||||
export LDFLAGS="%{extension_ldflags} -g $(pkg-config --libs-only-L openssl)"
|
||||
export LDFLAGS_NODIST="%{build_ldflags} -g $(pkg-config --libs-only-L openssl)"
|
||||
export LDFLAGS_NODIST="%{build_ldflags} -fno-semantic-interposition -g $(pkg-config --libs-only-L openssl)"
|
||||
|
||||
# We can build several different configurations of Python: regular and debug.
|
||||
# Define a common function that does one build:
|
||||
|
@ -889,6 +1321,9 @@ BuildPython() {
|
|||
$ExtraConfigArgs \
|
||||
%{nil}
|
||||
|
||||
# Regenerate generated importlib frozen modules (see patch 353)
|
||||
%make_build CFLAGS_NODIST="$CFLAGS_NODIST $MoreCFlags" regen-importlib
|
||||
|
||||
# Invoke the build
|
||||
%make_build CFLAGS_NODIST="$CFLAGS_NODIST $MoreCFlags"
|
||||
|
||||
|
@ -952,6 +1387,7 @@ mkdir -p %{buildroot}$DirHoldingGdbPy
|
|||
%global _pyconfig64_h pyconfig-64.h
|
||||
%global _pyconfig_h pyconfig-%{wordsize}.h
|
||||
|
||||
|
||||
# Use a common function to do an install for all our configurations:
|
||||
InstallPython() {
|
||||
|
||||
|
@ -1088,7 +1524,7 @@ do
|
|||
LD_LIBRARY_PATH=./build/optimized ./build/optimized/python \
|
||||
Tools/scripts/pathfix.py \
|
||||
-i "%{_libexecdir}/platform-python${LDVersion}" -pn \
|
||||
%{buildroot}%{pylibdir}/config-${LDVersion}-%{_arch}-linux%{_gnu}/python-config.py
|
||||
%{buildroot}%{pylibdir}/config-${LDVersion}-%{platform_triplet}/python-config.py
|
||||
done
|
||||
|
||||
# Remove tests for python3-tools which was removed in
|
||||
|
@ -1169,6 +1605,11 @@ touch %{buildroot}%{_bindir}/unversioned-python
|
|||
touch %{buildroot}%{_bindir}/idle3
|
||||
touch %{buildroot}%{_mandir}/man1/python.1.gz
|
||||
|
||||
# Strip the LTO bytecode from python.o
|
||||
# Based on the fedora brp-strip-lto scriptlet
|
||||
# https://src.fedoraproject.org/rpms/redhat-rpm-config/blob/9dd5528cf9805ebfe31cff04fe7828ad06a6023f/f/brp-strip-lto
|
||||
find %{buildroot} -type f -name 'python.o' -print0 | xargs -0 \
|
||||
bash -c "strip -p -R .gnu.lto_* -R .gnu.debuglto_* -N __gnu_lto_v1 \"\$@\"" ARG0
|
||||
|
||||
# ======================================================
|
||||
# Checks for packaging issues
|
||||
|
@ -1262,7 +1703,7 @@ alternatives --install %{_bindir}/unversioned-python \
|
|||
%postun -n platform-python
|
||||
# Do this only during uninstall process (not during update)
|
||||
if [ $1 -eq 0 ]; then
|
||||
alternatives --remove python \
|
||||
alternatives --keep-foreign --remove python \
|
||||
%{_libexecdir}/no-python
|
||||
|
||||
fi
|
||||
|
@ -1277,7 +1718,7 @@ alternatives --add-slave python3 %{_bindir}/python3.6 \
|
|||
%postun -n python3-idle
|
||||
# Do this only during uninstall process (not during update)
|
||||
if [ $1 -eq 0 ]; then
|
||||
alternatives --remove-slave python3 %{_bindir}/python3.6 \
|
||||
alternatives --keep-foreign --remove-slave python3 %{_bindir}/python3.6 \
|
||||
idle3
|
||||
fi
|
||||
|
||||
|
@ -1508,8 +1949,8 @@ fi
|
|||
# "Makefile" and the config-32/64.h file are needed by
|
||||
# distutils/sysconfig.py:_init_posix(), so we include them in the core
|
||||
# package, along with their parent directories (bug 531901):
|
||||
%dir %{pylibdir}/config-%{LDVERSION_optimized}-%{_arch}-linux%{_gnu}/
|
||||
%{pylibdir}/config-%{LDVERSION_optimized}-%{_arch}-linux%{_gnu}/Makefile
|
||||
%dir %{pylibdir}/config-%{LDVERSION_optimized}-%{platform_triplet}/
|
||||
%{pylibdir}/config-%{LDVERSION_optimized}-%{platform_triplet}/Makefile
|
||||
%dir %{_includedir}/python%{LDVERSION_optimized}/
|
||||
%{_includedir}/python%{LDVERSION_optimized}/%{_pyconfig_h}
|
||||
|
||||
|
@ -1523,8 +1964,8 @@ fi
|
|||
%{_bindir}/2to3
|
||||
# TODO: Remove 2to3-3.7 once rebased to 3.7
|
||||
%{_bindir}/2to3-%{pybasever}
|
||||
%{pylibdir}/config-%{LDVERSION_optimized}-%{_arch}-linux%{_gnu}/*
|
||||
%exclude %{pylibdir}/config-%{LDVERSION_optimized}-%{_arch}-linux%{_gnu}/Makefile
|
||||
%{pylibdir}/config-%{LDVERSION_optimized}-%{platform_triplet}/*
|
||||
%exclude %{pylibdir}/config-%{LDVERSION_optimized}-%{platform_triplet}/Makefile
|
||||
%exclude %{pylibdir}/distutils/command/wininst-*.exe
|
||||
%{_includedir}/python%{LDVERSION_optimized}/*.h
|
||||
%exclude %{_includedir}/python%{LDVERSION_optimized}/%{_pyconfig_h}
|
||||
|
@ -1672,7 +2113,7 @@ fi
|
|||
%{_libdir}/%{py_INSTSONAME_debug}
|
||||
|
||||
# Analog of the -devel subpackage's files:
|
||||
%{pylibdir}/config-%{LDVERSION_debug}-%{_arch}-linux%{_gnu}
|
||||
%{pylibdir}/config-%{LDVERSION_debug}-%{platform_triplet}
|
||||
%{_includedir}/python%{LDVERSION_debug}
|
||||
|
||||
%exclude %{_bindir}/python%{LDVERSION_debug}-config
|
||||
|
@ -1720,6 +2161,173 @@ fi
|
|||
# ======================================================
|
||||
|
||||
%changelog
|
||||
* Wed Apr 24 2024 Lumír Balhar <lbalhar@redhat.com> - 3.6.8-62
|
||||
- Security fix for CVE-2024-0450
|
||||
Resolves: RHEL-33683
|
||||
|
||||
* Wed Apr 24 2024 Lumír Balhar <lbalhar@redhat.com> - 3.6.8-61
|
||||
- Security fix for CVE-2023-6597
|
||||
Resolves: RHEL-33671
|
||||
|
||||
* Wed Apr 24 2024 Lumír Balhar <lbalhar@redhat.com> - 3.6.8-60
|
||||
- Fix build with expat with fixed CVE-2023-52425
|
||||
Related: RHEL-33671
|
||||
|
||||
* Thu Jan 04 2024 Lumír Balhar <lbalhar@redhat.com> - 3.6.8-59
|
||||
- Security fix for CVE-2023-27043
|
||||
Resolves: RHEL-20610
|
||||
|
||||
* Tue Dec 12 2023 Lumír Balhar <lbalhar@redhat.com> - 3.6.8-58
|
||||
- Security fix for CVE-2022-48564
|
||||
Resolves: RHEL-16674
|
||||
- Skip tests failing on s390x
|
||||
Resolves: RHEL-19252
|
||||
|
||||
* Thu Nov 23 2023 Lumír Balhar <lbalhar@redhat.com> - 3.6.8-57
|
||||
- Security fix for CVE-2022-48560
|
||||
Resolves: RHEL-16707
|
||||
|
||||
* Thu Sep 07 2023 Charalampos Stratakis <cstratak@redhat.com> - 3.6.8-56
|
||||
- Security fix for CVE-2023-40217
|
||||
Resolves: RHEL-3041
|
||||
|
||||
* Wed Aug 09 2023 Petr Viktorin <pviktori@redhat.com> - 3.6.8-55
|
||||
- Fix symlink handling in the fix for CVE-2007-4559
|
||||
Resolves: rhbz#263261
|
||||
|
||||
* Fri Jul 07 2023 Charalampos Stratakis <cstratak@redhat.com> - 3.6.8-54
|
||||
- Bump release for rebuild
|
||||
Resolves: rhbz#2173917
|
||||
|
||||
* Fri Jun 30 2023 Charalampos Stratakis <cstratak@redhat.com> - 3.6.8-53
|
||||
- Security fix for CVE-2023-24329
|
||||
Resolves: rhbz#2173917
|
||||
|
||||
* Tue Jun 06 2023 Petr Viktorin <pviktori@redhat.com> - 3.6.8-52
|
||||
- Add filters for tarfile extraction (CVE-2007-4559, PEP-706)
|
||||
Resolves: rhbz#263261
|
||||
|
||||
* Tue Jan 24 2023 Charalampos Stratakis <cstratak@redhat.com> - 3.6.8-51
|
||||
- Properly strip the LTO bytecode from python.o
|
||||
Resolves: rhbz#2137707
|
||||
|
||||
* Wed Dec 21 2022 Charalampos Stratakis <cstratak@redhat.com> - 3.6.8-50
|
||||
- Security fix for CVE-2022-45061
|
||||
- Strip the LTO bytecode from python.o
|
||||
Resolves: rhbz#2144072, rhbz#2137707
|
||||
|
||||
* Tue Oct 25 2022 Charalampos Stratakis <cstratak@redhat.com> - 3.6.8-49
|
||||
- Security fixes for CVE-2020-10735 and CVE-2021-28861
|
||||
Resolves: rhbz#1834423, rhbz#2120642
|
||||
|
||||
* Thu Oct 20 2022 Charalampos Stratakis <cstratak@redhat.com> - 3.6.8-48
|
||||
- Release bump
|
||||
Resolves: rhbz#2136435
|
||||
|
||||
* Tue Jun 14 2022 Charalampos Stratakis <cstratak@redhat.com> - 3.6.8-47
|
||||
- Security fix for CVE-2015-20107
|
||||
Resolves: rhbz#2075390
|
||||
|
||||
* Wed Mar 09 2022 Charalampos Stratakis <cstratak@redhat.com> - 3.6.8-46
|
||||
- Security fix for CVE-2022-0391: urlparse does not sanitize URLs containing ASCII newline and tabs
|
||||
- Fix the test suite support for Expat >= 2.4.5
|
||||
Resolves: rhbz#2047376, rhbz#2060435
|
||||
|
||||
* Fri Jan 07 2022 Charalampos Stratakis <cstratak@redhat.com> - 3.6.8-45
|
||||
- Security fix for CVE-2021-4189: ftplib should not use the host from the PASV response
|
||||
Resolves: rhbz#2036020
|
||||
|
||||
* Tue Oct 12 2021 Charalampos Stratakis <cstratak@redhat.com> - 3.6.8-44
|
||||
- Use the monotonic clock for theading.Condition
|
||||
- Use the monotonic clock for the global interpreter lock
|
||||
Resolves: rhbz#2003758
|
||||
|
||||
* Mon Oct 11 2021 Charalampos Stratakis <cstratak@redhat.com> - 3.6.8-43
|
||||
- Change shouldRollover() methods of logging.handlers to only rollover regular files
|
||||
Resolves: rhbz#2009200
|
||||
|
||||
* Fri Sep 17 2021 Lumír Balhar <lbalhar@redhat.com> - 3.6.8-42
|
||||
- Security fix for CVE-2021-3737
|
||||
Resolves: rhbz#1995162
|
||||
|
||||
* Thu Sep 09 2021 Lumír Balhar <lbalhar@redhat.com> - 3.6.8-41
|
||||
- Security fix for CVE-2021-3733: Denial of service when identifying crafted invalid RFCs
|
||||
Resolves: rhbz#1995234
|
||||
|
||||
* Thu Jul 29 2021 Tomas Orsava <torsava@redhat.com> - 3.6.8-40
|
||||
- Adjusted the postun scriptlets to enable upgrading to RHEL 9
|
||||
- Resolves: rhbz#1933055
|
||||
|
||||
* Fri Jul 09 2021 Victor Stinner <vstinner@redhat.com> - 3.6.8-39
|
||||
- Fix reentrant call to threading.enumerate() (rhbz#1959459)
|
||||
- Don't exit Python with abort() when a thread exit and there is no available
|
||||
file descriptor to load dynamically the libgcc_s.so.1 library (rhbz#1972293)
|
||||
|
||||
* Fri Apr 30 2021 Charalampos Stratakis <cstratak@redhat.com> - 3.6.8-38
|
||||
- Security fix for CVE-2021-3426: information disclosure via pydoc
|
||||
Resolves: rhbz#1935913
|
||||
|
||||
* Thu Mar 04 2021 Petr Viktorin <pviktori@redhat.com> - 3.6.8-37
|
||||
- Fix for CVE-2021-23336
|
||||
Resolves: rhbz#1928904
|
||||
|
||||
* Fri Jan 22 2021 Lumír Balhar <lbalhar@redhat.com> - 3.6.8-36
|
||||
- Fix for CVE-2021-3177
|
||||
Resolves: rhbz#1918168
|
||||
|
||||
* Mon Jan 18 2021 Lumír Balhar <lbalhar@redhat.com> - 3.6.8-35
|
||||
- New options -a and -k for pathfix.py script backported from upstream
|
||||
Resolves: rhbz#1917691
|
||||
|
||||
* Fri Dec 04 2020 Charalampos Stratakis <cstratak@redhat.com> - 3.6.8-34
|
||||
- Security fix for CVE-2020-27619: eval() call on content received via HTTP in the CJK codec tests
|
||||
Resolves: rhbz#1890237
|
||||
|
||||
* Tue Nov 24 2020 Lumír Balhar <lbalhar@redhat.com> - 3.6.8-33
|
||||
- Add support for upstream architecture names
|
||||
https://fedoraproject.org/wiki/Changes/Python_Upstream_Architecture_Names
|
||||
Resolves: rhbz#1868003
|
||||
|
||||
* Mon Nov 09 2020 Charalampos Stratakis <cstratak@redhat.com> - 3.6.8-32
|
||||
- Security fix for CVE-2020-26116: Reject control chars in HTTP method in http.client
|
||||
Resolves: rhbz#1883257
|
||||
|
||||
* Mon Aug 17 2020 Tomas Orsava <torsava@redhat.com> - 3.6.8-31
|
||||
- Avoid infinite loop when reading specially crafted TAR files (CVE-2019-20907)
|
||||
Resolves: rhbz#1856481
|
||||
- Resolve hash collisions for Pv4Interface and IPv6Interface (CVE-2020-14422)
|
||||
Resolves: rhbz#1854926
|
||||
|
||||
* Thu Jun 25 2020 Victor Stinner <vstinner@python.org> - 3.6.8-30
|
||||
- Remove downstream 00178-dont-duplicate-flags-in-sysconfig.patch which
|
||||
introduced a bug on distutils.sysconfig.get_config_var('LIBPL')
|
||||
(rhbz#1851090).
|
||||
|
||||
* Thu Jun 18 2020 Victor Stinner <vstinner@python.org> - 3.6.8-29
|
||||
- Fix python3-config --configdir (rhbz#1772992).
|
||||
|
||||
* Fri Apr 03 2020 Charalampos Stratakis <cstratak@redhat.com> - 3.6.8-28
|
||||
- Security fix for CVE-2020-8492
|
||||
Resolves: rhbz#1810618
|
||||
|
||||
* Tue Mar 31 2020 Charalampos Stratakis <cstratak@redhat.com> - 3.6.8-27
|
||||
- Add a sentinel value on the Hmac_members table of the fips compliant hmac module
|
||||
Resolves: rhbz#1800512
|
||||
|
||||
* Mon Mar 23 2020 Charalampos Stratakis <cstratak@redhat.com> - 3.6.8-26
|
||||
- Skip test_startup_imports from test_site if we have a .pth file in sys.path
|
||||
Resolves: rhbz#1814392
|
||||
|
||||
* Fri Mar 20 2020 Charalampos Stratakis <cstratak@redhat.com> - 3.6.8-25
|
||||
- Security fix for CVE-2019-16935
|
||||
Resolves: rhbz#1798001
|
||||
|
||||
* Mon Mar 16 2020 Charalampos Stratakis <cstratak@redhat.com> - 3.6.8-24
|
||||
- Build Python with -fno-semantic-interposition for better performance
|
||||
- https://fedoraproject.org/wiki/Changes/PythonNoSemanticInterpositionSpeedup
|
||||
- Also fix test_gdb failures with Link Time Optimizations
|
||||
Resolves: rhbz#1724996
|
||||
|
||||
* Wed Nov 27 2019 Charalampos Stratakis <cstratak@redhat.com> - 3.6.8-23
|
||||
- Modify the test suite to better handle disabled SSL/TLS versions and FIPS mode
|
||||
- Use OpenSSL's DRBG and disable os.getrandom() function in FIPS mode
|
||||
|
|
Loading…
Reference in New Issue