Compare commits
17 Commits
imports/c8
...
c8
Author | SHA1 | Date |
---|---|---|
eabdullin | d0c83e320b | |
eabdullin | c51dc1346c | |
eabdullin | a37af4d525 | |
eabdullin | 346df28398 | |
eabdullin | 1563e29a86 | |
eabdullin | c3b837042b | |
Andrew Lukoshko | b140357418 | |
Andrew Lukoshko | a4d48ab5dc | |
CentOS Sources | 6ac83c33f3 | |
CentOS Sources | d01f487196 | |
CentOS Sources | 93994c4d72 | |
CentOS Sources | 69319ca074 | |
CentOS Sources | a62df8a19c | |
CentOS Sources | 3a450dae9d | |
CentOS Sources | d72d9cb256 | |
CentOS Sources | b2efcd69f9 | |
CentOS Sources | 7c0aac331f |
|
@ -0,0 +1,40 @@
|
|||
diff --git a/Lib/threading.py b/Lib/threading.py
|
||||
index 7ab9ad8..dcedd3b 100644
|
||||
--- a/Lib/threading.py
|
||||
+++ b/Lib/threading.py
|
||||
@@ -3,7 +3,7 @@
|
||||
import sys as _sys
|
||||
import _thread
|
||||
|
||||
-from time import monotonic as _time
|
||||
+from time import monotonic as _time, sleep as _sleep
|
||||
from traceback import format_exc as _format_exc
|
||||
from _weakrefset import WeakSet
|
||||
from itertools import islice as _islice, count as _count
|
||||
@@ -296,7 +296,25 @@ class Condition:
|
||||
gotit = True
|
||||
else:
|
||||
if timeout > 0:
|
||||
- gotit = waiter.acquire(True, timeout)
|
||||
+ # rhbz#2003758: Avoid waiter.acquire(True, timeout) since
|
||||
+ # it uses the system clock internally.
|
||||
+ #
|
||||
+ # Balancing act: We can't afford a pure busy loop, so we
|
||||
+ # have to sleep; but if we sleep the whole timeout time,
|
||||
+ # we'll be unresponsive. The scheme here sleeps very
|
||||
+ # little at first, longer as time goes on, but never longer
|
||||
+ # than 20 times per second (or the timeout time remaining).
|
||||
+ endtime = _time() + timeout
|
||||
+ delay = 0.0005 # 500 us -> initial delay of 1 ms
|
||||
+ while True:
|
||||
+ gotit = waiter.acquire(0)
|
||||
+ if gotit:
|
||||
+ break
|
||||
+ remaining = min(endtime - _time(), timeout)
|
||||
+ if remaining <= 0:
|
||||
+ break
|
||||
+ delay = min(delay * 2, remaining, .05)
|
||||
+ _sleep(delay)
|
||||
else:
|
||||
gotit = waiter.acquire(False)
|
||||
return gotit
|
|
@ -1,58 +0,0 @@
|
|||
diff -up Python-3.5.0/configure.ac.than Python-3.5.0/configure.ac
|
||||
--- Python-3.5.0/configure.ac.than 2015-11-13 11:51:32.039560172 -0500
|
||||
+++ Python-3.5.0/configure.ac 2015-11-13 11:52:11.670168157 -0500
|
||||
@@ -788,9 +788,9 @@ cat >> conftest.c <<EOF
|
||||
alpha-linux-gnu
|
||||
# elif defined(__ARM_EABI__) && defined(__ARM_PCS_VFP)
|
||||
# if defined(__ARMEL__)
|
||||
- arm-linux-gnueabihf
|
||||
+ arm-linux-gnueabi
|
||||
# else
|
||||
- armeb-linux-gnueabihf
|
||||
+ armeb-linux-gnueabi
|
||||
# endif
|
||||
# elif defined(__ARM_EABI__) && !defined(__ARM_PCS_VFP)
|
||||
# if defined(__ARMEL__)
|
||||
@@ -810,7 +810,7 @@ cat >> conftest.c <<EOF
|
||||
# elif _MIPS_SIM == _ABIN32
|
||||
mips64el-linux-gnuabin32
|
||||
# elif _MIPS_SIM == _ABI64
|
||||
- mips64el-linux-gnuabi64
|
||||
+ mips64el-linux-gnu
|
||||
# else
|
||||
# error unknown platform triplet
|
||||
# endif
|
||||
@@ -820,7 +820,7 @@ cat >> conftest.c <<EOF
|
||||
# elif _MIPS_SIM == _ABIN32
|
||||
mips64-linux-gnuabin32
|
||||
# elif _MIPS_SIM == _ABI64
|
||||
- mips64-linux-gnuabi64
|
||||
+ mips64-linux-gnu
|
||||
# else
|
||||
# error unknown platform triplet
|
||||
# endif
|
||||
@@ -830,9 +830,9 @@ cat >> conftest.c <<EOF
|
||||
powerpc-linux-gnuspe
|
||||
# elif defined(__powerpc64__)
|
||||
# if defined(__LITTLE_ENDIAN__)
|
||||
- powerpc64le-linux-gnu
|
||||
+ ppc64le-linux-gnu
|
||||
# else
|
||||
- powerpc64-linux-gnu
|
||||
+ ppc64-linux-gnu
|
||||
# endif
|
||||
# elif defined(__powerpc__)
|
||||
powerpc-linux-gnu
|
||||
diff --git a/config.sub b/config.sub
|
||||
index 40ea5df..932128b 100755
|
||||
--- a/config.sub
|
||||
+++ b/config.sub
|
||||
@@ -1045,7 +1045,7 @@ case $basic_machine in
|
||||
;;
|
||||
ppc64) basic_machine=powerpc64-unknown
|
||||
;;
|
||||
- ppc64-*) basic_machine=powerpc64-`echo $basic_machine | sed 's/^[^-]*-//'`
|
||||
+ ppc64-* | ppc64p7-*) basic_machine=powerpc64-`echo $basic_machine | sed 's/^[^-]*-//'`
|
||||
;;
|
||||
ppc64le | powerpc64little)
|
||||
basic_machine=powerpc64le-unknown
|
|
@ -0,0 +1,97 @@
|
|||
From 0000000000000000000000000000000000000000 Mon Sep 17 00:00:00 2001
|
||||
From: Lumir Balhar <lbalhar@redhat.com>
|
||||
Date: Tue, 4 Aug 2020 12:04:03 +0200
|
||||
Subject: [PATCH] 00353: Original names for architectures with different names
|
||||
downstream
|
||||
MIME-Version: 1.0
|
||||
Content-Type: text/plain; charset=UTF-8
|
||||
Content-Transfer-Encoding: 8bit
|
||||
|
||||
https://fedoraproject.org/wiki/Changes/Python_Upstream_Architecture_Names
|
||||
|
||||
Pythons in RHEL/Fedora used different names for some architectures
|
||||
than upstream and other distros (for example ppc64 vs. powerpc64).
|
||||
This was patched in patch 274, now it is sedded if %with legacy_archnames.
|
||||
|
||||
That meant that an extension built with the default upstream settings
|
||||
(on other distro or as an manylinux wheel) could not been found by Python
|
||||
on RHEL/Fedora because it had a different suffix.
|
||||
This patch adds the legacy names to importlib so Python is able
|
||||
to import extensions with a legacy architecture name in its
|
||||
file name.
|
||||
It work both ways, so it support both %with and %without legacy_archnames.
|
||||
|
||||
WARNING: This patch has no effect on Python built with bootstrap
|
||||
enabled because Python/importlib_external.h is not regenerated
|
||||
and therefore Python during bootstrap contains importlib from
|
||||
upstream without this feature. It's possible to include
|
||||
Python/importlib_external.h to this patch but it'd make rebasing
|
||||
a nightmare because it's basically a binary file.
|
||||
|
||||
Co-authored-by: Miro Hrončok <miro@hroncok.cz>
|
||||
---
|
||||
Lib/importlib/_bootstrap_external.py | 40 ++++++++++++++++++++++++++--
|
||||
1 file changed, 38 insertions(+), 2 deletions(-)
|
||||
|
||||
diff --git a/Lib/importlib/_bootstrap_external.py b/Lib/importlib/_bootstrap_external.py
|
||||
index 9feec50842..5bb2454a5c 100644
|
||||
--- a/Lib/importlib/_bootstrap_external.py
|
||||
+++ b/Lib/importlib/_bootstrap_external.py
|
||||
@@ -1361,7 +1361,7 @@ def _get_supported_file_loaders():
|
||||
|
||||
Each item is a tuple (loader, suffixes).
|
||||
"""
|
||||
- extensions = ExtensionFileLoader, _imp.extension_suffixes()
|
||||
+ extensions = ExtensionFileLoader, _alternative_architectures(_imp.extension_suffixes())
|
||||
source = SourceFileLoader, SOURCE_SUFFIXES
|
||||
bytecode = SourcelessFileLoader, BYTECODE_SUFFIXES
|
||||
return [extensions, source, bytecode]
|
||||
@@ -1428,7 +1428,7 @@ def _setup(_bootstrap_module):
|
||||
|
||||
# Constants
|
||||
setattr(self_module, '_relax_case', _make_relax_case())
|
||||
- EXTENSION_SUFFIXES.extend(_imp.extension_suffixes())
|
||||
+ EXTENSION_SUFFIXES.extend(_alternative_architectures(_imp.extension_suffixes()))
|
||||
if builtin_os == 'nt':
|
||||
SOURCE_SUFFIXES.append('.pyw')
|
||||
if '_d.pyd' in EXTENSION_SUFFIXES:
|
||||
@@ -1441,3 +1441,39 @@ def _install(_bootstrap_module):
|
||||
supported_loaders = _get_supported_file_loaders()
|
||||
sys.path_hooks.extend([FileFinder.path_hook(*supported_loaders)])
|
||||
sys.meta_path.append(PathFinder)
|
||||
+
|
||||
+
|
||||
+_ARCH_MAP = {
|
||||
+ "-arm-linux-gnueabi.": "-arm-linux-gnueabihf.",
|
||||
+ "-armeb-linux-gnueabi.": "-armeb-linux-gnueabihf.",
|
||||
+ "-mips64-linux-gnu.": "-mips64-linux-gnuabi64.",
|
||||
+ "-mips64el-linux-gnu.": "-mips64el-linux-gnuabi64.",
|
||||
+ "-ppc-linux-gnu.": "-powerpc-linux-gnu.",
|
||||
+ "-ppc-linux-gnuspe.": "-powerpc-linux-gnuspe.",
|
||||
+ "-ppc64-linux-gnu.": "-powerpc64-linux-gnu.",
|
||||
+ "-ppc64le-linux-gnu.": "-powerpc64le-linux-gnu.",
|
||||
+ # The above, but the other way around:
|
||||
+ "-arm-linux-gnueabihf.": "-arm-linux-gnueabi.",
|
||||
+ "-armeb-linux-gnueabihf.": "-armeb-linux-gnueabi.",
|
||||
+ "-mips64-linux-gnuabi64.": "-mips64-linux-gnu.",
|
||||
+ "-mips64el-linux-gnuabi64.": "-mips64el-linux-gnu.",
|
||||
+ "-powerpc-linux-gnu.": "-ppc-linux-gnu.",
|
||||
+ "-powerpc-linux-gnuspe.": "-ppc-linux-gnuspe.",
|
||||
+ "-powerpc64-linux-gnu.": "-ppc64-linux-gnu.",
|
||||
+ "-powerpc64le-linux-gnu.": "-ppc64le-linux-gnu.",
|
||||
+}
|
||||
+
|
||||
+
|
||||
+def _alternative_architectures(suffixes):
|
||||
+ """Add a suffix with an alternative architecture name
|
||||
+ to the list of suffixes so an extension built with
|
||||
+ the default (upstream) setting is loadable with our Pythons
|
||||
+ """
|
||||
+
|
||||
+ for suffix in suffixes:
|
||||
+ for original, alternative in _ARCH_MAP.items():
|
||||
+ if original in suffix:
|
||||
+ suffixes.append(suffix.replace(original, alternative))
|
||||
+ return suffixes
|
||||
+
|
||||
+ return suffixes
|
|
@ -0,0 +1,73 @@
|
|||
diff --git a/Lib/http/client.py b/Lib/http/client.py
|
||||
index f0d2642..0a044e9 100644
|
||||
--- a/Lib/http/client.py
|
||||
+++ b/Lib/http/client.py
|
||||
@@ -151,6 +151,10 @@ _contains_disallowed_url_pchar_re = re.compile('[\x00-\x20\x7f]')
|
||||
# _is_allowed_url_pchars_re = re.compile(r"^[/!$&'()*+,;=:@%a-zA-Z0-9._~-]+$")
|
||||
# We are more lenient for assumed real world compatibility purposes.
|
||||
|
||||
+# These characters are not allowed within HTTP method names
|
||||
+# to prevent http header injection.
|
||||
+_contains_disallowed_method_pchar_re = re.compile('[\x00-\x1f]')
|
||||
+
|
||||
# We always set the Content-Length header for these methods because some
|
||||
# servers will otherwise respond with a 411
|
||||
_METHODS_EXPECTING_BODY = {'PATCH', 'POST', 'PUT'}
|
||||
@@ -1117,6 +1121,8 @@ class HTTPConnection:
|
||||
else:
|
||||
raise CannotSendRequest(self.__state)
|
||||
|
||||
+ self._validate_method(method)
|
||||
+
|
||||
# Save the method we use, we need it later in the response phase
|
||||
self._method = method
|
||||
if not url:
|
||||
@@ -1207,6 +1213,15 @@ class HTTPConnection:
|
||||
# For HTTP/1.0, the server will assume "not chunked"
|
||||
pass
|
||||
|
||||
+ def _validate_method(self, method):
|
||||
+ """Validate a method name for putrequest."""
|
||||
+ # prevent http header injection
|
||||
+ match = _contains_disallowed_method_pchar_re.search(method)
|
||||
+ if match:
|
||||
+ raise ValueError(
|
||||
+ f"method can't contain control characters. {method!r} "
|
||||
+ f"(found at least {match.group()!r})")
|
||||
+
|
||||
def putheader(self, header, *values):
|
||||
"""Send a request header line to the server.
|
||||
|
||||
diff --git a/Lib/test/test_httplib.py b/Lib/test/test_httplib.py
|
||||
index 5795b7a..af0350f 100644
|
||||
--- a/Lib/test/test_httplib.py
|
||||
+++ b/Lib/test/test_httplib.py
|
||||
@@ -359,6 +359,28 @@ class HeaderTests(TestCase):
|
||||
self.assertEqual(lines[2], "header: Second: val")
|
||||
|
||||
|
||||
+class HttpMethodTests(TestCase):
|
||||
+ def test_invalid_method_names(self):
|
||||
+ methods = (
|
||||
+ 'GET\r',
|
||||
+ 'POST\n',
|
||||
+ 'PUT\n\r',
|
||||
+ 'POST\nValue',
|
||||
+ 'POST\nHOST:abc',
|
||||
+ 'GET\nrHost:abc\n',
|
||||
+ 'POST\rRemainder:\r',
|
||||
+ 'GET\rHOST:\n',
|
||||
+ '\nPUT'
|
||||
+ )
|
||||
+
|
||||
+ for method in methods:
|
||||
+ with self.assertRaisesRegex(
|
||||
+ ValueError, "method can't contain control characters"):
|
||||
+ conn = client.HTTPConnection('example.com')
|
||||
+ conn.sock = FakeSocket(None)
|
||||
+ conn.request(method=method, url="/")
|
||||
+
|
||||
+
|
||||
class TransferEncodingTest(TestCase):
|
||||
expected_body = b"It's just a flesh wound"
|
||||
|
|
@ -0,0 +1,42 @@
|
|||
diff --git a/Lib/test/multibytecodec_support.py b/Lib/test/multibytecodec_support.py
|
||||
index f9884c6..98feec2 100644
|
||||
--- a/Lib/test/multibytecodec_support.py
|
||||
+++ b/Lib/test/multibytecodec_support.py
|
||||
@@ -300,29 +300,23 @@ class TestBase_Mapping(unittest.TestCase):
|
||||
self._test_mapping_file_plain()
|
||||
|
||||
def _test_mapping_file_plain(self):
|
||||
- unichrs = lambda s: ''.join(map(chr, map(eval, s.split('+'))))
|
||||
+ def unichrs(s):
|
||||
+ return ''.join(chr(int(x, 16)) for x in s.split('+'))
|
||||
+
|
||||
urt_wa = {}
|
||||
|
||||
with self.open_mapping_file() as f:
|
||||
for line in f:
|
||||
if not line:
|
||||
break
|
||||
- data = line.split('#')[0].strip().split()
|
||||
+ data = line.split('#')[0].split()
|
||||
if len(data) != 2:
|
||||
continue
|
||||
|
||||
- csetval = eval(data[0])
|
||||
- if csetval <= 0x7F:
|
||||
- csetch = bytes([csetval & 0xff])
|
||||
- elif csetval >= 0x1000000:
|
||||
- csetch = bytes([(csetval >> 24), ((csetval >> 16) & 0xff),
|
||||
- ((csetval >> 8) & 0xff), (csetval & 0xff)])
|
||||
- elif csetval >= 0x10000:
|
||||
- csetch = bytes([(csetval >> 16), ((csetval >> 8) & 0xff),
|
||||
- (csetval & 0xff)])
|
||||
- elif csetval >= 0x100:
|
||||
- csetch = bytes([(csetval >> 8), (csetval & 0xff)])
|
||||
- else:
|
||||
+ if data[0][:2] != '0x':
|
||||
+ self.fail(f"Invalid line: {line!r}")
|
||||
+ csetch = bytes.fromhex(data[0][2:])
|
||||
+ if len(csetch) == 1 and 0x80 <= csetch[0]:
|
||||
continue
|
||||
|
||||
unich = unichrs(data[1])
|
|
@ -0,0 +1,269 @@
|
|||
From 0cfd9a7f26488567b9a3e5ec192099a8b80ad9df Mon Sep 17 00:00:00 2001
|
||||
From: Lumir Balhar <lbalhar@redhat.com>
|
||||
Date: Tue, 19 Jan 2021 07:55:37 +0100
|
||||
Subject: [PATCH] [PATCH] bpo-37064: Add -k and -a options to pathfix.py tool
|
||||
(GH-16387)
|
||||
|
||||
* bpo-37064: Add option -k to Tools/scripts/pathfix.py (GH-15548)
|
||||
|
||||
Add flag -k to pathscript.py script: preserve shebang flags.
|
||||
|
||||
(cherry picked from commit 50254ac4c179cb412e90682098c97db786143929)
|
||||
|
||||
* bpo-37064: Add option -a to pathfix.py tool (GH-15717)
|
||||
|
||||
Add option -a to Tools/Scripts/pathfix.py script: add flags.
|
||||
|
||||
(cherry picked from commit 1dc1acbd73f05f14c974b7ce1041787d7abef31e)
|
||||
---
|
||||
Lib/test/test_tools/test_pathfix.py | 104 ++++++++++++++++++++++++++++
|
||||
Tools/scripts/pathfix.py | 64 +++++++++++++++--
|
||||
2 files changed, 163 insertions(+), 5 deletions(-)
|
||||
create mode 100644 Lib/test/test_tools/test_pathfix.py
|
||||
|
||||
diff --git a/Lib/test/test_tools/test_pathfix.py b/Lib/test/test_tools/test_pathfix.py
|
||||
new file mode 100644
|
||||
index 0000000..1f0585e
|
||||
--- /dev/null
|
||||
+++ b/Lib/test/test_tools/test_pathfix.py
|
||||
@@ -0,0 +1,104 @@
|
||||
+import os
|
||||
+import subprocess
|
||||
+import sys
|
||||
+import unittest
|
||||
+from test import support
|
||||
+from test.test_tools import import_tool, scriptsdir
|
||||
+
|
||||
+
|
||||
+class TestPathfixFunctional(unittest.TestCase):
|
||||
+ script = os.path.join(scriptsdir, 'pathfix.py')
|
||||
+
|
||||
+ def setUp(self):
|
||||
+ self.temp_file = support.TESTFN
|
||||
+ self.addCleanup(support.unlink, support.TESTFN)
|
||||
+
|
||||
+ def pathfix(self, shebang, pathfix_flags, exitcode=0, stdout='', stderr=''):
|
||||
+ with open(self.temp_file, 'w', encoding='utf8') as f:
|
||||
+ f.write(f'{shebang}\n' + 'print("Hello world")\n')
|
||||
+
|
||||
+ proc = subprocess.run(
|
||||
+ [sys.executable, self.script,
|
||||
+ *pathfix_flags, '-n', self.temp_file],
|
||||
+ universal_newlines=True, stdout=subprocess.PIPE,
|
||||
+ stderr=subprocess.PIPE)
|
||||
+
|
||||
+ if stdout == '' and proc.returncode == 0:
|
||||
+ stdout = f'{self.temp_file}: updating\n'
|
||||
+ self.assertEqual(proc.returncode, exitcode, proc)
|
||||
+ self.assertEqual(proc.stdout, stdout, proc)
|
||||
+ self.assertEqual(proc.stderr, stderr, proc)
|
||||
+
|
||||
+ with open(self.temp_file, 'r', encoding='utf8') as f:
|
||||
+ output = f.read()
|
||||
+
|
||||
+ lines = output.split('\n')
|
||||
+ self.assertEqual(lines[1:], ['print("Hello world")', ''])
|
||||
+ new_shebang = lines[0]
|
||||
+
|
||||
+ if proc.returncode != 0:
|
||||
+ self.assertEqual(shebang, new_shebang)
|
||||
+
|
||||
+ return new_shebang
|
||||
+
|
||||
+ def test_pathfix(self):
|
||||
+ self.assertEqual(
|
||||
+ self.pathfix(
|
||||
+ '#! /usr/bin/env python',
|
||||
+ ['-i', '/usr/bin/python3']),
|
||||
+ '#! /usr/bin/python3')
|
||||
+ self.assertEqual(
|
||||
+ self.pathfix(
|
||||
+ '#! /usr/bin/env python -R',
|
||||
+ ['-i', '/usr/bin/python3']),
|
||||
+ '#! /usr/bin/python3')
|
||||
+
|
||||
+ def test_pathfix_keeping_flags(self):
|
||||
+ self.assertEqual(
|
||||
+ self.pathfix(
|
||||
+ '#! /usr/bin/env python -R',
|
||||
+ ['-i', '/usr/bin/python3', '-k']),
|
||||
+ '#! /usr/bin/python3 -R')
|
||||
+ self.assertEqual(
|
||||
+ self.pathfix(
|
||||
+ '#! /usr/bin/env python',
|
||||
+ ['-i', '/usr/bin/python3', '-k']),
|
||||
+ '#! /usr/bin/python3')
|
||||
+
|
||||
+ def test_pathfix_adding_flag(self):
|
||||
+ self.assertEqual(
|
||||
+ self.pathfix(
|
||||
+ '#! /usr/bin/env python',
|
||||
+ ['-i', '/usr/bin/python3', '-a', 's']),
|
||||
+ '#! /usr/bin/python3 -s')
|
||||
+ self.assertEqual(
|
||||
+ self.pathfix(
|
||||
+ '#! /usr/bin/env python -S',
|
||||
+ ['-i', '/usr/bin/python3', '-a', 's']),
|
||||
+ '#! /usr/bin/python3 -s')
|
||||
+ self.assertEqual(
|
||||
+ self.pathfix(
|
||||
+ '#! /usr/bin/env python -V',
|
||||
+ ['-i', '/usr/bin/python3', '-a', 'v', '-k']),
|
||||
+ '#! /usr/bin/python3 -vV')
|
||||
+ self.assertEqual(
|
||||
+ self.pathfix(
|
||||
+ '#! /usr/bin/env python',
|
||||
+ ['-i', '/usr/bin/python3', '-a', 'Rs']),
|
||||
+ '#! /usr/bin/python3 -Rs')
|
||||
+ self.assertEqual(
|
||||
+ self.pathfix(
|
||||
+ '#! /usr/bin/env python -W default',
|
||||
+ ['-i', '/usr/bin/python3', '-a', 's', '-k']),
|
||||
+ '#! /usr/bin/python3 -sW default')
|
||||
+
|
||||
+ def test_pathfix_adding_errors(self):
|
||||
+ self.pathfix(
|
||||
+ '#! /usr/bin/env python -E',
|
||||
+ ['-i', '/usr/bin/python3', '-a', 'W default', '-k'],
|
||||
+ exitcode=2,
|
||||
+ stderr="-a option doesn't support whitespaces")
|
||||
+
|
||||
+
|
||||
+if __name__ == '__main__':
|
||||
+ unittest.main()
|
||||
diff --git a/Tools/scripts/pathfix.py b/Tools/scripts/pathfix.py
|
||||
index c5bf984..2dfa6e8 100755
|
||||
--- a/Tools/scripts/pathfix.py
|
||||
+++ b/Tools/scripts/pathfix.py
|
||||
@@ -1,6 +1,6 @@
|
||||
#!/usr/bin/env python3
|
||||
|
||||
-# Change the #! line occurring in Python scripts. The new interpreter
|
||||
+# Change the #! line (shebang) occurring in Python scripts. The new interpreter
|
||||
# pathname must be given with a -i option.
|
||||
#
|
||||
# Command line arguments are files or directories to be processed.
|
||||
@@ -10,7 +10,13 @@
|
||||
# arguments).
|
||||
# The original file is kept as a back-up (with a "~" attached to its name),
|
||||
# -n flag can be used to disable this.
|
||||
-#
|
||||
+
|
||||
+# Sometimes you may find shebangs with flags such as `#! /usr/bin/env python -si`.
|
||||
+# Normally, pathfix overwrites the entire line, including the flags.
|
||||
+# To change interpreter and keep flags from the original shebang line, use -k.
|
||||
+# If you want to keep flags and add to them one single literal flag, use option -a.
|
||||
+
|
||||
+
|
||||
# Undoubtedly you can do this using find and sed or perl, but this is
|
||||
# a nice example of Python code that recurses down a directory tree
|
||||
# and uses regular expressions. Also note several subtleties like
|
||||
@@ -33,16 +39,21 @@ rep = sys.stdout.write
|
||||
new_interpreter = None
|
||||
preserve_timestamps = False
|
||||
create_backup = True
|
||||
+keep_flags = False
|
||||
+add_flags = b''
|
||||
|
||||
|
||||
def main():
|
||||
global new_interpreter
|
||||
global preserve_timestamps
|
||||
global create_backup
|
||||
- usage = ('usage: %s -i /interpreter -p -n file-or-directory ...\n' %
|
||||
+ global keep_flags
|
||||
+ global add_flags
|
||||
+
|
||||
+ usage = ('usage: %s -i /interpreter -p -n -k -a file-or-directory ...\n' %
|
||||
sys.argv[0])
|
||||
try:
|
||||
- opts, args = getopt.getopt(sys.argv[1:], 'i:pn')
|
||||
+ opts, args = getopt.getopt(sys.argv[1:], 'i:a:kpn')
|
||||
except getopt.error as msg:
|
||||
err(str(msg) + '\n')
|
||||
err(usage)
|
||||
@@ -54,6 +65,13 @@ def main():
|
||||
preserve_timestamps = True
|
||||
if o == '-n':
|
||||
create_backup = False
|
||||
+ if o == '-k':
|
||||
+ keep_flags = True
|
||||
+ if o == '-a':
|
||||
+ add_flags = a.encode()
|
||||
+ if b' ' in add_flags:
|
||||
+ err("-a option doesn't support whitespaces")
|
||||
+ sys.exit(2)
|
||||
if not new_interpreter or not new_interpreter.startswith(b'/') or \
|
||||
not args:
|
||||
err('-i option or file-or-directory missing\n')
|
||||
@@ -70,10 +88,14 @@ def main():
|
||||
if fix(arg): bad = 1
|
||||
sys.exit(bad)
|
||||
|
||||
+
|
||||
ispythonprog = re.compile(r'^[a-zA-Z0-9_]+\.py$')
|
||||
+
|
||||
+
|
||||
def ispython(name):
|
||||
return bool(ispythonprog.match(name))
|
||||
|
||||
+
|
||||
def recursedown(dirname):
|
||||
dbg('recursedown(%r)\n' % (dirname,))
|
||||
bad = 0
|
||||
@@ -96,6 +118,7 @@ def recursedown(dirname):
|
||||
if recursedown(fullname): bad = 1
|
||||
return bad
|
||||
|
||||
+
|
||||
def fix(filename):
|
||||
## dbg('fix(%r)\n' % (filename,))
|
||||
try:
|
||||
@@ -166,12 +189,43 @@ def fix(filename):
|
||||
# Return success
|
||||
return 0
|
||||
|
||||
+
|
||||
+def parse_shebang(shebangline):
|
||||
+ shebangline = shebangline.rstrip(b'\n')
|
||||
+ start = shebangline.find(b' -')
|
||||
+ if start == -1:
|
||||
+ return b''
|
||||
+ return shebangline[start:]
|
||||
+
|
||||
+
|
||||
+def populate_flags(shebangline):
|
||||
+ old_flags = b''
|
||||
+ if keep_flags:
|
||||
+ old_flags = parse_shebang(shebangline)
|
||||
+ if old_flags:
|
||||
+ old_flags = old_flags[2:]
|
||||
+ if not (old_flags or add_flags):
|
||||
+ return b''
|
||||
+ # On Linux, the entire string following the interpreter name
|
||||
+ # is passed as a single argument to the interpreter.
|
||||
+ # e.g. "#! /usr/bin/python3 -W Error -s" runs "/usr/bin/python3 "-W Error -s"
|
||||
+ # so shebang should have single '-' where flags are given and
|
||||
+ # flag might need argument for that reasons adding new flags is
|
||||
+ # between '-' and original flags
|
||||
+ # e.g. #! /usr/bin/python3 -sW Error
|
||||
+ return b' -' + add_flags + old_flags
|
||||
+
|
||||
+
|
||||
def fixline(line):
|
||||
if not line.startswith(b'#!'):
|
||||
return line
|
||||
+
|
||||
if b"python" not in line:
|
||||
return line
|
||||
- return b'#! ' + new_interpreter + b'\n'
|
||||
+
|
||||
+ flags = populate_flags(line)
|
||||
+ return b'#! ' + new_interpreter + flags + b'\n'
|
||||
+
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
--
|
||||
2.29.2
|
||||
|
|
@ -0,0 +1,184 @@
|
|||
From e92381a0a6a3e1f000956e1f1e70e543b9c2bcd5 Mon Sep 17 00:00:00 2001
|
||||
From: Benjamin Peterson <benjamin@python.org>
|
||||
Date: Mon, 18 Jan 2021 14:47:05 -0600
|
||||
Subject: [PATCH] [3.6] closes bpo-42938: Replace snprintf with Python unicode
|
||||
formatting in ctypes param reprs. (24239). (cherry picked from commit
|
||||
916610ef90a0d0761f08747f7b0905541f0977c7)
|
||||
|
||||
Co-authored-by: Benjamin Peterson <benjamin@python.org>
|
||||
---
|
||||
Lib/ctypes/test/test_parameters.py | 43 +++++++++++++++
|
||||
.../2021-01-18-09-27-31.bpo-42938.4Zn4Mp.rst | 2 +
|
||||
Modules/_ctypes/callproc.c | 55 +++++++------------
|
||||
3 files changed, 66 insertions(+), 34 deletions(-)
|
||||
create mode 100644 Misc/NEWS.d/next/Security/2021-01-18-09-27-31.bpo-42938.4Zn4Mp.rst
|
||||
|
||||
diff --git a/Lib/ctypes/test/test_parameters.py b/Lib/ctypes/test/test_parameters.py
|
||||
index e4c25fd880cef..531894fdec838 100644
|
||||
--- a/Lib/ctypes/test/test_parameters.py
|
||||
+++ b/Lib/ctypes/test/test_parameters.py
|
||||
@@ -201,6 +201,49 @@ def __dict__(self):
|
||||
with self.assertRaises(ZeroDivisionError):
|
||||
WorseStruct().__setstate__({}, b'foo')
|
||||
|
||||
+ def test_parameter_repr(self):
|
||||
+ from ctypes import (
|
||||
+ c_bool,
|
||||
+ c_char,
|
||||
+ c_wchar,
|
||||
+ c_byte,
|
||||
+ c_ubyte,
|
||||
+ c_short,
|
||||
+ c_ushort,
|
||||
+ c_int,
|
||||
+ c_uint,
|
||||
+ c_long,
|
||||
+ c_ulong,
|
||||
+ c_longlong,
|
||||
+ c_ulonglong,
|
||||
+ c_float,
|
||||
+ c_double,
|
||||
+ c_longdouble,
|
||||
+ c_char_p,
|
||||
+ c_wchar_p,
|
||||
+ c_void_p,
|
||||
+ )
|
||||
+ self.assertRegex(repr(c_bool.from_param(True)), r"^<cparam '\?' at 0x[A-Fa-f0-9]+>$")
|
||||
+ self.assertEqual(repr(c_char.from_param(97)), "<cparam 'c' ('a')>")
|
||||
+ self.assertRegex(repr(c_wchar.from_param('a')), r"^<cparam 'u' at 0x[A-Fa-f0-9]+>$")
|
||||
+ self.assertEqual(repr(c_byte.from_param(98)), "<cparam 'b' (98)>")
|
||||
+ self.assertEqual(repr(c_ubyte.from_param(98)), "<cparam 'B' (98)>")
|
||||
+ self.assertEqual(repr(c_short.from_param(511)), "<cparam 'h' (511)>")
|
||||
+ self.assertEqual(repr(c_ushort.from_param(511)), "<cparam 'H' (511)>")
|
||||
+ self.assertRegex(repr(c_int.from_param(20000)), r"^<cparam '[li]' \(20000\)>$")
|
||||
+ self.assertRegex(repr(c_uint.from_param(20000)), r"^<cparam '[LI]' \(20000\)>$")
|
||||
+ self.assertRegex(repr(c_long.from_param(20000)), r"^<cparam '[li]' \(20000\)>$")
|
||||
+ self.assertRegex(repr(c_ulong.from_param(20000)), r"^<cparam '[LI]' \(20000\)>$")
|
||||
+ self.assertRegex(repr(c_longlong.from_param(20000)), r"^<cparam '[liq]' \(20000\)>$")
|
||||
+ self.assertRegex(repr(c_ulonglong.from_param(20000)), r"^<cparam '[LIQ]' \(20000\)>$")
|
||||
+ self.assertEqual(repr(c_float.from_param(1.5)), "<cparam 'f' (1.5)>")
|
||||
+ self.assertEqual(repr(c_double.from_param(1.5)), "<cparam 'd' (1.5)>")
|
||||
+ self.assertEqual(repr(c_double.from_param(1e300)), "<cparam 'd' (1e+300)>")
|
||||
+ self.assertRegex(repr(c_longdouble.from_param(1.5)), r"^<cparam ('d' \(1.5\)|'g' at 0x[A-Fa-f0-9]+)>$")
|
||||
+ self.assertRegex(repr(c_char_p.from_param(b'hihi')), "^<cparam 'z' \(0x[A-Fa-f0-9]+\)>$")
|
||||
+ self.assertRegex(repr(c_wchar_p.from_param('hihi')), "^<cparam 'Z' \(0x[A-Fa-f0-9]+\)>$")
|
||||
+ self.assertRegex(repr(c_void_p.from_param(0x12)), r"^<cparam 'P' \(0x0*12\)>$")
|
||||
+
|
||||
################################################################
|
||||
|
||||
if __name__ == '__main__':
|
||||
diff --git a/Misc/NEWS.d/next/Security/2021-01-18-09-27-31.bpo-42938.4Zn4Mp.rst b/Misc/NEWS.d/next/Security/2021-01-18-09-27-31.bpo-42938.4Zn4Mp.rst
|
||||
new file mode 100644
|
||||
index 0000000000000..7df65a156feab
|
||||
--- /dev/null
|
||||
+++ b/Misc/NEWS.d/next/Security/2021-01-18-09-27-31.bpo-42938.4Zn4Mp.rst
|
||||
@@ -0,0 +1,2 @@
|
||||
+Avoid static buffers when computing the repr of :class:`ctypes.c_double` and
|
||||
+:class:`ctypes.c_longdouble` values.
|
||||
diff --git a/Modules/_ctypes/callproc.c b/Modules/_ctypes/callproc.c
|
||||
index d1c190f359108..2bb289bce043f 100644
|
||||
--- a/Modules/_ctypes/callproc.c
|
||||
+++ b/Modules/_ctypes/callproc.c
|
||||
@@ -461,58 +461,47 @@ is_literal_char(unsigned char c)
|
||||
static PyObject *
|
||||
PyCArg_repr(PyCArgObject *self)
|
||||
{
|
||||
- char buffer[256];
|
||||
switch(self->tag) {
|
||||
case 'b':
|
||||
case 'B':
|
||||
- sprintf(buffer, "<cparam '%c' (%d)>",
|
||||
+ return PyUnicode_FromFormat("<cparam '%c' (%d)>",
|
||||
self->tag, self->value.b);
|
||||
- break;
|
||||
case 'h':
|
||||
case 'H':
|
||||
- sprintf(buffer, "<cparam '%c' (%d)>",
|
||||
+ return PyUnicode_FromFormat("<cparam '%c' (%d)>",
|
||||
self->tag, self->value.h);
|
||||
- break;
|
||||
case 'i':
|
||||
case 'I':
|
||||
- sprintf(buffer, "<cparam '%c' (%d)>",
|
||||
+ return PyUnicode_FromFormat("<cparam '%c' (%d)>",
|
||||
self->tag, self->value.i);
|
||||
- break;
|
||||
case 'l':
|
||||
case 'L':
|
||||
- sprintf(buffer, "<cparam '%c' (%ld)>",
|
||||
+ return PyUnicode_FromFormat("<cparam '%c' (%ld)>",
|
||||
self->tag, self->value.l);
|
||||
- break;
|
||||
|
||||
case 'q':
|
||||
case 'Q':
|
||||
- sprintf(buffer,
|
||||
-#ifdef MS_WIN32
|
||||
- "<cparam '%c' (%I64d)>",
|
||||
-#else
|
||||
- "<cparam '%c' (%lld)>",
|
||||
-#endif
|
||||
+ return PyUnicode_FromFormat("<cparam '%c' (%lld)>",
|
||||
self->tag, self->value.q);
|
||||
- break;
|
||||
case 'd':
|
||||
- sprintf(buffer, "<cparam '%c' (%f)>",
|
||||
- self->tag, self->value.d);
|
||||
- break;
|
||||
- case 'f':
|
||||
- sprintf(buffer, "<cparam '%c' (%f)>",
|
||||
- self->tag, self->value.f);
|
||||
- break;
|
||||
-
|
||||
+ case 'f': {
|
||||
+ PyObject *f = PyFloat_FromDouble((self->tag == 'f') ? self->value.f : self->value.d);
|
||||
+ if (f == NULL) {
|
||||
+ return NULL;
|
||||
+ }
|
||||
+ PyObject *result = PyUnicode_FromFormat("<cparam '%c' (%R)>", self->tag, f);
|
||||
+ Py_DECREF(f);
|
||||
+ return result;
|
||||
+ }
|
||||
case 'c':
|
||||
if (is_literal_char((unsigned char)self->value.c)) {
|
||||
- sprintf(buffer, "<cparam '%c' ('%c')>",
|
||||
+ return PyUnicode_FromFormat("<cparam '%c' ('%c')>",
|
||||
self->tag, self->value.c);
|
||||
}
|
||||
else {
|
||||
- sprintf(buffer, "<cparam '%c' ('\\x%02x')>",
|
||||
+ return PyUnicode_FromFormat("<cparam '%c' ('\\x%02x')>",
|
||||
self->tag, (unsigned char)self->value.c);
|
||||
}
|
||||
- break;
|
||||
|
||||
/* Hm, are these 'z' and 'Z' codes useful at all?
|
||||
Shouldn't they be replaced by the functionality of c_string
|
||||
@@ -521,22 +510,20 @@ PyCArg_repr(PyCArgObject *self)
|
||||
case 'z':
|
||||
case 'Z':
|
||||
case 'P':
|
||||
- sprintf(buffer, "<cparam '%c' (%p)>",
|
||||
+ return PyUnicode_FromFormat("<cparam '%c' (%p)>",
|
||||
self->tag, self->value.p);
|
||||
break;
|
||||
|
||||
default:
|
||||
if (is_literal_char((unsigned char)self->tag)) {
|
||||
- sprintf(buffer, "<cparam '%c' at %p>",
|
||||
- (unsigned char)self->tag, self);
|
||||
+ return PyUnicode_FromFormat("<cparam '%c' at %p>",
|
||||
+ (unsigned char)self->tag, (void *)self);
|
||||
}
|
||||
else {
|
||||
- sprintf(buffer, "<cparam 0x%02x at %p>",
|
||||
- (unsigned char)self->tag, self);
|
||||
+ return PyUnicode_FromFormat("<cparam 0x%02x at %p>",
|
||||
+ (unsigned char)self->tag, (void *)self);
|
||||
}
|
||||
- break;
|
||||
}
|
||||
- return PyUnicode_FromString(buffer);
|
||||
}
|
||||
|
||||
static PyMemberDef PyCArgType_members[] = {
|
|
@ -0,0 +1,684 @@
|
|||
commit 9e77ec82c40ab59846f9447b7c483e7b8e368b16
|
||||
Author: Petr Viktorin <pviktori@redhat.com>
|
||||
Date: Thu Mar 4 13:59:56 2021 +0100
|
||||
|
||||
CVE-2021-23336: Add `separator` argument to parse_qs; warn with default
|
||||
|
||||
Partially backports https://bugs.python.org/issue42967 : [security] Address a web cache-poisoning issue reported in urllib.parse.parse_qsl().
|
||||
However, this solution is different than the upstream solution in Python 3.6.13.
|
||||
|
||||
An optional argument seperator is added to specify the separator.
|
||||
It is recommended to set it to '&' or ';' to match the application or proxy in use.
|
||||
The default can be set with an env variable of a config file.
|
||||
If neither the argument, env var or config file specifies a separator, "&" is used
|
||||
but a warning is raised if parse_qs is used on input that contains ';'.
|
||||
|
||||
Co-authors of the upstream change (who do not necessarily agree with this):
|
||||
Co-authored-by: Adam Goldschmidt <adamgold7@gmail.com>
|
||||
Co-authored-by: Ken Jin <28750310+Fidget-Spinner@users.noreply.github.com>
|
||||
Co-authored-by: Éric Araujo <merwok@netwok.org>
|
||||
|
||||
diff --git a/Doc/library/cgi.rst b/Doc/library/cgi.rst
|
||||
index 41219eeaaba..ddecc0af23a 100644
|
||||
--- a/Doc/library/cgi.rst
|
||||
+++ b/Doc/library/cgi.rst
|
||||
@@ -277,13 +277,12 @@ These are useful if you want more control, or if you want to employ some of the
|
||||
algorithms implemented in this module in other circumstances.
|
||||
|
||||
|
||||
-.. function:: parse(fp=None, environ=os.environ, keep_blank_values=False, strict_parsing=False)
|
||||
+.. function:: parse(fp=None, environ=os.environ, keep_blank_values=False, strict_parsing=False, separator=None)
|
||||
|
||||
Parse a query in the environment or from a file (the file defaults to
|
||||
- ``sys.stdin``). The *keep_blank_values* and *strict_parsing* parameters are
|
||||
+ ``sys.stdin``). The *keep_blank_values*, *strict_parsing* and *separator* parameters are
|
||||
passed to :func:`urllib.parse.parse_qs` unchanged.
|
||||
|
||||
-
|
||||
.. function:: parse_qs(qs, keep_blank_values=False, strict_parsing=False)
|
||||
|
||||
This function is deprecated in this module. Use :func:`urllib.parse.parse_qs`
|
||||
@@ -308,7 +307,6 @@ algorithms implemented in this module in other circumstances.
|
||||
Note that this does not parse nested multipart parts --- use
|
||||
:class:`FieldStorage` for that.
|
||||
|
||||
-
|
||||
.. function:: parse_header(string)
|
||||
|
||||
Parse a MIME header (such as :mailheader:`Content-Type`) into a main value and a
|
||||
diff --git a/Doc/library/urllib.parse.rst b/Doc/library/urllib.parse.rst
|
||||
index 647af613a31..bcab7c142bc 100644
|
||||
--- a/Doc/library/urllib.parse.rst
|
||||
+++ b/Doc/library/urllib.parse.rst
|
||||
@@ -143,7 +143,7 @@ or on combining URL components into a URL string.
|
||||
now raise :exc:`ValueError`.
|
||||
|
||||
|
||||
-.. function:: parse_qs(qs, keep_blank_values=False, strict_parsing=False, encoding='utf-8', errors='replace', max_num_fields=None)
|
||||
+.. function:: parse_qs(qs, keep_blank_values=False, strict_parsing=False, encoding='utf-8', errors='replace', max_num_fields=None, separator=None)
|
||||
|
||||
Parse a query string given as a string argument (data of type
|
||||
:mimetype:`application/x-www-form-urlencoded`). Data are returned as a
|
||||
@@ -168,6 +168,15 @@ or on combining URL components into a URL string.
|
||||
read. If set, then throws a :exc:`ValueError` if there are more than
|
||||
*max_num_fields* fields read.
|
||||
|
||||
+ The optional argument *separator* is the symbol to use for separating the
|
||||
+ query arguments. It is recommended to set it to ``'&'`` or ``';'``.
|
||||
+ It defaults to ``'&'``; a warning is raised if this default is used.
|
||||
+ This default may be changed with the following environment variable settings:
|
||||
+
|
||||
+ - ``PYTHON_URLLIB_QS_SEPARATOR='&'``: use only ``&`` as separator, without warning (as in Python 3.6.13+ or 3.10)
|
||||
+ - ``PYTHON_URLLIB_QS_SEPARATOR=';'``: use only ``;`` as separator
|
||||
+ - ``PYTHON_URLLIB_QS_SEPARATOR=legacy``: use both ``&`` and ``;`` (as in previous versions of Python)
|
||||
+
|
||||
Use the :func:`urllib.parse.urlencode` function (with the ``doseq``
|
||||
parameter set to ``True``) to convert such dictionaries into query
|
||||
strings.
|
||||
@@ -204,6 +213,9 @@ or on combining URL components into a URL string.
|
||||
read. If set, then throws a :exc:`ValueError` if there are more than
|
||||
*max_num_fields* fields read.
|
||||
|
||||
+ The optional argument *separator* is the symbol to use for separating the
|
||||
+ query arguments. It works as in :py:func:`parse_qs`.
|
||||
+
|
||||
Use the :func:`urllib.parse.urlencode` function to convert such lists of pairs into
|
||||
query strings.
|
||||
|
||||
@@ -213,7 +225,6 @@ or on combining URL components into a URL string.
|
||||
.. versionchanged:: 3.6.8
|
||||
Added *max_num_fields* parameter.
|
||||
|
||||
-
|
||||
.. function:: urlunparse(parts)
|
||||
|
||||
Construct a URL from a tuple as returned by ``urlparse()``. The *parts*
|
||||
diff --git a/Lib/cgi.py b/Lib/cgi.py
|
||||
index 56f243e09f0..5ab2a5d6af6 100755
|
||||
--- a/Lib/cgi.py
|
||||
+++ b/Lib/cgi.py
|
||||
@@ -117,7 +117,8 @@ log = initlog # The current logging function
|
||||
# 0 ==> unlimited input
|
||||
maxlen = 0
|
||||
|
||||
-def parse(fp=None, environ=os.environ, keep_blank_values=0, strict_parsing=0):
|
||||
+def parse(fp=None, environ=os.environ, keep_blank_values=0,
|
||||
+ strict_parsing=0, separator=None):
|
||||
"""Parse a query in the environment or from a file (default stdin)
|
||||
|
||||
Arguments, all optional:
|
||||
@@ -136,6 +137,8 @@ def parse(fp=None, environ=os.environ, keep_blank_values=0, strict_parsing=0):
|
||||
strict_parsing: flag indicating what to do with parsing errors.
|
||||
If false (the default), errors are silently ignored.
|
||||
If true, errors raise a ValueError exception.
|
||||
+
|
||||
+ separator: str. The symbol to use for separating the query arguments.
|
||||
"""
|
||||
if fp is None:
|
||||
fp = sys.stdin
|
||||
@@ -156,7 +159,7 @@ def parse(fp=None, environ=os.environ, keep_blank_values=0, strict_parsing=0):
|
||||
if environ['REQUEST_METHOD'] == 'POST':
|
||||
ctype, pdict = parse_header(environ['CONTENT_TYPE'])
|
||||
if ctype == 'multipart/form-data':
|
||||
- return parse_multipart(fp, pdict)
|
||||
+ return parse_multipart(fp, pdict, separator=separator)
|
||||
elif ctype == 'application/x-www-form-urlencoded':
|
||||
clength = int(environ['CONTENT_LENGTH'])
|
||||
if maxlen and clength > maxlen:
|
||||
@@ -182,21 +185,21 @@ def parse(fp=None, environ=os.environ, keep_blank_values=0, strict_parsing=0):
|
||||
return urllib.parse.parse_qs(qs, keep_blank_values, strict_parsing,
|
||||
encoding=encoding)
|
||||
|
||||
-
|
||||
# parse query string function called from urlparse,
|
||||
# this is done in order to maintain backward compatibility.
|
||||
-
|
||||
-def parse_qs(qs, keep_blank_values=0, strict_parsing=0):
|
||||
+def parse_qs(qs, keep_blank_values=0, strict_parsing=0, separator=None):
|
||||
"""Parse a query given as a string argument."""
|
||||
warn("cgi.parse_qs is deprecated, use urllib.parse.parse_qs instead",
|
||||
DeprecationWarning, 2)
|
||||
- return urllib.parse.parse_qs(qs, keep_blank_values, strict_parsing)
|
||||
+ return urllib.parse.parse_qs(qs, keep_blank_values, strict_parsing,
|
||||
+ separator=separator)
|
||||
|
||||
-def parse_qsl(qs, keep_blank_values=0, strict_parsing=0):
|
||||
+def parse_qsl(qs, keep_blank_values=0, strict_parsing=0, separator=None):
|
||||
"""Parse a query given as a string argument."""
|
||||
warn("cgi.parse_qsl is deprecated, use urllib.parse.parse_qsl instead",
|
||||
DeprecationWarning, 2)
|
||||
- return urllib.parse.parse_qsl(qs, keep_blank_values, strict_parsing)
|
||||
+ return urllib.parse.parse_qsl(qs, keep_blank_values, strict_parsing,
|
||||
+ separator=separator)
|
||||
|
||||
def parse_multipart(fp, pdict):
|
||||
"""Parse multipart input.
|
||||
@@ -297,7 +300,6 @@ def parse_multipart(fp, pdict):
|
||||
|
||||
return partdict
|
||||
|
||||
-
|
||||
def _parseparam(s):
|
||||
while s[:1] == ';':
|
||||
s = s[1:]
|
||||
@@ -405,7 +407,7 @@ class FieldStorage:
|
||||
def __init__(self, fp=None, headers=None, outerboundary=b'',
|
||||
environ=os.environ, keep_blank_values=0, strict_parsing=0,
|
||||
limit=None, encoding='utf-8', errors='replace',
|
||||
- max_num_fields=None):
|
||||
+ max_num_fields=None, separator=None):
|
||||
"""Constructor. Read multipart/* until last part.
|
||||
|
||||
Arguments, all optional:
|
||||
@@ -453,6 +455,7 @@ class FieldStorage:
|
||||
self.keep_blank_values = keep_blank_values
|
||||
self.strict_parsing = strict_parsing
|
||||
self.max_num_fields = max_num_fields
|
||||
+ self.separator = separator
|
||||
if 'REQUEST_METHOD' in environ:
|
||||
method = environ['REQUEST_METHOD'].upper()
|
||||
self.qs_on_post = None
|
||||
@@ -678,7 +681,7 @@ class FieldStorage:
|
||||
query = urllib.parse.parse_qsl(
|
||||
qs, self.keep_blank_values, self.strict_parsing,
|
||||
encoding=self.encoding, errors=self.errors,
|
||||
- max_num_fields=self.max_num_fields)
|
||||
+ max_num_fields=self.max_num_fields, separator=self.separator)
|
||||
self.list = [MiniFieldStorage(key, value) for key, value in query]
|
||||
self.skip_lines()
|
||||
|
||||
@@ -694,7 +697,7 @@ class FieldStorage:
|
||||
query = urllib.parse.parse_qsl(
|
||||
self.qs_on_post, self.keep_blank_values, self.strict_parsing,
|
||||
encoding=self.encoding, errors=self.errors,
|
||||
- max_num_fields=self.max_num_fields)
|
||||
+ max_num_fields=self.max_num_fields, separator=self.separator)
|
||||
self.list.extend(MiniFieldStorage(key, value) for key, value in query)
|
||||
|
||||
klass = self.FieldStorageClass or self.__class__
|
||||
@@ -736,7 +739,8 @@ class FieldStorage:
|
||||
|
||||
part = klass(self.fp, headers, ib, environ, keep_blank_values,
|
||||
strict_parsing,self.limit-self.bytes_read,
|
||||
- self.encoding, self.errors, max_num_fields)
|
||||
+ self.encoding, self.errors, max_num_fields,
|
||||
+ separator=self.separator)
|
||||
|
||||
if max_num_fields is not None:
|
||||
max_num_fields -= 1
|
||||
diff --git a/Lib/test/test_cgi.py b/Lib/test/test_cgi.py
|
||||
index b3e2d4cce8e..5ae3e085e1e 100644
|
||||
--- a/Lib/test/test_cgi.py
|
||||
+++ b/Lib/test/test_cgi.py
|
||||
@@ -55,12 +55,9 @@ parse_strict_test_cases = [
|
||||
("", ValueError("bad query field: ''")),
|
||||
("&", ValueError("bad query field: ''")),
|
||||
("&&", ValueError("bad query field: ''")),
|
||||
- (";", ValueError("bad query field: ''")),
|
||||
- (";&;", ValueError("bad query field: ''")),
|
||||
# Should the next few really be valid?
|
||||
("=", {}),
|
||||
("=&=", {}),
|
||||
- ("=;=", {}),
|
||||
# This rest seem to make sense
|
||||
("=a", {'': ['a']}),
|
||||
("&=a", ValueError("bad query field: ''")),
|
||||
@@ -75,8 +72,6 @@ parse_strict_test_cases = [
|
||||
("a=a+b&b=b+c", {'a': ['a b'], 'b': ['b c']}),
|
||||
("a=a+b&a=b+a", {'a': ['a b', 'b a']}),
|
||||
("x=1&y=2.0&z=2-3.%2b0", {'x': ['1'], 'y': ['2.0'], 'z': ['2-3.+0']}),
|
||||
- ("x=1;y=2.0&z=2-3.%2b0", {'x': ['1'], 'y': ['2.0'], 'z': ['2-3.+0']}),
|
||||
- ("x=1;y=2.0;z=2-3.%2b0", {'x': ['1'], 'y': ['2.0'], 'z': ['2-3.+0']}),
|
||||
("Hbc5161168c542333633315dee1182227:key_store_seqid=400006&cuyer=r&view=bustomer&order_id=0bb2e248638833d48cb7fed300000f1b&expire=964546263&lobale=en-US&kid=130003.300038&ss=env",
|
||||
{'Hbc5161168c542333633315dee1182227:key_store_seqid': ['400006'],
|
||||
'cuyer': ['r'],
|
||||
@@ -164,6 +159,35 @@ class CgiTests(unittest.TestCase):
|
||||
|
||||
env = {'QUERY_STRING': orig}
|
||||
fs = cgi.FieldStorage(environ=env)
|
||||
+ if isinstance(expect, dict):
|
||||
+ # test dict interface
|
||||
+ self.assertEqual(len(expect), len(fs))
|
||||
+ self.assertCountEqual(expect.keys(), fs.keys())
|
||||
+ self.assertEqual(fs.getvalue("nonexistent field", "default"), "default")
|
||||
+ # test individual fields
|
||||
+ for key in expect.keys():
|
||||
+ expect_val = expect[key]
|
||||
+ self.assertIn(key, fs)
|
||||
+ if len(expect_val) > 1:
|
||||
+ self.assertEqual(fs.getvalue(key), expect_val)
|
||||
+ else:
|
||||
+ self.assertEqual(fs.getvalue(key), expect_val[0])
|
||||
+
|
||||
+ def test_separator(self):
|
||||
+ parse_semicolon = [
|
||||
+ ("x=1;y=2.0", {'x': ['1'], 'y': ['2.0']}),
|
||||
+ ("x=1;y=2.0;z=2-3.%2b0", {'x': ['1'], 'y': ['2.0'], 'z': ['2-3.+0']}),
|
||||
+ (";", ValueError("bad query field: ''")),
|
||||
+ (";;", ValueError("bad query field: ''")),
|
||||
+ ("=;a", ValueError("bad query field: 'a'")),
|
||||
+ (";b=a", ValueError("bad query field: ''")),
|
||||
+ ("b;=a", ValueError("bad query field: 'b'")),
|
||||
+ ("a=a+b;b=b+c", {'a': ['a b'], 'b': ['b c']}),
|
||||
+ ("a=a+b;a=b+a", {'a': ['a b', 'b a']}),
|
||||
+ ]
|
||||
+ for orig, expect in parse_semicolon:
|
||||
+ env = {'QUERY_STRING': orig}
|
||||
+ fs = cgi.FieldStorage(separator=';', environ=env)
|
||||
if isinstance(expect, dict):
|
||||
# test dict interface
|
||||
self.assertEqual(len(expect), len(fs))
|
||||
diff --git a/Lib/test/test_urlparse.py b/Lib/test/test_urlparse.py
|
||||
index 68f633ca3a7..1ec86ba0fc2 100644
|
||||
--- a/Lib/test/test_urlparse.py
|
||||
+++ b/Lib/test/test_urlparse.py
|
||||
@@ -2,6 +2,11 @@ import sys
|
||||
import unicodedata
|
||||
import unittest
|
||||
import urllib.parse
|
||||
+from test.support import EnvironmentVarGuard
|
||||
+from warnings import catch_warnings
|
||||
+import tempfile
|
||||
+import contextlib
|
||||
+import os.path
|
||||
|
||||
RFC1808_BASE = "http://a/b/c/d;p?q#f"
|
||||
RFC2396_BASE = "http://a/b/c/d;p?q"
|
||||
@@ -32,6 +37,9 @@ parse_qsl_test_cases = [
|
||||
(b"&a=b", [(b'a', b'b')]),
|
||||
(b"a=a+b&b=b+c", [(b'a', b'a b'), (b'b', b'b c')]),
|
||||
(b"a=1&a=2", [(b'a', b'1'), (b'a', b'2')]),
|
||||
+]
|
||||
+
|
||||
+parse_qsl_test_cases_semicolon = [
|
||||
(";", []),
|
||||
(";;", []),
|
||||
(";a=b", [('a', 'b')]),
|
||||
@@ -44,6 +52,21 @@ parse_qsl_test_cases = [
|
||||
(b"a=1;a=2", [(b'a', b'1'), (b'a', b'2')]),
|
||||
]
|
||||
|
||||
+parse_qsl_test_cases_legacy = [
|
||||
+ (b"a=1;a=2&a=3", [(b'a', b'1'), (b'a', b'2'), (b'a', b'3')]),
|
||||
+ (b"a=1;b=2&c=3", [(b'a', b'1'), (b'b', b'2'), (b'c', b'3')]),
|
||||
+ (b"a=1&b=2&c=3;", [(b'a', b'1'), (b'b', b'2'), (b'c', b'3')]),
|
||||
+]
|
||||
+
|
||||
+parse_qsl_test_cases_warn = [
|
||||
+ (";a=b", [(';a', 'b')]),
|
||||
+ ("a=a+b;b=b+c", [('a', 'a b;b=b c')]),
|
||||
+ (b";a=b", [(b';a', b'b')]),
|
||||
+ (b"a=a+b;b=b+c", [(b'a', b'a b;b=b c')]),
|
||||
+ ("a=1;a=2&a=3", [('a', '1;a=2'), ('a', '3')]),
|
||||
+ (b"a=1;a=2&a=3", [(b'a', b'1;a=2'), (b'a', b'3')]),
|
||||
+]
|
||||
+
|
||||
# Each parse_qs testcase is a two-tuple that contains
|
||||
# a string with the query and a dictionary with the expected result.
|
||||
|
||||
@@ -68,6 +91,9 @@ parse_qs_test_cases = [
|
||||
(b"&a=b", {b'a': [b'b']}),
|
||||
(b"a=a+b&b=b+c", {b'a': [b'a b'], b'b': [b'b c']}),
|
||||
(b"a=1&a=2", {b'a': [b'1', b'2']}),
|
||||
+]
|
||||
+
|
||||
+parse_qs_test_cases_semicolon = [
|
||||
(";", {}),
|
||||
(";;", {}),
|
||||
(";a=b", {'a': ['b']}),
|
||||
@@ -80,6 +106,24 @@ parse_qs_test_cases = [
|
||||
(b"a=1;a=2", {b'a': [b'1', b'2']}),
|
||||
]
|
||||
|
||||
+parse_qs_test_cases_legacy = [
|
||||
+ ("a=1;a=2&a=3", {'a': ['1', '2', '3']}),
|
||||
+ ("a=1;b=2&c=3", {'a': ['1'], 'b': ['2'], 'c': ['3']}),
|
||||
+ ("a=1&b=2&c=3;", {'a': ['1'], 'b': ['2'], 'c': ['3']}),
|
||||
+ (b"a=1;a=2&a=3", {b'a': [b'1', b'2', b'3']}),
|
||||
+ (b"a=1;b=2&c=3", {b'a': [b'1'], b'b': [b'2'], b'c': [b'3']}),
|
||||
+ (b"a=1&b=2&c=3;", {b'a': [b'1'], b'b': [b'2'], b'c': [b'3']}),
|
||||
+]
|
||||
+
|
||||
+parse_qs_test_cases_warn = [
|
||||
+ (";a=b", {';a': ['b']}),
|
||||
+ ("a=a+b;b=b+c", {'a': ['a b;b=b c']}),
|
||||
+ (b";a=b", {b';a': [b'b']}),
|
||||
+ (b"a=a+b;b=b+c", {b'a':[ b'a b;b=b c']}),
|
||||
+ ("a=1;a=2&a=3", {'a': ['1;a=2', '3']}),
|
||||
+ (b"a=1;a=2&a=3", {b'a': [b'1;a=2', b'3']}),
|
||||
+]
|
||||
+
|
||||
class UrlParseTestCase(unittest.TestCase):
|
||||
|
||||
def checkRoundtrips(self, url, parsed, split):
|
||||
@@ -152,6 +196,40 @@ class UrlParseTestCase(unittest.TestCase):
|
||||
self.assertEqual(result, expect_without_blanks,
|
||||
"Error parsing %r" % orig)
|
||||
|
||||
+ def test_qs_default_warn(self):
|
||||
+ for orig, expect in parse_qs_test_cases_warn:
|
||||
+ with self.subTest(orig=orig, expect=expect):
|
||||
+ with catch_warnings(record=True) as w:
|
||||
+ result = urllib.parse.parse_qs(orig, keep_blank_values=True)
|
||||
+ self.assertEqual(result, expect, "Error parsing %r" % orig)
|
||||
+ self.assertEqual(len(w), 1)
|
||||
+ self.assertEqual(w[0].category, urllib.parse._QueryStringSeparatorWarning)
|
||||
+
|
||||
+ def test_qsl_default_warn(self):
|
||||
+ for orig, expect in parse_qsl_test_cases_warn:
|
||||
+ with self.subTest(orig=orig, expect=expect):
|
||||
+ with catch_warnings(record=True) as w:
|
||||
+ result = urllib.parse.parse_qsl(orig, keep_blank_values=True)
|
||||
+ self.assertEqual(result, expect, "Error parsing %r" % orig)
|
||||
+ self.assertEqual(len(w), 1)
|
||||
+ self.assertEqual(w[0].category, urllib.parse._QueryStringSeparatorWarning)
|
||||
+
|
||||
+ def test_default_qs_no_warnings(self):
|
||||
+ for orig, expect in parse_qs_test_cases:
|
||||
+ with self.subTest(orig=orig, expect=expect):
|
||||
+ with catch_warnings(record=True) as w:
|
||||
+ result = urllib.parse.parse_qs(orig, keep_blank_values=True)
|
||||
+ self.assertEqual(result, expect, "Error parsing %r" % orig)
|
||||
+ self.assertEqual(len(w), 0)
|
||||
+
|
||||
+ def test_default_qsl_no_warnings(self):
|
||||
+ for orig, expect in parse_qsl_test_cases:
|
||||
+ with self.subTest(orig=orig, expect=expect):
|
||||
+ with catch_warnings(record=True) as w:
|
||||
+ result = urllib.parse.parse_qsl(orig, keep_blank_values=True)
|
||||
+ self.assertEqual(result, expect, "Error parsing %r" % orig)
|
||||
+ self.assertEqual(len(w), 0)
|
||||
+
|
||||
def test_roundtrips(self):
|
||||
str_cases = [
|
||||
('file:///tmp/junk.txt',
|
||||
@@ -885,8 +963,151 @@ class UrlParseTestCase(unittest.TestCase):
|
||||
with self.assertRaises(ValueError):
|
||||
urllib.parse.parse_qs('&'.join(['a=a']*11), max_num_fields=10)
|
||||
with self.assertRaises(ValueError):
|
||||
- urllib.parse.parse_qs(';'.join(['a=a']*11), max_num_fields=10)
|
||||
+ urllib.parse.parse_qs(';'.join(['a=a']*11), separator=';', max_num_fields=10)
|
||||
+ with self.assertRaises(ValueError):
|
||||
+ urllib.parse.parse_qs('SEP'.join(['a=a']*11), separator='SEP', max_num_fields=10)
|
||||
urllib.parse.parse_qs('&'.join(['a=a']*10), max_num_fields=10)
|
||||
+ urllib.parse.parse_qs(';'.join(['a=a']*10), separator=';', max_num_fields=10)
|
||||
+ urllib.parse.parse_qs('SEP'.join(['a=a']*10), separator='SEP', max_num_fields=10)
|
||||
+
|
||||
+ def test_parse_qs_separator_bytes(self):
|
||||
+ expected = {b'a': [b'1'], b'b': [b'2']}
|
||||
+
|
||||
+ result = urllib.parse.parse_qs(b'a=1;b=2', separator=b';')
|
||||
+ self.assertEqual(result, expected)
|
||||
+ result = urllib.parse.parse_qs(b'a=1;b=2', separator=';')
|
||||
+ self.assertEqual(result, expected)
|
||||
+ result = urllib.parse.parse_qs('a=1;b=2', separator=';')
|
||||
+ self.assertEqual(result, {'a': ['1'], 'b': ['2']})
|
||||
+
|
||||
+ @contextlib.contextmanager
|
||||
+ def _qsl_sep_config(self, sep):
|
||||
+ """Context for the given parse_qsl default separator configured in config file"""
|
||||
+ old_filename = urllib.parse._QS_SEPARATOR_CONFIG_FILENAME
|
||||
+ urllib.parse._default_qs_separator = None
|
||||
+ try:
|
||||
+ with tempfile.TemporaryDirectory() as tmpdirname:
|
||||
+ filename = os.path.join(tmpdirname, 'conf.cfg')
|
||||
+ with open(filename, 'w') as file:
|
||||
+ file.write(f'[parse_qs]\n')
|
||||
+ file.write(f'PYTHON_URLLIB_QS_SEPARATOR = {sep}')
|
||||
+ urllib.parse._QS_SEPARATOR_CONFIG_FILENAME = filename
|
||||
+ yield
|
||||
+ finally:
|
||||
+ urllib.parse._QS_SEPARATOR_CONFIG_FILENAME = old_filename
|
||||
+ urllib.parse._default_qs_separator = None
|
||||
+
|
||||
+ def test_parse_qs_separator_semicolon(self):
|
||||
+ for orig, expect in parse_qs_test_cases_semicolon:
|
||||
+ with self.subTest(orig=orig, expect=expect, method='arg'):
|
||||
+ result = urllib.parse.parse_qs(orig, separator=';')
|
||||
+ self.assertEqual(result, expect, "Error parsing %r" % orig)
|
||||
+ with self.subTest(orig=orig, expect=expect, method='env'):
|
||||
+ with EnvironmentVarGuard() as environ, catch_warnings(record=True) as w:
|
||||
+ environ['PYTHON_URLLIB_QS_SEPARATOR'] = ';'
|
||||
+ result = urllib.parse.parse_qs(orig)
|
||||
+ self.assertEqual(result, expect, "Error parsing %r" % orig)
|
||||
+ self.assertEqual(len(w), 0)
|
||||
+ with self.subTest(orig=orig, expect=expect, method='conf'):
|
||||
+ with self._qsl_sep_config(';'), catch_warnings(record=True) as w:
|
||||
+ result = urllib.parse.parse_qs(orig)
|
||||
+ self.assertEqual(result, expect, "Error parsing %r" % orig)
|
||||
+ self.assertEqual(len(w), 0)
|
||||
+
|
||||
+ def test_parse_qsl_separator_semicolon(self):
|
||||
+ for orig, expect in parse_qsl_test_cases_semicolon:
|
||||
+ with self.subTest(orig=orig, expect=expect, method='arg'):
|
||||
+ result = urllib.parse.parse_qsl(orig, separator=';')
|
||||
+ self.assertEqual(result, expect, "Error parsing %r" % orig)
|
||||
+ with self.subTest(orig=orig, expect=expect, method='env'):
|
||||
+ with EnvironmentVarGuard() as environ, catch_warnings(record=True) as w:
|
||||
+ environ['PYTHON_URLLIB_QS_SEPARATOR'] = ';'
|
||||
+ result = urllib.parse.parse_qsl(orig)
|
||||
+ self.assertEqual(result, expect, "Error parsing %r" % orig)
|
||||
+ self.assertEqual(len(w), 0)
|
||||
+ with self.subTest(orig=orig, expect=expect, method='conf'):
|
||||
+ with self._qsl_sep_config(';'), catch_warnings(record=True) as w:
|
||||
+ result = urllib.parse.parse_qsl(orig)
|
||||
+ self.assertEqual(result, expect, "Error parsing %r" % orig)
|
||||
+ self.assertEqual(len(w), 0)
|
||||
+
|
||||
+ def test_parse_qs_separator_legacy(self):
|
||||
+ for orig, expect in parse_qs_test_cases_legacy:
|
||||
+ with self.subTest(orig=orig, expect=expect, method='env'):
|
||||
+ with EnvironmentVarGuard() as environ, catch_warnings(record=True) as w:
|
||||
+ environ['PYTHON_URLLIB_QS_SEPARATOR'] = 'legacy'
|
||||
+ result = urllib.parse.parse_qs(orig)
|
||||
+ self.assertEqual(result, expect, "Error parsing %r" % orig)
|
||||
+ self.assertEqual(len(w), 0)
|
||||
+ with self.subTest(orig=orig, expect=expect, method='conf'):
|
||||
+ with self._qsl_sep_config('legacy'), catch_warnings(record=True) as w:
|
||||
+ result = urllib.parse.parse_qs(orig)
|
||||
+ self.assertEqual(result, expect, "Error parsing %r" % orig)
|
||||
+ self.assertEqual(len(w), 0)
|
||||
+
|
||||
+ def test_parse_qsl_separator_legacy(self):
|
||||
+ for orig, expect in parse_qsl_test_cases_legacy:
|
||||
+ with self.subTest(orig=orig, expect=expect, method='env'):
|
||||
+ with EnvironmentVarGuard() as environ, catch_warnings(record=True) as w:
|
||||
+ environ['PYTHON_URLLIB_QS_SEPARATOR'] = 'legacy'
|
||||
+ result = urllib.parse.parse_qsl(orig)
|
||||
+ self.assertEqual(result, expect, "Error parsing %r" % orig)
|
||||
+ self.assertEqual(len(w), 0)
|
||||
+ with self.subTest(orig=orig, expect=expect, method='conf'):
|
||||
+ with self._qsl_sep_config('legacy'), catch_warnings(record=True) as w:
|
||||
+ result = urllib.parse.parse_qsl(orig)
|
||||
+ self.assertEqual(result, expect, "Error parsing %r" % orig)
|
||||
+ self.assertEqual(len(w), 0)
|
||||
+
|
||||
+ def test_parse_qs_separator_bad_value_env_or_config(self):
|
||||
+ for bad_sep in '', 'abc', 'safe', '&;', 'SEP':
|
||||
+ with self.subTest(bad_sep, method='env'):
|
||||
+ with EnvironmentVarGuard() as environ, catch_warnings(record=True) as w:
|
||||
+ environ['PYTHON_URLLIB_QS_SEPARATOR'] = bad_sep
|
||||
+ with self.assertRaises(ValueError):
|
||||
+ urllib.parse.parse_qsl('a=1;b=2')
|
||||
+ with self.subTest(bad_sep, method='conf'):
|
||||
+ with self._qsl_sep_config('bad_sep'), catch_warnings(record=True) as w:
|
||||
+ with self.assertRaises(ValueError):
|
||||
+ urllib.parse.parse_qsl('a=1;b=2')
|
||||
+
|
||||
+ def test_parse_qs_separator_bad_value_arg(self):
|
||||
+ for bad_sep in True, {}, '':
|
||||
+ with self.subTest(bad_sep):
|
||||
+ with self.assertRaises(ValueError):
|
||||
+ urllib.parse.parse_qsl('a=1;b=2', separator=bad_sep)
|
||||
+
|
||||
+ def test_parse_qs_separator_num_fields(self):
|
||||
+ for qs, sep in (
|
||||
+ ('a&b&c', '&'),
|
||||
+ ('a;b;c', ';'),
|
||||
+ ('a&b;c', 'legacy'),
|
||||
+ ):
|
||||
+ with self.subTest(qs=qs, sep=sep):
|
||||
+ with EnvironmentVarGuard() as environ, catch_warnings(record=True) as w:
|
||||
+ if sep != 'legacy':
|
||||
+ with self.assertRaises(ValueError):
|
||||
+ urllib.parse.parse_qsl(qs, separator=sep, max_num_fields=2)
|
||||
+ if sep:
|
||||
+ environ['PYTHON_URLLIB_QS_SEPARATOR'] = sep
|
||||
+ with self.assertRaises(ValueError):
|
||||
+ urllib.parse.parse_qsl(qs, max_num_fields=2)
|
||||
+
|
||||
+ def test_parse_qs_separator_priority(self):
|
||||
+ # env variable trumps config file
|
||||
+ with self._qsl_sep_config('~'), EnvironmentVarGuard() as environ:
|
||||
+ environ['PYTHON_URLLIB_QS_SEPARATOR'] = '!'
|
||||
+ result = urllib.parse.parse_qs('a=1!b=2~c=3')
|
||||
+ self.assertEqual(result, {'a': ['1'], 'b': ['2~c=3']})
|
||||
+ # argument trumps config file
|
||||
+ with self._qsl_sep_config('~'):
|
||||
+ result = urllib.parse.parse_qs('a=1$b=2~c=3', separator='$')
|
||||
+ self.assertEqual(result, {'a': ['1'], 'b': ['2~c=3']})
|
||||
+ # argument trumps env variable
|
||||
+ with EnvironmentVarGuard() as environ:
|
||||
+ environ['PYTHON_URLLIB_QS_SEPARATOR'] = '~'
|
||||
+ result = urllib.parse.parse_qs('a=1$b=2~c=3', separator='$')
|
||||
+ self.assertEqual(result, {'a': ['1'], 'b': ['2~c=3']})
|
||||
|
||||
def test_urlencode_sequences(self):
|
||||
# Other tests incidentally urlencode things; test non-covered cases:
|
||||
diff --git a/Lib/urllib/parse.py b/Lib/urllib/parse.py
|
||||
index fa8827a9fa7..57b8fcf8bbd 100644
|
||||
--- a/Lib/urllib/parse.py
|
||||
+++ b/Lib/urllib/parse.py
|
||||
@@ -28,6 +28,7 @@ test_urlparse.py provides a good indicator of parsing behavior.
|
||||
"""
|
||||
|
||||
import re
|
||||
+import os
|
||||
import sys
|
||||
import collections
|
||||
|
||||
@@ -644,7 +645,8 @@ def unquote(string, encoding='utf-8', errors='replace'):
|
||||
|
||||
|
||||
def parse_qs(qs, keep_blank_values=False, strict_parsing=False,
|
||||
- encoding='utf-8', errors='replace', max_num_fields=None):
|
||||
+ encoding='utf-8', errors='replace', max_num_fields=None,
|
||||
+ separator=None):
|
||||
"""Parse a query given as a string argument.
|
||||
|
||||
Arguments:
|
||||
@@ -673,7 +675,8 @@ def parse_qs(qs, keep_blank_values=False, strict_parsing=False,
|
||||
parsed_result = {}
|
||||
pairs = parse_qsl(qs, keep_blank_values, strict_parsing,
|
||||
encoding=encoding, errors=errors,
|
||||
- max_num_fields=max_num_fields)
|
||||
+ max_num_fields=max_num_fields,
|
||||
+ separator=separator)
|
||||
for name, value in pairs:
|
||||
if name in parsed_result:
|
||||
parsed_result[name].append(value)
|
||||
@@ -681,9 +684,16 @@ def parse_qs(qs, keep_blank_values=False, strict_parsing=False,
|
||||
parsed_result[name] = [value]
|
||||
return parsed_result
|
||||
|
||||
+class _QueryStringSeparatorWarning(RuntimeWarning):
|
||||
+ """Warning for using default `separator` in parse_qs or parse_qsl"""
|
||||
+
|
||||
+# The default "separator" for parse_qsl can be specified in a config file.
|
||||
+# It's cached after first read.
|
||||
+_QS_SEPARATOR_CONFIG_FILENAME = '/etc/python/urllib.cfg'
|
||||
+_default_qs_separator = None
|
||||
|
||||
def parse_qsl(qs, keep_blank_values=False, strict_parsing=False,
|
||||
- encoding='utf-8', errors='replace', max_num_fields=None):
|
||||
+ encoding='utf-8', errors='replace', max_num_fields=None, separator=None):
|
||||
"""Parse a query given as a string argument.
|
||||
|
||||
Arguments:
|
||||
@@ -710,15 +720,77 @@ def parse_qsl(qs, keep_blank_values=False, strict_parsing=False,
|
||||
"""
|
||||
qs, _coerce_result = _coerce_args(qs)
|
||||
|
||||
+ if isinstance(separator, bytes):
|
||||
+ separator = separator.decode('ascii')
|
||||
+
|
||||
+ if (not separator or (not isinstance(separator, (str, bytes)))) and separator is not None:
|
||||
+ raise ValueError("Separator must be of type string or bytes.")
|
||||
+
|
||||
+ # Used when both "&" and ";" act as separators. (Need a non-string value.)
|
||||
+ _legacy = object()
|
||||
+
|
||||
+ if separator is None:
|
||||
+ global _default_qs_separator
|
||||
+ separator = _default_qs_separator
|
||||
+ envvar_name = 'PYTHON_URLLIB_QS_SEPARATOR'
|
||||
+ if separator is None:
|
||||
+ # Set default separator from environment variable
|
||||
+ separator = os.environ.get(envvar_name)
|
||||
+ config_source = 'environment variable'
|
||||
+ if separator is None:
|
||||
+ # Set default separator from the configuration file
|
||||
+ try:
|
||||
+ file = open(_QS_SEPARATOR_CONFIG_FILENAME)
|
||||
+ except FileNotFoundError:
|
||||
+ pass
|
||||
+ else:
|
||||
+ with file:
|
||||
+ import configparser
|
||||
+ config = configparser.ConfigParser(
|
||||
+ interpolation=None,
|
||||
+ comment_prefixes=('#', ),
|
||||
+ )
|
||||
+ config.read_file(file)
|
||||
+ separator = config.get('parse_qs', envvar_name, fallback=None)
|
||||
+ _default_qs_separator = separator
|
||||
+ config_source = _QS_SEPARATOR_CONFIG_FILENAME
|
||||
+ if separator is None:
|
||||
+ # The default is '&', but warn if not specified explicitly
|
||||
+ if ';' in qs:
|
||||
+ from warnings import warn
|
||||
+ warn("The default separator of urllib.parse.parse_qsl and "
|
||||
+ + "parse_qs was changed to '&' to avoid a web cache "
|
||||
+ + "poisoning issue (CVE-2021-23336). "
|
||||
+ + "By default, semicolons no longer act as query field "
|
||||
+ + "separators. "
|
||||
+ + "See https://access.redhat.com/articles/5860431 for "
|
||||
+ + "more details.",
|
||||
+ _QueryStringSeparatorWarning, stacklevel=2)
|
||||
+ separator = '&'
|
||||
+ elif separator == 'legacy':
|
||||
+ separator = _legacy
|
||||
+ elif len(separator) != 1:
|
||||
+ raise ValueError(
|
||||
+ f'{envvar_name} (from {config_source}) must contain '
|
||||
+ + '1 character, or "legacy". See '
|
||||
+ + 'https://access.redhat.com/articles/5860431 for more details.'
|
||||
+ )
|
||||
+
|
||||
# If max_num_fields is defined then check that the number of fields
|
||||
# is less than max_num_fields. This prevents a memory exhaustion DOS
|
||||
# attack via post bodies with many fields.
|
||||
if max_num_fields is not None:
|
||||
- num_fields = 1 + qs.count('&') + qs.count(';')
|
||||
+ if separator is _legacy:
|
||||
+ num_fields = 1 + qs.count('&') + qs.count(';')
|
||||
+ else:
|
||||
+ num_fields = 1 + qs.count(separator)
|
||||
if max_num_fields < num_fields:
|
||||
raise ValueError('Max number of fields exceeded')
|
||||
|
||||
- pairs = [s2 for s1 in qs.split('&') for s2 in s1.split(';')]
|
||||
+ if separator is _legacy:
|
||||
+ pairs = [s2 for s1 in qs.split('&') for s2 in s1.split(';')]
|
||||
+ else:
|
||||
+ pairs = [s1 for s1 in qs.split(separator)]
|
||||
r = []
|
||||
for name_value in pairs:
|
||||
if not name_value and not strict_parsing:
|
||||
diff --git a/Misc/NEWS.d/next/Security/2021-02-14-15-59-16.bpo-42967.YApqDS.rst b/Misc/NEWS.d/next/Security/2021-02-14-15-59-16.bpo-42967.YApqDS.rst
|
||||
new file mode 100644
|
||||
index 00000000000..bc82c963067
|
||||
--- /dev/null
|
||||
+++ b/Misc/NEWS.d/next/Security/2021-02-14-15-59-16.bpo-42967.YApqDS.rst
|
||||
@@ -0,0 +1 @@
|
||||
+Make it possible to fix web cache poisoning vulnerability by allowing the user to choose a custom separator query args.
|
|
@ -0,0 +1,101 @@
|
|||
From 5b1e50256b6532667b6d31debc350f6c7d3f30aa Mon Sep 17 00:00:00 2001
|
||||
From: "Miss Islington (bot)"
|
||||
<31488909+miss-islington@users.noreply.github.com>
|
||||
Date: Mon, 29 Mar 2021 08:40:53 -0700
|
||||
Subject: [PATCH] bpo-42988: Remove the pydoc getfile feature (GH-25015)
|
||||
(GH-25067)
|
||||
MIME-Version: 1.0
|
||||
Content-Type: text/plain; charset=UTF-8
|
||||
Content-Transfer-Encoding: 8bit
|
||||
|
||||
CVE-2021-3426: Remove the "getfile" feature of the pydoc module which
|
||||
could be abused to read arbitrary files on the disk (directory
|
||||
traversal vulnerability). Moreover, even source code of Python
|
||||
modules can contain sensitive data like passwords. Vulnerability
|
||||
reported by David Schwörer.
|
||||
(cherry picked from commit 9b999479c0022edfc9835a8a1f06e046f3881048)
|
||||
|
||||
Co-authored-by: Victor Stinner <vstinner@python.org>
|
||||
---
|
||||
Lib/pydoc.py | 18 ------------------
|
||||
Lib/test/test_pydoc.py | 6 ------
|
||||
.../2021-03-24-14-16-56.bpo-42988.P2aNco.rst | 4 ++++
|
||||
3 files changed, 4 insertions(+), 24 deletions(-)
|
||||
create mode 100644 Misc/NEWS.d/next/Security/2021-03-24-14-16-56.bpo-42988.P2aNco.rst
|
||||
|
||||
diff --git a/Lib/pydoc.py b/Lib/pydoc.py
|
||||
index b521a5504728c4..5247ef9ea27aa1 100644
|
||||
--- a/Lib/pydoc.py
|
||||
+++ b/Lib/pydoc.py
|
||||
@@ -2312,9 +2312,6 @@ def page(self, title, contents):
|
||||
%s</head><body bgcolor="#f0f0f8">%s<div style="clear:both;padding-top:.5em;">%s</div>
|
||||
</body></html>''' % (title, css_link, html_navbar(), contents)
|
||||
|
||||
- def filelink(self, url, path):
|
||||
- return '<a href="getfile?key=%s">%s</a>' % (url, path)
|
||||
-
|
||||
|
||||
html = _HTMLDoc()
|
||||
|
||||
@@ -2400,19 +2397,6 @@ def bltinlink(name):
|
||||
'key = %s' % key, '#ffffff', '#ee77aa', '<br>'.join(results))
|
||||
return 'Search Results', contents
|
||||
|
||||
- def html_getfile(path):
|
||||
- """Get and display a source file listing safely."""
|
||||
- path = urllib.parse.unquote(path)
|
||||
- with tokenize.open(path) as fp:
|
||||
- lines = html.escape(fp.read())
|
||||
- body = '<pre>%s</pre>' % lines
|
||||
- heading = html.heading(
|
||||
- '<big><big><strong>File Listing</strong></big></big>',
|
||||
- '#ffffff', '#7799ee')
|
||||
- contents = heading + html.bigsection(
|
||||
- 'File: %s' % path, '#ffffff', '#ee77aa', body)
|
||||
- return 'getfile %s' % path, contents
|
||||
-
|
||||
def html_topics():
|
||||
"""Index of topic texts available."""
|
||||
|
||||
@@ -2504,8 +2488,6 @@ def get_html_page(url):
|
||||
op, _, url = url.partition('=')
|
||||
if op == "search?key":
|
||||
title, content = html_search(url)
|
||||
- elif op == "getfile?key":
|
||||
- title, content = html_getfile(url)
|
||||
elif op == "topic?key":
|
||||
# try topics first, then objects.
|
||||
try:
|
||||
diff --git a/Lib/test/test_pydoc.py b/Lib/test/test_pydoc.py
|
||||
index 00803d3305cb53..49bc3eb164b19c 100644
|
||||
--- a/Lib/test/test_pydoc.py
|
||||
+++ b/Lib/test/test_pydoc.py
|
||||
@@ -1052,18 +1052,12 @@ def test_url_requests(self):
|
||||
("topic?key=def", "Pydoc: KEYWORD def"),
|
||||
("topic?key=STRINGS", "Pydoc: TOPIC STRINGS"),
|
||||
("foobar", "Pydoc: Error - foobar"),
|
||||
- ("getfile?key=foobar", "Pydoc: Error - getfile?key=foobar"),
|
||||
]
|
||||
|
||||
with self.restrict_walk_packages():
|
||||
for url, title in requests:
|
||||
self.call_url_handler(url, title)
|
||||
|
||||
- path = string.__file__
|
||||
- title = "Pydoc: getfile " + path
|
||||
- url = "getfile?key=" + path
|
||||
- self.call_url_handler(url, title)
|
||||
-
|
||||
|
||||
class TestHelper(unittest.TestCase):
|
||||
def test_keywords(self):
|
||||
diff --git a/Misc/NEWS.d/next/Security/2021-03-24-14-16-56.bpo-42988.P2aNco.rst b/Misc/NEWS.d/next/Security/2021-03-24-14-16-56.bpo-42988.P2aNco.rst
|
||||
new file mode 100644
|
||||
index 00000000000000..4b42dd05305a83
|
||||
--- /dev/null
|
||||
+++ b/Misc/NEWS.d/next/Security/2021-03-24-14-16-56.bpo-42988.P2aNco.rst
|
||||
@@ -0,0 +1,4 @@
|
||||
+CVE-2021-3426: Remove the ``getfile`` feature of the :mod:`pydoc` module which
|
||||
+could be abused to read arbitrary files on the disk (directory traversal
|
||||
+vulnerability). Moreover, even source code of Python modules can contain
|
||||
+sensitive data like passwords. Vulnerability reported by David Schwörer.
|
|
@ -0,0 +1,36 @@
|
|||
bpo-44422: Fix threading.enumerate() reentrant call (GH-26727)
|
||||
|
||||
The threading.enumerate() function now uses a reentrant lock to
|
||||
prevent a hang on reentrant call.
|
||||
|
||||
https://github.com/python/cpython/commit/243fd01047ddce1a7eb0f99a49732d123e942c63
|
||||
|
||||
Resolves: rhbz#1959459
|
||||
|
||||
diff --git a/Lib/threading.py b/Lib/threading.py
|
||||
index 0ab1e46..7ab9ad8 100644
|
||||
--- a/Lib/threading.py
|
||||
+++ b/Lib/threading.py
|
||||
@@ -727,8 +727,11 @@ _counter() # Consume 0 so first non-main thread has id 1.
|
||||
def _newname(template="Thread-%d"):
|
||||
return template % _counter()
|
||||
|
||||
-# Active thread administration
|
||||
-_active_limbo_lock = _allocate_lock()
|
||||
+# Active thread administration.
|
||||
+#
|
||||
+# bpo-44422: Use a reentrant lock to allow reentrant calls to functions like
|
||||
+# threading.enumerate().
|
||||
+_active_limbo_lock = RLock()
|
||||
_active = {} # maps thread id to Thread object
|
||||
_limbo = {}
|
||||
_dangling = WeakSet()
|
||||
@@ -1325,7 +1328,7 @@ def _after_fork():
|
||||
# Reset _active_limbo_lock, in case we forked while the lock was held
|
||||
# by another (non-forked) thread. http://bugs.python.org/issue874900
|
||||
global _active_limbo_lock, _main_thread
|
||||
- _active_limbo_lock = _allocate_lock()
|
||||
+ _active_limbo_lock = RLock()
|
||||
|
||||
# fork() only copied the current thread; clear references to others.
|
||||
new_active = {}
|
|
@ -0,0 +1,43 @@
|
|||
bpo-44434: Don't call PyThread_exit_thread() explicitly (GH-26758)
|
||||
|
||||
_thread.start_new_thread() no longer calls PyThread_exit_thread()
|
||||
explicitly at the thread exit, the call was redundant.
|
||||
|
||||
On Linux with the glibc, pthread_cancel() loads dynamically the
|
||||
libgcc_s.so.1 library. dlopen() can fail if there is no more
|
||||
available file descriptor to open the file. In this case, the process
|
||||
aborts with the error message:
|
||||
|
||||
"libgcc_s.so.1 must be installed for pthread_cancel to work"
|
||||
|
||||
pthread_cancel() unwinds back to the thread's wrapping function that
|
||||
calls the thread entry point.
|
||||
|
||||
The unwind function is dynamically loaded from the libgcc_s library
|
||||
since it is tightly coupled to the C compiler (GCC). The unwinder
|
||||
depends on DWARF, the compiler generates DWARF, so the unwinder
|
||||
belongs to the compiler.
|
||||
|
||||
Thanks Florian Weimer and Carlos O'Donell for their help on
|
||||
investigating this issue.
|
||||
|
||||
https://github.com/python/cpython/commit/45a78f906d2d5fe5381d78466b11763fc56d57ba
|
||||
|
||||
Resolves: rhbz#1972293
|
||||
|
||||
diff --git a/Modules/_threadmodule.c b/Modules/_threadmodule.c
|
||||
index a13b2e0..8cc035b 100644
|
||||
--- a/Modules/_threadmodule.c
|
||||
+++ b/Modules/_threadmodule.c
|
||||
@@ -1027,7 +1027,10 @@ t_bootstrap(void *boot_raw)
|
||||
nb_threads--;
|
||||
PyThreadState_Clear(tstate);
|
||||
PyThreadState_DeleteCurrent();
|
||||
- PyThread_exit_thread();
|
||||
+
|
||||
+ // bpo-44434: Don't call explicitly PyThread_exit_thread(). On Linux with
|
||||
+ // the glibc, pthread_exit() can abort the whole process if dlopen() fails
|
||||
+ // to open the libgcc_s.so library (ex: EMFILE error).
|
||||
}
|
||||
|
||||
static PyObject *
|
|
@ -0,0 +1,40 @@
|
|||
From 29c669440dddba61d18e1b7fdd57180cae9e4ae3 Mon Sep 17 00:00:00 2001
|
||||
From: Yeting Li <liyt@ios.ac.cn>
|
||||
Date: Wed, 7 Apr 2021 19:27:41 +0800
|
||||
Subject: [PATCH] bpo-43075: Fix ReDoS in urllib AbstractBasicAuthHandler
|
||||
(GH-24391)
|
||||
|
||||
Fix Regular Expression Denial of Service (ReDoS) vulnerability in
|
||||
urllib.request.AbstractBasicAuthHandler. The ReDoS-vulnerable regex
|
||||
has quadratic worst-case complexity and it allows cause a denial of
|
||||
service when identifying crafted invalid RFCs. This ReDoS issue is on
|
||||
the client side and needs remote attackers to control the HTTP server.
|
||||
(cherry picked from commit 7215d1ae25525c92b026166f9d5cac85fb1defe1)
|
||||
|
||||
Co-authored-by: Yeting Li <liyt@ios.ac.cn>
|
||||
---
|
||||
Lib/urllib/request.py | 2 +-
|
||||
.../next/Security/2021-01-31-05-28-14.bpo-43075.DoAXqO.rst | 1 +
|
||||
2 files changed, 2 insertions(+), 1 deletion(-)
|
||||
create mode 100644 Misc/NEWS.d/next/Security/2021-01-31-05-28-14.bpo-43075.DoAXqO.rst
|
||||
|
||||
diff --git a/Lib/urllib/request.py b/Lib/urllib/request.py
|
||||
index 6624e04317ba2..56565405a7097 100644
|
||||
--- a/Lib/urllib/request.py
|
||||
+++ b/Lib/urllib/request.py
|
||||
@@ -947,7 +947,7 @@ class AbstractBasicAuthHandler:
|
||||
# (single quotes are a violation of the RFC, but appear in the wild)
|
||||
rx = re.compile('(?:^|,)' # start of the string or ','
|
||||
'[ \t]*' # optional whitespaces
|
||||
- '([^ \t]+)' # scheme like "Basic"
|
||||
+ '([^ \t,]+)' # scheme like "Basic"
|
||||
'[ \t]+' # mandatory whitespaces
|
||||
# realm=xxx
|
||||
# realm='xxx'
|
||||
diff --git a/Misc/NEWS.d/next/Security/2021-01-31-05-28-14.bpo-43075.DoAXqO.rst b/Misc/NEWS.d/next/Security/2021-01-31-05-28-14.bpo-43075.DoAXqO.rst
|
||||
new file mode 100644
|
||||
index 0000000000000..1c9f727e965fb
|
||||
--- /dev/null
|
||||
+++ b/Misc/NEWS.d/next/Security/2021-01-31-05-28-14.bpo-43075.DoAXqO.rst
|
||||
@@ -0,0 +1 @@
|
||||
+Fix Regular Expression Denial of Service (ReDoS) vulnerability in :class:`urllib.request.AbstractBasicAuthHandler`. The ReDoS-vulnerable regex has quadratic worst-case complexity and it allows cause a denial of service when identifying crafted invalid RFCs. This ReDoS issue is on the client side and needs remote attackers to control the HTTP server.
|
|
@ -0,0 +1,119 @@
|
|||
From f7fb35b563a9182c22fbdd03c72ec3724dafe918 Mon Sep 17 00:00:00 2001
|
||||
From: Gen Xu <xgbarry@gmail.com>
|
||||
Date: Wed, 5 May 2021 15:42:41 -0700
|
||||
Subject: [PATCH] bpo-44022: Fix http client infinite line reading (DoS) after
|
||||
a HTTP 100 Continue (GH-25916)
|
||||
|
||||
Fixes http.client potential denial of service where it could get stuck reading lines from a malicious server after a 100 Continue response.
|
||||
|
||||
Co-authored-by: Gregory P. Smith <greg@krypto.org>
|
||||
(cherry picked from commit 47895e31b6f626bc6ce47d175fe9d43c1098909d)
|
||||
|
||||
Co-authored-by: Gen Xu <xgbarry@gmail.com>
|
||||
---
|
||||
Lib/http/client.py | 38 ++++++++++---------
|
||||
Lib/test/test_httplib.py | 10 ++++-
|
||||
.../2021-05-05-17-37-04.bpo-44022.bS3XJ9.rst | 2 +
|
||||
3 files changed, 32 insertions(+), 18 deletions(-)
|
||||
create mode 100644 Misc/NEWS.d/next/Security/2021-05-05-17-37-04.bpo-44022.bS3XJ9.rst
|
||||
|
||||
diff --git a/Lib/http/client.py b/Lib/http/client.py
|
||||
index 53581eca20587..07e675fac5981 100644
|
||||
--- a/Lib/http/client.py
|
||||
+++ b/Lib/http/client.py
|
||||
@@ -205,15 +205,11 @@ def getallmatchingheaders(self, name):
|
||||
lst.append(line)
|
||||
return lst
|
||||
|
||||
-def parse_headers(fp, _class=HTTPMessage):
|
||||
- """Parses only RFC2822 headers from a file pointer.
|
||||
-
|
||||
- email Parser wants to see strings rather than bytes.
|
||||
- But a TextIOWrapper around self.rfile would buffer too many bytes
|
||||
- from the stream, bytes which we later need to read as bytes.
|
||||
- So we read the correct bytes here, as bytes, for email Parser
|
||||
- to parse.
|
||||
+def _read_headers(fp):
|
||||
+ """Reads potential header lines into a list from a file pointer.
|
||||
|
||||
+ Length of line is limited by _MAXLINE, and number of
|
||||
+ headers is limited by _MAXHEADERS.
|
||||
"""
|
||||
headers = []
|
||||
while True:
|
||||
@@ -225,6 +221,19 @@ def parse_headers(fp, _class=HTTPMessage):
|
||||
raise HTTPException("got more than %d headers" % _MAXHEADERS)
|
||||
if line in (b'\r\n', b'\n', b''):
|
||||
break
|
||||
+ return headers
|
||||
+
|
||||
+def parse_headers(fp, _class=HTTPMessage):
|
||||
+ """Parses only RFC2822 headers from a file pointer.
|
||||
+
|
||||
+ email Parser wants to see strings rather than bytes.
|
||||
+ But a TextIOWrapper around self.rfile would buffer too many bytes
|
||||
+ from the stream, bytes which we later need to read as bytes.
|
||||
+ So we read the correct bytes here, as bytes, for email Parser
|
||||
+ to parse.
|
||||
+
|
||||
+ """
|
||||
+ headers = _read_headers(fp)
|
||||
hstring = b''.join(headers).decode('iso-8859-1')
|
||||
return email.parser.Parser(_class=_class).parsestr(hstring)
|
||||
|
||||
@@ -312,15 +321,10 @@ def begin(self):
|
||||
if status != CONTINUE:
|
||||
break
|
||||
# skip the header from the 100 response
|
||||
- while True:
|
||||
- skip = self.fp.readline(_MAXLINE + 1)
|
||||
- if len(skip) > _MAXLINE:
|
||||
- raise LineTooLong("header line")
|
||||
- skip = skip.strip()
|
||||
- if not skip:
|
||||
- break
|
||||
- if self.debuglevel > 0:
|
||||
- print("header:", skip)
|
||||
+ skipped_headers = _read_headers(self.fp)
|
||||
+ if self.debuglevel > 0:
|
||||
+ print("headers:", skipped_headers)
|
||||
+ del skipped_headers
|
||||
|
||||
self.code = self.status = status
|
||||
self.reason = reason.strip()
|
||||
diff --git a/Lib/test/test_httplib.py b/Lib/test/test_httplib.py
|
||||
index 03e049b13fd21..0db287507c7bf 100644
|
||||
--- a/Lib/test/test_httplib.py
|
||||
+++ b/Lib/test/test_httplib.py
|
||||
@@ -971,6 +971,14 @@ def test_overflowing_header_line(self):
|
||||
resp = client.HTTPResponse(FakeSocket(body))
|
||||
self.assertRaises(client.LineTooLong, resp.begin)
|
||||
|
||||
+ def test_overflowing_header_limit_after_100(self):
|
||||
+ body = (
|
||||
+ 'HTTP/1.1 100 OK\r\n'
|
||||
+ 'r\n' * 32768
|
||||
+ )
|
||||
+ resp = client.HTTPResponse(FakeSocket(body))
|
||||
+ self.assertRaises(client.HTTPException, resp.begin)
|
||||
+
|
||||
def test_overflowing_chunked_line(self):
|
||||
body = (
|
||||
'HTTP/1.1 200 OK\r\n'
|
||||
@@ -1377,7 +1385,7 @@ def readline(self, limit):
|
||||
class OfflineTest(TestCase):
|
||||
def test_all(self):
|
||||
# Documented objects defined in the module should be in __all__
|
||||
- expected = {"responses"} # White-list documented dict() object
|
||||
+ expected = {"responses"} # Allowlist documented dict() object
|
||||
# HTTPMessage, parse_headers(), and the HTTP status code constants are
|
||||
# intentionally omitted for simplicity
|
||||
blacklist = {"HTTPMessage", "parse_headers"}
|
||||
diff --git a/Misc/NEWS.d/next/Security/2021-05-05-17-37-04.bpo-44022.bS3XJ9.rst b/Misc/NEWS.d/next/Security/2021-05-05-17-37-04.bpo-44022.bS3XJ9.rst
|
||||
new file mode 100644
|
||||
index 0000000000000..cf6b63e396155
|
||||
--- /dev/null
|
||||
+++ b/Misc/NEWS.d/next/Security/2021-05-05-17-37-04.bpo-44022.bS3XJ9.rst
|
||||
@@ -0,0 +1,2 @@
|
||||
+mod:`http.client` now avoids infinitely reading potential HTTP headers after a
|
||||
+``100 Continue`` status response from the server.
|
|
@ -0,0 +1,74 @@
|
|||
diff --git a/Lib/logging/handlers.py b/Lib/logging/handlers.py
|
||||
index 11ebcf1..ee3d960 100644
|
||||
--- a/Lib/logging/handlers.py
|
||||
+++ b/Lib/logging/handlers.py
|
||||
@@ -181,14 +181,17 @@ class RotatingFileHandler(BaseRotatingHandler):
|
||||
Basically, see if the supplied record would cause the file to exceed
|
||||
the size limit we have.
|
||||
"""
|
||||
+ # See bpo-45401: Never rollover anything other than regular files
|
||||
+ if os.path.exists(self.baseFilename) and not os.path.isfile(self.baseFilename):
|
||||
+ return False
|
||||
if self.stream is None: # delay was set...
|
||||
self.stream = self._open()
|
||||
if self.maxBytes > 0: # are we rolling over?
|
||||
msg = "%s\n" % self.format(record)
|
||||
self.stream.seek(0, 2) #due to non-posix-compliant Windows feature
|
||||
if self.stream.tell() + len(msg) >= self.maxBytes:
|
||||
- return 1
|
||||
- return 0
|
||||
+ return True
|
||||
+ return False
|
||||
|
||||
class TimedRotatingFileHandler(BaseRotatingHandler):
|
||||
"""
|
||||
@@ -335,10 +338,13 @@ class TimedRotatingFileHandler(BaseRotatingHandler):
|
||||
record is not used, as we are just comparing times, but it is needed so
|
||||
the method signatures are the same
|
||||
"""
|
||||
+ # See bpo-45401: Never rollover anything other than regular files
|
||||
+ if os.path.exists(self.baseFilename) and not os.path.isfile(self.baseFilename):
|
||||
+ return False
|
||||
t = int(time.time())
|
||||
if t >= self.rolloverAt:
|
||||
- return 1
|
||||
- return 0
|
||||
+ return True
|
||||
+ return False
|
||||
|
||||
def getFilesToDelete(self):
|
||||
"""
|
||||
diff --git a/Lib/test/test_logging.py b/Lib/test/test_logging.py
|
||||
index 45b72e3..055b8e3 100644
|
||||
--- a/Lib/test/test_logging.py
|
||||
+++ b/Lib/test/test_logging.py
|
||||
@@ -4219,6 +4219,13 @@ class RotatingFileHandlerTest(BaseFileTest):
|
||||
rh = logging.handlers.RotatingFileHandler(self.fn, maxBytes=0)
|
||||
self.assertFalse(rh.shouldRollover(None))
|
||||
rh.close()
|
||||
+ # bpo-45401 - test with special file
|
||||
+ # We set maxBytes to 1 so that rollover would normally happen, except
|
||||
+ # for the check for regular files
|
||||
+ rh = logging.handlers.RotatingFileHandler(
|
||||
+ os.devnull, encoding="utf-8", maxBytes=1)
|
||||
+ self.assertFalse(rh.shouldRollover(self.next_rec()))
|
||||
+ rh.close()
|
||||
|
||||
def test_should_rollover(self):
|
||||
rh = logging.handlers.RotatingFileHandler(self.fn, maxBytes=1)
|
||||
@@ -4294,6 +4301,15 @@ class RotatingFileHandlerTest(BaseFileTest):
|
||||
rh.close()
|
||||
|
||||
class TimedRotatingFileHandlerTest(BaseFileTest):
|
||||
+ def test_should_not_rollover(self):
|
||||
+ # See bpo-45401. Should only ever rollover regular files
|
||||
+ fh = logging.handlers.TimedRotatingFileHandler(
|
||||
+ os.devnull, 'S', encoding="utf-8", backupCount=1)
|
||||
+ time.sleep(1.1) # a little over a second ...
|
||||
+ r = logging.makeLogRecord({'msg': 'testing - device file'})
|
||||
+ self.assertFalse(fh.shouldRollover(r))
|
||||
+ fh.close()
|
||||
+
|
||||
# other test methods added below
|
||||
def test_rollover(self):
|
||||
fh = logging.handlers.TimedRotatingFileHandler(self.fn, 'S',
|
|
@ -0,0 +1,267 @@
|
|||
diff --git a/Makefile.pre.in b/Makefile.pre.in
|
||||
index 8da1965..9864fe2 100644
|
||||
--- a/Makefile.pre.in
|
||||
+++ b/Makefile.pre.in
|
||||
@@ -884,7 +884,8 @@ regen-opcode-targets:
|
||||
$(srcdir)/Python/opcode_targets.h.new
|
||||
$(UPDATE_FILE) $(srcdir)/Python/opcode_targets.h $(srcdir)/Python/opcode_targets.h.new
|
||||
|
||||
-Python/ceval.o: $(srcdir)/Python/opcode_targets.h $(srcdir)/Python/ceval_gil.h
|
||||
+Python/ceval.o: $(srcdir)/Python/opcode_targets.h $(srcdir)/Python/ceval_gil.h \
|
||||
+ $(srcdir)/Python/condvar.h
|
||||
|
||||
Python/frozen.o: $(srcdir)/Python/importlib.h $(srcdir)/Python/importlib_external.h
|
||||
|
||||
@@ -1706,7 +1707,7 @@ patchcheck: @DEF_MAKE_RULE@
|
||||
|
||||
# Dependencies
|
||||
|
||||
-Python/thread.o: @THREADHEADERS@
|
||||
+Python/thread.o: @THREADHEADERS@ $(srcdir)/Python/condvar.h
|
||||
|
||||
# Declare targets that aren't real files
|
||||
.PHONY: all build_all sharedmods check-clean-src oldsharedmods test quicktest
|
||||
diff --git a/Python/ceval.c b/Python/ceval.c
|
||||
index 0b30cc1..3f1300c 100644
|
||||
--- a/Python/ceval.c
|
||||
+++ b/Python/ceval.c
|
||||
@@ -232,6 +232,7 @@ PyEval_InitThreads(void)
|
||||
{
|
||||
if (gil_created())
|
||||
return;
|
||||
+ PyThread_init_thread();
|
||||
create_gil();
|
||||
take_gil(PyThreadState_GET());
|
||||
main_thread = PyThread_get_thread_ident();
|
||||
diff --git a/Python/condvar.h b/Python/condvar.h
|
||||
index 9a71b17..39a420f 100644
|
||||
--- a/Python/condvar.h
|
||||
+++ b/Python/condvar.h
|
||||
@@ -59,20 +59,6 @@
|
||||
|
||||
#include <pthread.h>
|
||||
|
||||
-#define PyCOND_ADD_MICROSECONDS(tv, interval) \
|
||||
-do { /* TODO: add overflow and truncation checks */ \
|
||||
- tv.tv_usec += (long) interval; \
|
||||
- tv.tv_sec += tv.tv_usec / 1000000; \
|
||||
- tv.tv_usec %= 1000000; \
|
||||
-} while (0)
|
||||
-
|
||||
-/* We assume all modern POSIX systems have gettimeofday() */
|
||||
-#ifdef GETTIMEOFDAY_NO_TZ
|
||||
-#define PyCOND_GETTIMEOFDAY(ptv) gettimeofday(ptv)
|
||||
-#else
|
||||
-#define PyCOND_GETTIMEOFDAY(ptv) gettimeofday(ptv, (struct timezone *)NULL)
|
||||
-#endif
|
||||
-
|
||||
/* The following functions return 0 on success, nonzero on error */
|
||||
#define PyMUTEX_T pthread_mutex_t
|
||||
#define PyMUTEX_INIT(mut) pthread_mutex_init((mut), NULL)
|
||||
@@ -81,32 +67,30 @@ do { /* TODO: add overflow and truncation checks */ \
|
||||
#define PyMUTEX_UNLOCK(mut) pthread_mutex_unlock(mut)
|
||||
|
||||
#define PyCOND_T pthread_cond_t
|
||||
-#define PyCOND_INIT(cond) pthread_cond_init((cond), NULL)
|
||||
+#define PyCOND_INIT(cond) _PyThread_cond_init(cond)
|
||||
#define PyCOND_FINI(cond) pthread_cond_destroy(cond)
|
||||
#define PyCOND_SIGNAL(cond) pthread_cond_signal(cond)
|
||||
#define PyCOND_BROADCAST(cond) pthread_cond_broadcast(cond)
|
||||
#define PyCOND_WAIT(cond, mut) pthread_cond_wait((cond), (mut))
|
||||
|
||||
+/* These private functions are implemented in Python/thread_pthread.h */
|
||||
+int _PyThread_cond_init(PyCOND_T *cond);
|
||||
+void _PyThread_cond_after(long long us, struct timespec *abs);
|
||||
+
|
||||
/* return 0 for success, 1 on timeout, -1 on error */
|
||||
Py_LOCAL_INLINE(int)
|
||||
PyCOND_TIMEDWAIT(PyCOND_T *cond, PyMUTEX_T *mut, long long us)
|
||||
{
|
||||
- int r;
|
||||
- struct timespec ts;
|
||||
- struct timeval deadline;
|
||||
-
|
||||
- PyCOND_GETTIMEOFDAY(&deadline);
|
||||
- PyCOND_ADD_MICROSECONDS(deadline, us);
|
||||
- ts.tv_sec = deadline.tv_sec;
|
||||
- ts.tv_nsec = deadline.tv_usec * 1000;
|
||||
-
|
||||
- r = pthread_cond_timedwait((cond), (mut), &ts);
|
||||
- if (r == ETIMEDOUT)
|
||||
+ struct timespec abs;
|
||||
+ _PyThread_cond_after(us, &abs);
|
||||
+ int ret = pthread_cond_timedwait(cond, mut, &abs);
|
||||
+ if (ret == ETIMEDOUT) {
|
||||
return 1;
|
||||
- else if (r)
|
||||
+ }
|
||||
+ if (ret) {
|
||||
return -1;
|
||||
- else
|
||||
- return 0;
|
||||
+ }
|
||||
+ return 0;
|
||||
}
|
||||
|
||||
#elif defined(NT_THREADS)
|
||||
diff --git a/Python/thread.c b/Python/thread.c
|
||||
index 63eeb1e..c5d0e59 100644
|
||||
--- a/Python/thread.c
|
||||
+++ b/Python/thread.c
|
||||
@@ -6,6 +6,7 @@
|
||||
Stuff shared by all thread_*.h files is collected here. */
|
||||
|
||||
#include "Python.h"
|
||||
+#include "condvar.h"
|
||||
|
||||
#ifndef _POSIX_THREADS
|
||||
/* This means pthreads are not implemented in libc headers, hence the macro
|
||||
diff --git a/Python/thread_pthread.h b/Python/thread_pthread.h
|
||||
index baea71f..7dc295e 100644
|
||||
--- a/Python/thread_pthread.h
|
||||
+++ b/Python/thread_pthread.h
|
||||
@@ -66,16 +66,6 @@
|
||||
#endif
|
||||
#endif
|
||||
|
||||
-#if !defined(pthread_attr_default)
|
||||
-# define pthread_attr_default ((pthread_attr_t *)NULL)
|
||||
-#endif
|
||||
-#if !defined(pthread_mutexattr_default)
|
||||
-# define pthread_mutexattr_default ((pthread_mutexattr_t *)NULL)
|
||||
-#endif
|
||||
-#if !defined(pthread_condattr_default)
|
||||
-# define pthread_condattr_default ((pthread_condattr_t *)NULL)
|
||||
-#endif
|
||||
-
|
||||
|
||||
/* Whether or not to use semaphores directly rather than emulating them with
|
||||
* mutexes and condition variables:
|
||||
@@ -120,6 +110,56 @@ do { \
|
||||
} while(0)
|
||||
|
||||
|
||||
+/*
|
||||
+ * pthread_cond support
|
||||
+ */
|
||||
+
|
||||
+#if defined(HAVE_PTHREAD_CONDATTR_SETCLOCK) && defined(HAVE_CLOCK_GETTIME) && defined(CLOCK_MONOTONIC)
|
||||
+// monotonic is supported statically. It doesn't mean it works on runtime.
|
||||
+#define CONDATTR_MONOTONIC
|
||||
+#endif
|
||||
+
|
||||
+// NULL when pthread_condattr_setclock(CLOCK_MONOTONIC) is not supported.
|
||||
+static pthread_condattr_t *condattr_monotonic = NULL;
|
||||
+
|
||||
+static void
|
||||
+init_condattr()
|
||||
+{
|
||||
+#ifdef CONDATTR_MONOTONIC
|
||||
+ static pthread_condattr_t ca;
|
||||
+ pthread_condattr_init(&ca);
|
||||
+ if (pthread_condattr_setclock(&ca, CLOCK_MONOTONIC) == 0) {
|
||||
+ condattr_monotonic = &ca; // Use monotonic clock
|
||||
+ }
|
||||
+#endif
|
||||
+}
|
||||
+
|
||||
+int
|
||||
+_PyThread_cond_init(PyCOND_T *cond)
|
||||
+{
|
||||
+ return pthread_cond_init(cond, condattr_monotonic);
|
||||
+}
|
||||
+
|
||||
+void
|
||||
+_PyThread_cond_after(long long us, struct timespec *abs)
|
||||
+{
|
||||
+#ifdef CONDATTR_MONOTONIC
|
||||
+ if (condattr_monotonic) {
|
||||
+ clock_gettime(CLOCK_MONOTONIC, abs);
|
||||
+ abs->tv_sec += us / 1000000;
|
||||
+ abs->tv_nsec += (us % 1000000) * 1000;
|
||||
+ abs->tv_sec += abs->tv_nsec / 1000000000;
|
||||
+ abs->tv_nsec %= 1000000000;
|
||||
+ return;
|
||||
+ }
|
||||
+#endif
|
||||
+
|
||||
+ struct timespec ts;
|
||||
+ MICROSECONDS_TO_TIMESPEC(us, ts);
|
||||
+ *abs = ts;
|
||||
+}
|
||||
+
|
||||
+
|
||||
/* A pthread mutex isn't sufficient to model the Python lock type
|
||||
* because, according to Draft 5 of the docs (P1003.4a/D5), both of the
|
||||
* following are undefined:
|
||||
@@ -175,6 +215,7 @@ PyThread__init_thread(void)
|
||||
extern void pthread_init(void);
|
||||
pthread_init();
|
||||
#endif
|
||||
+ init_condattr();
|
||||
}
|
||||
|
||||
#endif /* !_HAVE_BSDI */
|
||||
@@ -449,8 +490,7 @@ PyThread_allocate_lock(void)
|
||||
memset((void *)lock, '\0', sizeof(pthread_lock));
|
||||
lock->locked = 0;
|
||||
|
||||
- status = pthread_mutex_init(&lock->mut,
|
||||
- pthread_mutexattr_default);
|
||||
+ status = pthread_mutex_init(&lock->mut, NULL);
|
||||
CHECK_STATUS_PTHREAD("pthread_mutex_init");
|
||||
/* Mark the pthread mutex underlying a Python mutex as
|
||||
pure happens-before. We can't simply mark the
|
||||
@@ -459,8 +499,7 @@ PyThread_allocate_lock(void)
|
||||
will cause errors. */
|
||||
_Py_ANNOTATE_PURE_HAPPENS_BEFORE_MUTEX(&lock->mut);
|
||||
|
||||
- status = pthread_cond_init(&lock->lock_released,
|
||||
- pthread_condattr_default);
|
||||
+ status = _PyThread_cond_init(&lock->lock_released);
|
||||
CHECK_STATUS_PTHREAD("pthread_cond_init");
|
||||
|
||||
if (error) {
|
||||
@@ -519,9 +558,10 @@ PyThread_acquire_lock_timed(PyThread_type_lock lock, PY_TIMEOUT_T microseconds,
|
||||
success = PY_LOCK_ACQUIRED;
|
||||
}
|
||||
else if (microseconds != 0) {
|
||||
- struct timespec ts;
|
||||
- if (microseconds > 0)
|
||||
- MICROSECONDS_TO_TIMESPEC(microseconds, ts);
|
||||
+ struct timespec abs;
|
||||
+ if (microseconds > 0) {
|
||||
+ _PyThread_cond_after(microseconds, &abs);
|
||||
+ }
|
||||
/* continue trying until we get the lock */
|
||||
|
||||
/* mut must be locked by me -- part of the condition
|
||||
@@ -530,10 +570,13 @@ PyThread_acquire_lock_timed(PyThread_type_lock lock, PY_TIMEOUT_T microseconds,
|
||||
if (microseconds > 0) {
|
||||
status = pthread_cond_timedwait(
|
||||
&thelock->lock_released,
|
||||
- &thelock->mut, &ts);
|
||||
+ &thelock->mut, &abs);
|
||||
+ if (status == 1) {
|
||||
+ break;
|
||||
+ }
|
||||
if (status == ETIMEDOUT)
|
||||
break;
|
||||
- CHECK_STATUS_PTHREAD("pthread_cond_timed_wait");
|
||||
+ CHECK_STATUS_PTHREAD("pthread_cond_timedwait");
|
||||
}
|
||||
else {
|
||||
status = pthread_cond_wait(
|
||||
diff --git a/configure.ac b/configure.ac
|
||||
index a0e3613..8a17559 100644
|
||||
--- a/configure.ac
|
||||
+++ b/configure.ac
|
||||
@@ -3582,7 +3582,7 @@ AC_CHECK_FUNCS(alarm accept4 setitimer getitimer bind_textdomain_codeset chown \
|
||||
memrchr mbrtowc mkdirat mkfifo \
|
||||
mkfifoat mknod mknodat mktime mremap nice openat pathconf pause pipe2 plock poll \
|
||||
posix_fallocate posix_fadvise pread \
|
||||
- pthread_init pthread_kill putenv pwrite readlink readlinkat readv realpath renameat \
|
||||
+ pthread_condattr_setclock pthread_init pthread_kill putenv pwrite readlink readlinkat readv realpath renameat \
|
||||
select sem_open sem_timedwait sem_getvalue sem_unlink sendfile setegid seteuid \
|
||||
setgid sethostname \
|
||||
setlocale setregid setreuid setresuid setresgid setsid setpgid setpgrp setpriority setuid setvbuf \
|
|
@ -0,0 +1,80 @@
|
|||
diff --git a/Lib/ftplib.py b/Lib/ftplib.py
|
||||
index 2ff251a..385e432 100644
|
||||
--- a/Lib/ftplib.py
|
||||
+++ b/Lib/ftplib.py
|
||||
@@ -104,6 +104,8 @@ class FTP:
|
||||
welcome = None
|
||||
passiveserver = 1
|
||||
encoding = "latin-1"
|
||||
+ # Disables https://bugs.python.org/issue43285 security if set to True.
|
||||
+ trust_server_pasv_ipv4_address = False
|
||||
|
||||
# Initialization method (called by class instantiation).
|
||||
# Initialize host to localhost, port to standard ftp port
|
||||
@@ -333,8 +335,13 @@ class FTP:
|
||||
return sock
|
||||
|
||||
def makepasv(self):
|
||||
+ """Internal: Does the PASV or EPSV handshake -> (address, port)"""
|
||||
if self.af == socket.AF_INET:
|
||||
- host, port = parse227(self.sendcmd('PASV'))
|
||||
+ untrusted_host, port = parse227(self.sendcmd('PASV'))
|
||||
+ if self.trust_server_pasv_ipv4_address:
|
||||
+ host = untrusted_host
|
||||
+ else:
|
||||
+ host = self.sock.getpeername()[0]
|
||||
else:
|
||||
host, port = parse229(self.sendcmd('EPSV'), self.sock.getpeername())
|
||||
return host, port
|
||||
diff --git a/Lib/test/test_ftplib.py b/Lib/test/test_ftplib.py
|
||||
index 4ff2f71..3ca7cc1 100644
|
||||
--- a/Lib/test/test_ftplib.py
|
||||
+++ b/Lib/test/test_ftplib.py
|
||||
@@ -94,6 +94,10 @@ class DummyFTPHandler(asynchat.async_chat):
|
||||
self.rest = None
|
||||
self.next_retr_data = RETR_DATA
|
||||
self.push('220 welcome')
|
||||
+ # We use this as the string IPv4 address to direct the client
|
||||
+ # to in response to a PASV command. To test security behavior.
|
||||
+ # https://bugs.python.org/issue43285/.
|
||||
+ self.fake_pasv_server_ip = '252.253.254.255'
|
||||
|
||||
def collect_incoming_data(self, data):
|
||||
self.in_buffer.append(data)
|
||||
@@ -136,7 +140,8 @@ class DummyFTPHandler(asynchat.async_chat):
|
||||
sock.bind((self.socket.getsockname()[0], 0))
|
||||
sock.listen()
|
||||
sock.settimeout(TIMEOUT)
|
||||
- ip, port = sock.getsockname()[:2]
|
||||
+ port = sock.getsockname()[1]
|
||||
+ ip = self.fake_pasv_server_ip
|
||||
ip = ip.replace('.', ','); p1 = port / 256; p2 = port % 256
|
||||
self.push('227 entering passive mode (%s,%d,%d)' %(ip, p1, p2))
|
||||
conn, addr = sock.accept()
|
||||
@@ -694,6 +699,26 @@ class TestFTPClass(TestCase):
|
||||
# IPv4 is in use, just make sure send_epsv has not been used
|
||||
self.assertEqual(self.server.handler_instance.last_received_cmd, 'pasv')
|
||||
|
||||
+ def test_makepasv_issue43285_security_disabled(self):
|
||||
+ """Test the opt-in to the old vulnerable behavior."""
|
||||
+ self.client.trust_server_pasv_ipv4_address = True
|
||||
+ bad_host, port = self.client.makepasv()
|
||||
+ self.assertEqual(
|
||||
+ bad_host, self.server.handler_instance.fake_pasv_server_ip)
|
||||
+ # Opening and closing a connection keeps the dummy server happy
|
||||
+ # instead of timing out on accept.
|
||||
+ socket.create_connection((self.client.sock.getpeername()[0], port),
|
||||
+ timeout=TIMEOUT).close()
|
||||
+
|
||||
+ def test_makepasv_issue43285_security_enabled_default(self):
|
||||
+ self.assertFalse(self.client.trust_server_pasv_ipv4_address)
|
||||
+ trusted_host, port = self.client.makepasv()
|
||||
+ self.assertNotEqual(
|
||||
+ trusted_host, self.server.handler_instance.fake_pasv_server_ip)
|
||||
+ # Opening and closing a connection keeps the dummy server happy
|
||||
+ # instead of timing out on accept.
|
||||
+ socket.create_connection((trusted_host, port), timeout=TIMEOUT).close()
|
||||
+
|
||||
def test_with_statement(self):
|
||||
self.client.quit()
|
||||
|
|
@ -0,0 +1,170 @@
|
|||
From 6c472d3a1d334d4eeb4a25eba7bf3b01611bf667 Mon Sep 17 00:00:00 2001
|
||||
From: "Miss Islington (bot)"
|
||||
<31488909+miss-islington@users.noreply.github.com>
|
||||
Date: Thu, 6 May 2021 09:56:01 -0700
|
||||
Subject: [PATCH] [3.6] bpo-43882 - urllib.parse should sanitize urls
|
||||
containing ASCII newline and tabs (GH-25924)
|
||||
|
||||
Co-authored-by: Gregory P. Smith <greg@krypto.org>
|
||||
Co-authored-by: Serhiy Storchaka <storchaka@gmail.com>
|
||||
(cherry picked from commit 76cd81d60310d65d01f9d7b48a8985d8ab89c8b4)
|
||||
Co-authored-by: Senthil Kumaran <senthil@uthcode.com>
|
||||
(cherry picked from commit 515a7bc4e13645d0945b46a8e1d9102b918cd407)
|
||||
|
||||
Co-authored-by: Miss Islington (bot) <31488909+miss-islington@users.noreply.github.com>
|
||||
---
|
||||
Doc/library/urllib.parse.rst | 13 +++++
|
||||
Lib/test/test_urlparse.py | 48 +++++++++++++++++++
|
||||
Lib/urllib/parse.py | 10 ++++
|
||||
.../2021-04-25-07-46-37.bpo-43882.Jpwx85.rst | 6 +++
|
||||
4 files changed, 77 insertions(+)
|
||||
create mode 100644 Misc/NEWS.d/next/Security/2021-04-25-07-46-37.bpo-43882.Jpwx85.rst
|
||||
|
||||
diff --git a/Doc/library/urllib.parse.rst b/Doc/library/urllib.parse.rst
|
||||
index 3c2e37ef2093a..b717d7cc05b2e 100644
|
||||
--- a/Doc/library/urllib.parse.rst
|
||||
+++ b/Doc/library/urllib.parse.rst
|
||||
@@ -288,6 +288,9 @@ or on combining URL components into a URL string.
|
||||
``#``, ``@``, or ``:`` will raise a :exc:`ValueError`. If the URL is
|
||||
decomposed before parsing, no error will be raised.
|
||||
|
||||
+ Following the `WHATWG spec`_ that updates RFC 3986, ASCII newline
|
||||
+ ``\n``, ``\r`` and tab ``\t`` characters are stripped from the URL.
|
||||
+
|
||||
.. versionchanged:: 3.6
|
||||
Out-of-range port numbers now raise :exc:`ValueError`, instead of
|
||||
returning :const:`None`.
|
||||
@@ -296,6 +299,10 @@ or on combining URL components into a URL string.
|
||||
Characters that affect netloc parsing under NFKC normalization will
|
||||
now raise :exc:`ValueError`.
|
||||
|
||||
+ .. versionchanged:: 3.6.14
|
||||
+ ASCII newline and tab characters are stripped from the URL.
|
||||
+
|
||||
+.. _WHATWG spec: https://url.spec.whatwg.org/#concept-basic-url-parser
|
||||
|
||||
.. function:: urlunsplit(parts)
|
||||
|
||||
@@ -633,6 +640,10 @@ task isn't already covered by the URL parsing functions above.
|
||||
|
||||
.. seealso::
|
||||
|
||||
+ `WHATWG`_ - URL Living standard
|
||||
+ Working Group for the URL Standard that defines URLs, domains, IP addresses, the
|
||||
+ application/x-www-form-urlencoded format, and their API.
|
||||
+
|
||||
:rfc:`3986` - Uniform Resource Identifiers
|
||||
This is the current standard (STD66). Any changes to urllib.parse module
|
||||
should conform to this. Certain deviations could be observed, which are
|
||||
@@ -656,3 +667,5 @@ task isn't already covered by the URL parsing functions above.
|
||||
|
||||
:rfc:`1738` - Uniform Resource Locators (URL)
|
||||
This specifies the formal syntax and semantics of absolute URLs.
|
||||
+
|
||||
+.. _WHATWG: https://url.spec.whatwg.org/
|
||||
diff --git a/Lib/test/test_urlparse.py b/Lib/test/test_urlparse.py
|
||||
index e3088b2f39bd7..3509278a01694 100644
|
||||
--- a/Lib/test/test_urlparse.py
|
||||
+++ b/Lib/test/test_urlparse.py
|
||||
@@ -612,6 +612,54 @@ def test_urlsplit_attributes(self):
|
||||
with self.assertRaisesRegex(ValueError, "out of range"):
|
||||
p.port
|
||||
|
||||
+ def test_urlsplit_remove_unsafe_bytes(self):
|
||||
+ # Remove ASCII tabs and newlines from input, for http common case scenario.
|
||||
+ url = "h\nttp://www.python\n.org\t/java\nscript:\talert('msg\r\n')/?query\n=\tsomething#frag\nment"
|
||||
+ p = urllib.parse.urlsplit(url)
|
||||
+ self.assertEqual(p.scheme, "http")
|
||||
+ self.assertEqual(p.netloc, "www.python.org")
|
||||
+ self.assertEqual(p.path, "/javascript:alert('msg')/")
|
||||
+ self.assertEqual(p.query, "query=something")
|
||||
+ self.assertEqual(p.fragment, "fragment")
|
||||
+ self.assertEqual(p.username, None)
|
||||
+ self.assertEqual(p.password, None)
|
||||
+ self.assertEqual(p.hostname, "www.python.org")
|
||||
+ self.assertEqual(p.port, None)
|
||||
+ self.assertEqual(p.geturl(), "http://www.python.org/javascript:alert('msg')/?query=something#fragment")
|
||||
+
|
||||
+ # Remove ASCII tabs and newlines from input as bytes, for http common case scenario.
|
||||
+ url = b"h\nttp://www.python\n.org\t/java\nscript:\talert('msg\r\n')/?query\n=\tsomething#frag\nment"
|
||||
+ p = urllib.parse.urlsplit(url)
|
||||
+ self.assertEqual(p.scheme, b"http")
|
||||
+ self.assertEqual(p.netloc, b"www.python.org")
|
||||
+ self.assertEqual(p.path, b"/javascript:alert('msg')/")
|
||||
+ self.assertEqual(p.query, b"query=something")
|
||||
+ self.assertEqual(p.fragment, b"fragment")
|
||||
+ self.assertEqual(p.username, None)
|
||||
+ self.assertEqual(p.password, None)
|
||||
+ self.assertEqual(p.hostname, b"www.python.org")
|
||||
+ self.assertEqual(p.port, None)
|
||||
+ self.assertEqual(p.geturl(), b"http://www.python.org/javascript:alert('msg')/?query=something#fragment")
|
||||
+
|
||||
+ # any scheme
|
||||
+ url = "x-new-scheme\t://www.python\n.org\t/java\nscript:\talert('msg\r\n')/?query\n=\tsomething#frag\nment"
|
||||
+ p = urllib.parse.urlsplit(url)
|
||||
+ self.assertEqual(p.geturl(), "x-new-scheme://www.python.org/javascript:alert('msg')/?query=something#fragment")
|
||||
+
|
||||
+ # Remove ASCII tabs and newlines from input as bytes, any scheme.
|
||||
+ url = b"x-new-scheme\t://www.python\n.org\t/java\nscript:\talert('msg\r\n')/?query\n=\tsomething#frag\nment"
|
||||
+ p = urllib.parse.urlsplit(url)
|
||||
+ self.assertEqual(p.geturl(), b"x-new-scheme://www.python.org/javascript:alert('msg')/?query=something#fragment")
|
||||
+
|
||||
+ # Unsafe bytes is not returned from urlparse cache.
|
||||
+ # scheme is stored after parsing, sending an scheme with unsafe bytes *will not* return an unsafe scheme
|
||||
+ url = "https://www.python\n.org\t/java\nscript:\talert('msg\r\n')/?query\n=\tsomething#frag\nment"
|
||||
+ scheme = "htt\nps"
|
||||
+ for _ in range(2):
|
||||
+ p = urllib.parse.urlsplit(url, scheme=scheme)
|
||||
+ self.assertEqual(p.scheme, "https")
|
||||
+ self.assertEqual(p.geturl(), "https://www.python.org/javascript:alert('msg')/?query=something#fragment")
|
||||
+
|
||||
def test_attributes_bad_port(self):
|
||||
"""Check handling of invalid ports."""
|
||||
for bytes in (False, True):
|
||||
diff --git a/Lib/urllib/parse.py b/Lib/urllib/parse.py
|
||||
index 66056bf589bf6..ac6e7a9cee0b9 100644
|
||||
--- a/Lib/urllib/parse.py
|
||||
+++ b/Lib/urllib/parse.py
|
||||
@@ -76,6 +76,9 @@
|
||||
'0123456789'
|
||||
'+-.')
|
||||
|
||||
+# Unsafe bytes to be removed per WHATWG spec
|
||||
+_UNSAFE_URL_BYTES_TO_REMOVE = ['\t', '\r', '\n']
|
||||
+
|
||||
# XXX: Consider replacing with functools.lru_cache
|
||||
MAX_CACHE_SIZE = 20
|
||||
_parse_cache = {}
|
||||
@@ -409,6 +412,11 @@ def _checknetloc(netloc):
|
||||
raise ValueError("netloc '" + netloc + "' contains invalid " +
|
||||
"characters under NFKC normalization")
|
||||
|
||||
+def _remove_unsafe_bytes_from_url(url):
|
||||
+ for b in _UNSAFE_URL_BYTES_TO_REMOVE:
|
||||
+ url = url.replace(b, "")
|
||||
+ return url
|
||||
+
|
||||
def urlsplit(url, scheme='', allow_fragments=True):
|
||||
"""Parse a URL into 5 components:
|
||||
<scheme>://<netloc>/<path>?<query>#<fragment>
|
||||
@@ -416,6 +424,8 @@ def urlsplit(url, scheme='', allow_fragments=True):
|
||||
Note that we don't break the components up in smaller bits
|
||||
(e.g. netloc is a single string) and we don't expand % escapes."""
|
||||
url, scheme, _coerce_result = _coerce_args(url, scheme)
|
||||
+ url = _remove_unsafe_bytes_from_url(url)
|
||||
+ scheme = _remove_unsafe_bytes_from_url(scheme)
|
||||
allow_fragments = bool(allow_fragments)
|
||||
key = url, scheme, allow_fragments, type(url), type(scheme)
|
||||
cached = _parse_cache.get(key, None)
|
||||
diff --git a/Misc/NEWS.d/next/Security/2021-04-25-07-46-37.bpo-43882.Jpwx85.rst b/Misc/NEWS.d/next/Security/2021-04-25-07-46-37.bpo-43882.Jpwx85.rst
|
||||
new file mode 100644
|
||||
index 0000000000000..a326d079dff4a
|
||||
--- /dev/null
|
||||
+++ b/Misc/NEWS.d/next/Security/2021-04-25-07-46-37.bpo-43882.Jpwx85.rst
|
||||
@@ -0,0 +1,6 @@
|
||||
+The presence of newline or tab characters in parts of a URL could allow
|
||||
+some forms of attacks.
|
||||
+
|
||||
+Following the controlling specification for URLs defined by WHATWG
|
||||
+:func:`urllib.parse` now removes ASCII newlines and tabs from URLs,
|
||||
+preventing such attacks.
|
|
@ -0,0 +1,98 @@
|
|||
From a5b78c6f1c802f6023bd4d7a248dc83be1eef6a3 Mon Sep 17 00:00:00 2001
|
||||
From: Sebastian Pipping <sebastian@pipping.org>
|
||||
Date: Mon, 21 Feb 2022 15:48:32 +0100
|
||||
Subject: [PATCH] 00378: Support expat 2.4.5
|
||||
|
||||
Curly brackets were never allowed in namespace URIs
|
||||
according to RFC 3986, and so-called namespace-validating
|
||||
XML parsers have the right to reject them a invalid URIs.
|
||||
|
||||
libexpat >=2.4.5 has become strcter in that regard due to
|
||||
related security issues; with ET.XML instantiating a
|
||||
namespace-aware parser under the hood, this test has no
|
||||
future in CPython.
|
||||
|
||||
References:
|
||||
- https://datatracker.ietf.org/doc/html/rfc3968
|
||||
- https://www.w3.org/TR/xml-names/
|
||||
|
||||
Also, test_minidom.py: Support Expat >=2.4.5
|
||||
|
||||
Upstream: https://bugs.python.org/issue46811
|
||||
|
||||
Co-authored-by: Sebastian Pipping <sebastian@pipping.org>
|
||||
---
|
||||
Lib/test/test_minidom.py | 12 +++++++++---
|
||||
Lib/test/test_xml_etree.py | 6 ------
|
||||
.../Library/2022-02-20-21-03-31.bpo-46811.8BxgdQ.rst | 1 +
|
||||
3 files changed, 10 insertions(+), 9 deletions(-)
|
||||
create mode 100644 Misc/NEWS.d/next/Library/2022-02-20-21-03-31.bpo-46811.8BxgdQ.rst
|
||||
|
||||
diff --git a/Lib/test/test_minidom.py b/Lib/test/test_minidom.py
|
||||
index d55e25e..e947382 100644
|
||||
--- a/Lib/test/test_minidom.py
|
||||
+++ b/Lib/test/test_minidom.py
|
||||
@@ -5,10 +5,12 @@ import pickle
|
||||
from test import support
|
||||
import unittest
|
||||
|
||||
+import pyexpat
|
||||
import xml.dom.minidom
|
||||
|
||||
from xml.dom.minidom import parse, Node, Document, parseString
|
||||
from xml.dom.minidom import getDOMImplementation
|
||||
+from xml.parsers.expat import ExpatError
|
||||
|
||||
|
||||
tstfile = support.findfile("test.xml", subdir="xmltestdata")
|
||||
@@ -1156,8 +1158,10 @@ class MinidomTest(unittest.TestCase):
|
||||
|
||||
# Verify that character decoding errors raise exceptions instead
|
||||
# of crashing
|
||||
- self.assertRaises(UnicodeDecodeError, parseString,
|
||||
- b'<fran\xe7ais>Comment \xe7a va ? Tr\xe8s bien ?</fran\xe7ais>')
|
||||
+ self.assertRaises(ExpatError, parseString,
|
||||
+ b'<fran\xe7ais></fran\xe7ais>')
|
||||
+ self.assertRaises(ExpatError, parseString,
|
||||
+ b'<franais>Comment \xe7a va ? Tr\xe8s bien ?</franais>')
|
||||
|
||||
doc.unlink()
|
||||
|
||||
@@ -1602,7 +1606,9 @@ class MinidomTest(unittest.TestCase):
|
||||
self.confirm(doc2.namespaceURI == xml.dom.EMPTY_NAMESPACE)
|
||||
|
||||
def testExceptionOnSpacesInXMLNSValue(self):
|
||||
- with self.assertRaisesRegex(ValueError, 'Unsupported syntax'):
|
||||
+ context = self.assertRaisesRegex(ExpatError, 'syntax error')
|
||||
+
|
||||
+ with context:
|
||||
parseString('<element xmlns:abc="http:abc.com/de f g/hi/j k"><abc:foo /></element>')
|
||||
|
||||
def testDocRemoveChild(self):
|
||||
diff --git a/Lib/test/test_xml_etree.py b/Lib/test/test_xml_etree.py
|
||||
index b01709e..acaa519 100644
|
||||
--- a/Lib/test/test_xml_etree.py
|
||||
+++ b/Lib/test/test_xml_etree.py
|
||||
@@ -1668,12 +1668,6 @@ class BugsTest(unittest.TestCase):
|
||||
b"<?xml version='1.0' encoding='ascii'?>\n"
|
||||
b'<body>tãg</body>')
|
||||
|
||||
- def test_issue3151(self):
|
||||
- e = ET.XML('<prefix:localname xmlns:prefix="${stuff}"/>')
|
||||
- self.assertEqual(e.tag, '{${stuff}}localname')
|
||||
- t = ET.ElementTree(e)
|
||||
- self.assertEqual(ET.tostring(e), b'<ns0:localname xmlns:ns0="${stuff}" />')
|
||||
-
|
||||
def test_issue6565(self):
|
||||
elem = ET.XML("<body><tag/></body>")
|
||||
self.assertEqual(summarize_list(elem), ['tag'])
|
||||
diff --git a/Misc/NEWS.d/next/Library/2022-02-20-21-03-31.bpo-46811.8BxgdQ.rst b/Misc/NEWS.d/next/Library/2022-02-20-21-03-31.bpo-46811.8BxgdQ.rst
|
||||
new file mode 100644
|
||||
index 0000000..6969bd1
|
||||
--- /dev/null
|
||||
+++ b/Misc/NEWS.d/next/Library/2022-02-20-21-03-31.bpo-46811.8BxgdQ.rst
|
||||
@@ -0,0 +1 @@
|
||||
+Make test suite support Expat >=2.4.5
|
||||
--
|
||||
2.35.1
|
||||
|
|
@ -0,0 +1,150 @@
|
|||
From 0000000000000000000000000000000000000000 Mon Sep 17 00:00:00 2001
|
||||
From: Petr Viktorin <encukou@gmail.com>
|
||||
Date: Fri, 3 Jun 2022 11:43:35 +0200
|
||||
Subject: [PATCH] 00382: CVE-2015-20107
|
||||
|
||||
Make mailcap refuse to match unsafe filenames/types/params (GH-91993)
|
||||
|
||||
Upstream: https://github.com/python/cpython/issues/68966
|
||||
|
||||
Tracker bug: https://bugzilla.redhat.com/show_bug.cgi?id=2075390
|
||||
---
|
||||
Doc/library/mailcap.rst | 12 +++++++++
|
||||
Lib/mailcap.py | 26 +++++++++++++++++--
|
||||
Lib/test/test_mailcap.py | 8 ++++--
|
||||
...2-04-27-18-25-30.gh-issue-68966.gjS8zs.rst | 4 +++
|
||||
4 files changed, 46 insertions(+), 4 deletions(-)
|
||||
create mode 100644 Misc/NEWS.d/next/Security/2022-04-27-18-25-30.gh-issue-68966.gjS8zs.rst
|
||||
|
||||
diff --git a/Doc/library/mailcap.rst b/Doc/library/mailcap.rst
|
||||
index 896afd1d73..849d0bc05f 100644
|
||||
--- a/Doc/library/mailcap.rst
|
||||
+++ b/Doc/library/mailcap.rst
|
||||
@@ -54,6 +54,18 @@ standard. However, mailcap files are supported on most Unix systems.
|
||||
use) to determine whether or not the mailcap line applies. :func:`findmatch`
|
||||
will automatically check such conditions and skip the entry if the check fails.
|
||||
|
||||
+ .. versionchanged:: 3.11
|
||||
+
|
||||
+ To prevent security issues with shell metacharacters (symbols that have
|
||||
+ special effects in a shell command line), ``findmatch`` will refuse
|
||||
+ to inject ASCII characters other than alphanumerics and ``@+=:,./-_``
|
||||
+ into the returned command line.
|
||||
+
|
||||
+ If a disallowed character appears in *filename*, ``findmatch`` will always
|
||||
+ return ``(None, None)`` as if no entry was found.
|
||||
+ If such a character appears elsewhere (a value in *plist* or in *MIMEtype*),
|
||||
+ ``findmatch`` will ignore all mailcap entries which use that value.
|
||||
+ A :mod:`warning <warnings>` will be raised in either case.
|
||||
|
||||
.. function:: getcaps()
|
||||
|
||||
diff --git a/Lib/mailcap.py b/Lib/mailcap.py
|
||||
index bd0fc0981c..dcd4b449e8 100644
|
||||
--- a/Lib/mailcap.py
|
||||
+++ b/Lib/mailcap.py
|
||||
@@ -2,6 +2,7 @@
|
||||
|
||||
import os
|
||||
import warnings
|
||||
+import re
|
||||
|
||||
__all__ = ["getcaps","findmatch"]
|
||||
|
||||
@@ -13,6 +14,11 @@ def lineno_sort_key(entry):
|
||||
else:
|
||||
return 1, 0
|
||||
|
||||
+_find_unsafe = re.compile(r'[^\xa1-\U0010FFFF\w@+=:,./-]').search
|
||||
+
|
||||
+class UnsafeMailcapInput(Warning):
|
||||
+ """Warning raised when refusing unsafe input"""
|
||||
+
|
||||
|
||||
# Part 1: top-level interface.
|
||||
|
||||
@@ -165,15 +171,22 @@ def findmatch(caps, MIMEtype, key='view', filename="/dev/null", plist=[]):
|
||||
entry to use.
|
||||
|
||||
"""
|
||||
+ if _find_unsafe(filename):
|
||||
+ msg = "Refusing to use mailcap with filename %r. Use a safe temporary filename." % (filename,)
|
||||
+ warnings.warn(msg, UnsafeMailcapInput)
|
||||
+ return None, None
|
||||
entries = lookup(caps, MIMEtype, key)
|
||||
# XXX This code should somehow check for the needsterminal flag.
|
||||
for e in entries:
|
||||
if 'test' in e:
|
||||
test = subst(e['test'], filename, plist)
|
||||
+ if test is None:
|
||||
+ continue
|
||||
if test and os.system(test) != 0:
|
||||
continue
|
||||
command = subst(e[key], MIMEtype, filename, plist)
|
||||
- return command, e
|
||||
+ if command is not None:
|
||||
+ return command, e
|
||||
return None, None
|
||||
|
||||
def lookup(caps, MIMEtype, key=None):
|
||||
@@ -206,6 +219,10 @@ def subst(field, MIMEtype, filename, plist=[]):
|
||||
elif c == 's':
|
||||
res = res + filename
|
||||
elif c == 't':
|
||||
+ if _find_unsafe(MIMEtype):
|
||||
+ msg = "Refusing to substitute MIME type %r into a shell command." % (MIMEtype,)
|
||||
+ warnings.warn(msg, UnsafeMailcapInput)
|
||||
+ return None
|
||||
res = res + MIMEtype
|
||||
elif c == '{':
|
||||
start = i
|
||||
@@ -213,7 +230,12 @@ def subst(field, MIMEtype, filename, plist=[]):
|
||||
i = i+1
|
||||
name = field[start:i]
|
||||
i = i+1
|
||||
- res = res + findparam(name, plist)
|
||||
+ param = findparam(name, plist)
|
||||
+ if _find_unsafe(param):
|
||||
+ msg = "Refusing to substitute parameter %r (%s) into a shell command" % (param, name)
|
||||
+ warnings.warn(msg, UnsafeMailcapInput)
|
||||
+ return None
|
||||
+ res = res + param
|
||||
# XXX To do:
|
||||
# %n == number of parts if type is multipart/*
|
||||
# %F == list of alternating type and filename for parts
|
||||
diff --git a/Lib/test/test_mailcap.py b/Lib/test/test_mailcap.py
|
||||
index c08423c670..920283d9a2 100644
|
||||
--- a/Lib/test/test_mailcap.py
|
||||
+++ b/Lib/test/test_mailcap.py
|
||||
@@ -121,7 +121,8 @@ class HelperFunctionTest(unittest.TestCase):
|
||||
(["", "audio/*", "foo.txt"], ""),
|
||||
(["echo foo", "audio/*", "foo.txt"], "echo foo"),
|
||||
(["echo %s", "audio/*", "foo.txt"], "echo foo.txt"),
|
||||
- (["echo %t", "audio/*", "foo.txt"], "echo audio/*"),
|
||||
+ (["echo %t", "audio/*", "foo.txt"], None),
|
||||
+ (["echo %t", "audio/wav", "foo.txt"], "echo audio/wav"),
|
||||
(["echo \\%t", "audio/*", "foo.txt"], "echo %t"),
|
||||
(["echo foo", "audio/*", "foo.txt", plist], "echo foo"),
|
||||
(["echo %{total}", "audio/*", "foo.txt", plist], "echo 3")
|
||||
@@ -205,7 +206,10 @@ class FindmatchTest(unittest.TestCase):
|
||||
('"An audio fragment"', audio_basic_entry)),
|
||||
([c, "audio/*"],
|
||||
{"filename": fname},
|
||||
- ("/usr/local/bin/showaudio audio/*", audio_entry)),
|
||||
+ (None, None)),
|
||||
+ ([c, "audio/wav"],
|
||||
+ {"filename": fname},
|
||||
+ ("/usr/local/bin/showaudio audio/wav", audio_entry)),
|
||||
([c, "message/external-body"],
|
||||
{"plist": plist},
|
||||
("showexternal /dev/null default john python.org /tmp foo bar", message_entry))
|
||||
diff --git a/Misc/NEWS.d/next/Security/2022-04-27-18-25-30.gh-issue-68966.gjS8zs.rst b/Misc/NEWS.d/next/Security/2022-04-27-18-25-30.gh-issue-68966.gjS8zs.rst
|
||||
new file mode 100644
|
||||
index 0000000000..da81a1f699
|
||||
--- /dev/null
|
||||
+++ b/Misc/NEWS.d/next/Security/2022-04-27-18-25-30.gh-issue-68966.gjS8zs.rst
|
||||
@@ -0,0 +1,4 @@
|
||||
+The deprecated mailcap module now refuses to inject unsafe text (filenames,
|
||||
+MIME types, parameters) into shell commands. Instead of using such text, it
|
||||
+will warn and act as if a match was not found (or for test commands, as if
|
||||
+the test failed).
|
|
@ -0,0 +1,130 @@
|
|||
From 0000000000000000000000000000000000000000 Mon Sep 17 00:00:00 2001
|
||||
From: "Miss Islington (bot)"
|
||||
<31488909+miss-islington@users.noreply.github.com>
|
||||
Date: Wed, 22 Jun 2022 15:05:00 -0700
|
||||
Subject: [PATCH] 00386: CVE-2021-28861
|
||||
|
||||
Fix an open redirection vulnerability in the `http.server` module when
|
||||
an URI path starts with `//` that could produce a 301 Location header
|
||||
with a misleading target. Vulnerability discovered, and logic fix
|
||||
proposed, by Hamza Avvan (@hamzaavvan).
|
||||
|
||||
Test and comments authored by Gregory P. Smith [Google].
|
||||
(cherry picked from commit 4abab6b603dd38bec1168e9a37c40a48ec89508e)
|
||||
|
||||
Upstream: https://github.com/python/cpython/pull/93879
|
||||
Tracking bugzilla: https://bugzilla.redhat.com/show_bug.cgi?id=2120642
|
||||
|
||||
Co-authored-by: Gregory P. Smith <greg@krypto.org>
|
||||
---
|
||||
Lib/http/server.py | 7 +++
|
||||
Lib/test/test_httpservers.py | 53 ++++++++++++++++++-
|
||||
...2-06-15-20-09-23.gh-issue-87389.QVaC3f.rst | 3 ++
|
||||
3 files changed, 61 insertions(+), 2 deletions(-)
|
||||
create mode 100644 Misc/NEWS.d/next/Security/2022-06-15-20-09-23.gh-issue-87389.QVaC3f.rst
|
||||
|
||||
diff --git a/Lib/http/server.py b/Lib/http/server.py
|
||||
index 60a4dadf03..ce05be13d3 100644
|
||||
--- a/Lib/http/server.py
|
||||
+++ b/Lib/http/server.py
|
||||
@@ -323,6 +323,13 @@ class BaseHTTPRequestHandler(socketserver.StreamRequestHandler):
|
||||
return False
|
||||
self.command, self.path, self.request_version = command, path, version
|
||||
|
||||
+ # gh-87389: The purpose of replacing '//' with '/' is to protect
|
||||
+ # against open redirect attacks possibly triggered if the path starts
|
||||
+ # with '//' because http clients treat //path as an absolute URI
|
||||
+ # without scheme (similar to http://path) rather than a path.
|
||||
+ if self.path.startswith('//'):
|
||||
+ self.path = '/' + self.path.lstrip('/') # Reduce to a single /
|
||||
+
|
||||
# Examine the headers and look for a Connection directive.
|
||||
try:
|
||||
self.headers = http.client.parse_headers(self.rfile,
|
||||
diff --git a/Lib/test/test_httpservers.py b/Lib/test/test_httpservers.py
|
||||
index 66e937e04b..5a0a7c3f74 100644
|
||||
--- a/Lib/test/test_httpservers.py
|
||||
+++ b/Lib/test/test_httpservers.py
|
||||
@@ -324,7 +324,7 @@ class SimpleHTTPServerTestCase(BaseTestCase):
|
||||
pass
|
||||
|
||||
def setUp(self):
|
||||
- BaseTestCase.setUp(self)
|
||||
+ super().setUp()
|
||||
self.cwd = os.getcwd()
|
||||
basetempdir = tempfile.gettempdir()
|
||||
os.chdir(basetempdir)
|
||||
@@ -343,7 +343,7 @@ class SimpleHTTPServerTestCase(BaseTestCase):
|
||||
except:
|
||||
pass
|
||||
finally:
|
||||
- BaseTestCase.tearDown(self)
|
||||
+ super().tearDown()
|
||||
|
||||
def check_status_and_reason(self, response, status, data=None):
|
||||
def close_conn():
|
||||
@@ -399,6 +399,55 @@ class SimpleHTTPServerTestCase(BaseTestCase):
|
||||
self.check_status_and_reason(response, HTTPStatus.OK,
|
||||
data=support.TESTFN_UNDECODABLE)
|
||||
|
||||
+ def test_get_dir_redirect_location_domain_injection_bug(self):
|
||||
+ """Ensure //evil.co/..%2f../../X does not put //evil.co/ in Location.
|
||||
+
|
||||
+ //netloc/ in a Location header is a redirect to a new host.
|
||||
+ https://github.com/python/cpython/issues/87389
|
||||
+
|
||||
+ This checks that a path resolving to a directory on our server cannot
|
||||
+ resolve into a redirect to another server.
|
||||
+ """
|
||||
+ os.mkdir(os.path.join(self.tempdir, 'existing_directory'))
|
||||
+ url = f'/python.org/..%2f..%2f..%2f..%2f..%2f../%0a%0d/../{self.tempdir_name}/existing_directory'
|
||||
+ expected_location = f'{url}/' # /python.org.../ single slash single prefix, trailing slash
|
||||
+ # Canonicalizes to /tmp/tempdir_name/existing_directory which does
|
||||
+ # exist and is a dir, triggering the 301 redirect logic.
|
||||
+ response = self.request(url)
|
||||
+ self.check_status_and_reason(response, HTTPStatus.MOVED_PERMANENTLY)
|
||||
+ location = response.getheader('Location')
|
||||
+ self.assertEqual(location, expected_location, msg='non-attack failed!')
|
||||
+
|
||||
+ # //python.org... multi-slash prefix, no trailing slash
|
||||
+ attack_url = f'/{url}'
|
||||
+ response = self.request(attack_url)
|
||||
+ self.check_status_and_reason(response, HTTPStatus.MOVED_PERMANENTLY)
|
||||
+ location = response.getheader('Location')
|
||||
+ self.assertFalse(location.startswith('//'), msg=location)
|
||||
+ self.assertEqual(location, expected_location,
|
||||
+ msg='Expected Location header to start with a single / and '
|
||||
+ 'end with a / as this is a directory redirect.')
|
||||
+
|
||||
+ # ///python.org... triple-slash prefix, no trailing slash
|
||||
+ attack3_url = f'//{url}'
|
||||
+ response = self.request(attack3_url)
|
||||
+ self.check_status_and_reason(response, HTTPStatus.MOVED_PERMANENTLY)
|
||||
+ self.assertEqual(response.getheader('Location'), expected_location)
|
||||
+
|
||||
+ # If the second word in the http request (Request-URI for the http
|
||||
+ # method) is a full URI, we don't worry about it, as that'll be parsed
|
||||
+ # and reassembled as a full URI within BaseHTTPRequestHandler.send_head
|
||||
+ # so no errant scheme-less //netloc//evil.co/ domain mixup can happen.
|
||||
+ attack_scheme_netloc_2slash_url = f'https://pypi.org/{url}'
|
||||
+ expected_scheme_netloc_location = f'{attack_scheme_netloc_2slash_url}/'
|
||||
+ response = self.request(attack_scheme_netloc_2slash_url)
|
||||
+ self.check_status_and_reason(response, HTTPStatus.MOVED_PERMANENTLY)
|
||||
+ location = response.getheader('Location')
|
||||
+ # We're just ensuring that the scheme and domain make it through, if
|
||||
+ # there are or aren't multiple slashes at the start of the path that
|
||||
+ # follows that isn't important in this Location: header.
|
||||
+ self.assertTrue(location.startswith('https://pypi.org/'), msg=location)
|
||||
+
|
||||
def test_get(self):
|
||||
#constructs the path relative to the root directory of the HTTPServer
|
||||
response = self.request(self.base_url + '/test')
|
||||
diff --git a/Misc/NEWS.d/next/Security/2022-06-15-20-09-23.gh-issue-87389.QVaC3f.rst b/Misc/NEWS.d/next/Security/2022-06-15-20-09-23.gh-issue-87389.QVaC3f.rst
|
||||
new file mode 100644
|
||||
index 0000000000..029d437190
|
||||
--- /dev/null
|
||||
+++ b/Misc/NEWS.d/next/Security/2022-06-15-20-09-23.gh-issue-87389.QVaC3f.rst
|
||||
@@ -0,0 +1,3 @@
|
||||
+:mod:`http.server`: Fix an open redirection vulnerability in the HTTP server
|
||||
+when an URI path starts with ``//``. Vulnerability discovered, and initial
|
||||
+fix proposed, by Hamza Avvan.
|
File diff suppressed because it is too large
Load Diff
|
@ -0,0 +1,95 @@
|
|||
From 0000000000000000000000000000000000000000 Mon Sep 17 00:00:00 2001
|
||||
From: "Miss Islington (bot)"
|
||||
<31488909+miss-islington@users.noreply.github.com>
|
||||
Date: Mon, 7 Nov 2022 19:22:14 -0800
|
||||
Subject: [PATCH] 00394: CVE-2022-45061: CPU denial of service via inefficient
|
||||
IDNA decoder
|
||||
|
||||
gh-98433: Fix quadratic time idna decoding.
|
||||
|
||||
There was an unnecessary quadratic loop in idna decoding. This restores
|
||||
the behavior to linear.
|
||||
|
||||
(cherry picked from commit a6f6c3a3d6f2b580f2d87885c9b8a9350ad7bf15)
|
||||
|
||||
Co-authored-by: Miss Islington (bot) <31488909+miss-islington@users.noreply.github.com>
|
||||
Co-authored-by: Gregory P. Smith <greg@krypto.org>
|
||||
---
|
||||
Lib/encodings/idna.py | 32 +++++++++----------
|
||||
Lib/test/test_codecs.py | 6 ++++
|
||||
...2-11-04-09-29-36.gh-issue-98433.l76c5G.rst | 6 ++++
|
||||
3 files changed, 27 insertions(+), 17 deletions(-)
|
||||
create mode 100644 Misc/NEWS.d/next/Security/2022-11-04-09-29-36.gh-issue-98433.l76c5G.rst
|
||||
|
||||
diff --git a/Lib/encodings/idna.py b/Lib/encodings/idna.py
|
||||
index ea4058512f..bf98f51336 100644
|
||||
--- a/Lib/encodings/idna.py
|
||||
+++ b/Lib/encodings/idna.py
|
||||
@@ -39,23 +39,21 @@ def nameprep(label):
|
||||
|
||||
# Check bidi
|
||||
RandAL = [stringprep.in_table_d1(x) for x in label]
|
||||
- for c in RandAL:
|
||||
- if c:
|
||||
- # There is a RandAL char in the string. Must perform further
|
||||
- # tests:
|
||||
- # 1) The characters in section 5.8 MUST be prohibited.
|
||||
- # This is table C.8, which was already checked
|
||||
- # 2) If a string contains any RandALCat character, the string
|
||||
- # MUST NOT contain any LCat character.
|
||||
- if any(stringprep.in_table_d2(x) for x in label):
|
||||
- raise UnicodeError("Violation of BIDI requirement 2")
|
||||
-
|
||||
- # 3) If a string contains any RandALCat character, a
|
||||
- # RandALCat character MUST be the first character of the
|
||||
- # string, and a RandALCat character MUST be the last
|
||||
- # character of the string.
|
||||
- if not RandAL[0] or not RandAL[-1]:
|
||||
- raise UnicodeError("Violation of BIDI requirement 3")
|
||||
+ if any(RandAL):
|
||||
+ # There is a RandAL char in the string. Must perform further
|
||||
+ # tests:
|
||||
+ # 1) The characters in section 5.8 MUST be prohibited.
|
||||
+ # This is table C.8, which was already checked
|
||||
+ # 2) If a string contains any RandALCat character, the string
|
||||
+ # MUST NOT contain any LCat character.
|
||||
+ if any(stringprep.in_table_d2(x) for x in label):
|
||||
+ raise UnicodeError("Violation of BIDI requirement 2")
|
||||
+ # 3) If a string contains any RandALCat character, a
|
||||
+ # RandALCat character MUST be the first character of the
|
||||
+ # string, and a RandALCat character MUST be the last
|
||||
+ # character of the string.
|
||||
+ if not RandAL[0] or not RandAL[-1]:
|
||||
+ raise UnicodeError("Violation of BIDI requirement 3")
|
||||
|
||||
return label
|
||||
|
||||
diff --git a/Lib/test/test_codecs.py b/Lib/test/test_codecs.py
|
||||
index 56485de3f6..a798d1f287 100644
|
||||
--- a/Lib/test/test_codecs.py
|
||||
+++ b/Lib/test/test_codecs.py
|
||||
@@ -1640,6 +1640,12 @@ class IDNACodecTest(unittest.TestCase):
|
||||
self.assertEqual("pyth\xf6n.org".encode("idna"), b"xn--pythn-mua.org")
|
||||
self.assertEqual("pyth\xf6n.org.".encode("idna"), b"xn--pythn-mua.org.")
|
||||
|
||||
+ def test_builtin_decode_length_limit(self):
|
||||
+ with self.assertRaisesRegex(UnicodeError, "too long"):
|
||||
+ (b"xn--016c"+b"a"*1100).decode("idna")
|
||||
+ with self.assertRaisesRegex(UnicodeError, "too long"):
|
||||
+ (b"xn--016c"+b"a"*70).decode("idna")
|
||||
+
|
||||
def test_stream(self):
|
||||
r = codecs.getreader("idna")(io.BytesIO(b"abc"))
|
||||
r.read(3)
|
||||
diff --git a/Misc/NEWS.d/next/Security/2022-11-04-09-29-36.gh-issue-98433.l76c5G.rst b/Misc/NEWS.d/next/Security/2022-11-04-09-29-36.gh-issue-98433.l76c5G.rst
|
||||
new file mode 100644
|
||||
index 0000000000..5185fac2e2
|
||||
--- /dev/null
|
||||
+++ b/Misc/NEWS.d/next/Security/2022-11-04-09-29-36.gh-issue-98433.l76c5G.rst
|
||||
@@ -0,0 +1,6 @@
|
||||
+The IDNA codec decoder used on DNS hostnames by :mod:`socket` or :mod:`asyncio`
|
||||
+related name resolution functions no longer involves a quadratic algorithm.
|
||||
+This prevents a potential CPU denial of service if an out-of-spec excessive
|
||||
+length hostname involving bidirectional characters were decoded. Some protocols
|
||||
+such as :mod:`urllib` http ``3xx`` redirects potentially allow for an attacker
|
||||
+to supply such a name.
|
File diff suppressed because it is too large
Load Diff
|
@ -0,0 +1,223 @@
|
|||
From 0000000000000000000000000000000000000000 Mon Sep 17 00:00:00 2001
|
||||
From: "Miss Islington (bot)"
|
||||
<31488909+miss-islington@users.noreply.github.com>
|
||||
Date: Mon, 22 May 2023 03:42:37 -0700
|
||||
Subject: [PATCH] 00399: CVE-2023-24329
|
||||
|
||||
gh-102153: Start stripping C0 control and space chars in `urlsplit` (GH-102508)
|
||||
|
||||
`urllib.parse.urlsplit` has already been respecting the WHATWG spec a bit GH-25595.
|
||||
|
||||
This adds more sanitizing to respect the "Remove any leading C0 control or space from input" [rule](https://url.spec.whatwg.org/GH-url-parsing:~:text=Remove%20any%20leading%20and%20trailing%20C0%20control%20or%20space%20from%20input.) in response to [CVE-2023-24329](https://nvd.nist.gov/vuln/detail/CVE-2023-24329).
|
||||
|
||||
Backported from Python 3.12
|
||||
|
||||
(cherry picked from commit f48a96a28012d28ae37a2f4587a780a5eb779946)
|
||||
|
||||
Co-authored-by: Illia Volochii <illia.volochii@gmail.com>
|
||||
Co-authored-by: Gregory P. Smith [Google] <greg@krypto.org>
|
||||
---
|
||||
Doc/library/urllib.parse.rst | 40 +++++++++++-
|
||||
Lib/test/test_urlparse.py | 61 ++++++++++++++++++-
|
||||
Lib/urllib/parse.py | 12 ++++
|
||||
...-03-07-20-59-17.gh-issue-102153.14CLSZ.rst | 3 +
|
||||
4 files changed, 113 insertions(+), 3 deletions(-)
|
||||
create mode 100644 Misc/NEWS.d/next/Security/2023-03-07-20-59-17.gh-issue-102153.14CLSZ.rst
|
||||
|
||||
diff --git a/Doc/library/urllib.parse.rst b/Doc/library/urllib.parse.rst
|
||||
index b717d7cc05..83a7a82089 100644
|
||||
--- a/Doc/library/urllib.parse.rst
|
||||
+++ b/Doc/library/urllib.parse.rst
|
||||
@@ -126,6 +126,12 @@ or on combining URL components into a URL string.
|
||||
``#``, ``@``, or ``:`` will raise a :exc:`ValueError`. If the URL is
|
||||
decomposed before parsing, no error will be raised.
|
||||
|
||||
+
|
||||
+ .. warning::
|
||||
+
|
||||
+ :func:`urlparse` does not perform validation. See :ref:`URL parsing
|
||||
+ security <url-parsing-security>` for details.
|
||||
+
|
||||
.. versionchanged:: 3.2
|
||||
Added IPv6 URL parsing capabilities.
|
||||
|
||||
@@ -288,8 +294,14 @@ or on combining URL components into a URL string.
|
||||
``#``, ``@``, or ``:`` will raise a :exc:`ValueError`. If the URL is
|
||||
decomposed before parsing, no error will be raised.
|
||||
|
||||
- Following the `WHATWG spec`_ that updates RFC 3986, ASCII newline
|
||||
- ``\n``, ``\r`` and tab ``\t`` characters are stripped from the URL.
|
||||
+ Following some of the `WHATWG spec`_ that updates RFC 3986, leading C0
|
||||
+ control and space characters are stripped from the URL. ``\n``,
|
||||
+ ``\r`` and tab ``\t`` characters are removed from the URL at any position.
|
||||
+
|
||||
+ .. warning::
|
||||
+
|
||||
+ :func:`urlsplit` does not perform validation. See :ref:`URL parsing
|
||||
+ security <url-parsing-security>` for details.
|
||||
|
||||
.. versionchanged:: 3.6
|
||||
Out-of-range port numbers now raise :exc:`ValueError`, instead of
|
||||
@@ -302,6 +314,9 @@ or on combining URL components into a URL string.
|
||||
.. versionchanged:: 3.6.14
|
||||
ASCII newline and tab characters are stripped from the URL.
|
||||
|
||||
+ .. versionchanged:: 3.6.15
|
||||
+ Leading WHATWG C0 control and space characters are stripped from the URL.
|
||||
+
|
||||
.. _WHATWG spec: https://url.spec.whatwg.org/#concept-basic-url-parser
|
||||
|
||||
.. function:: urlunsplit(parts)
|
||||
@@ -371,6 +386,27 @@ or on combining URL components into a URL string.
|
||||
.. versionchanged:: 3.2
|
||||
Result is a structured object rather than a simple 2-tuple.
|
||||
|
||||
+.. _url-parsing-security:
|
||||
+
|
||||
+URL parsing security
|
||||
+--------------------
|
||||
+
|
||||
+The :func:`urlsplit` and :func:`urlparse` APIs do not perform **validation** of
|
||||
+inputs. They may not raise errors on inputs that other applications consider
|
||||
+invalid. They may also succeed on some inputs that might not be considered
|
||||
+URLs elsewhere. Their purpose is for practical functionality rather than
|
||||
+purity.
|
||||
+
|
||||
+Instead of raising an exception on unusual input, they may instead return some
|
||||
+component parts as empty strings. Or components may contain more than perhaps
|
||||
+they should.
|
||||
+
|
||||
+We recommend that users of these APIs where the values may be used anywhere
|
||||
+with security implications code defensively. Do some verification within your
|
||||
+code before trusting a returned component part. Does that ``scheme`` make
|
||||
+sense? Is that a sensible ``path``? Is there anything strange about that
|
||||
+``hostname``? etc.
|
||||
+
|
||||
.. _parsing-ascii-encoded-bytes:
|
||||
|
||||
Parsing ASCII Encoded Bytes
|
||||
diff --git a/Lib/test/test_urlparse.py b/Lib/test/test_urlparse.py
|
||||
index 3509278a01..7fd61ffea9 100644
|
||||
--- a/Lib/test/test_urlparse.py
|
||||
+++ b/Lib/test/test_urlparse.py
|
||||
@@ -660,6 +660,65 @@ class UrlParseTestCase(unittest.TestCase):
|
||||
self.assertEqual(p.scheme, "https")
|
||||
self.assertEqual(p.geturl(), "https://www.python.org/javascript:alert('msg')/?query=something#fragment")
|
||||
|
||||
+ def test_urlsplit_strip_url(self):
|
||||
+ noise = bytes(range(0, 0x20 + 1))
|
||||
+ base_url = "http://User:Pass@www.python.org:080/doc/?query=yes#frag"
|
||||
+
|
||||
+ url = noise.decode("utf-8") + base_url
|
||||
+ p = urllib.parse.urlsplit(url)
|
||||
+ self.assertEqual(p.scheme, "http")
|
||||
+ self.assertEqual(p.netloc, "User:Pass@www.python.org:080")
|
||||
+ self.assertEqual(p.path, "/doc/")
|
||||
+ self.assertEqual(p.query, "query=yes")
|
||||
+ self.assertEqual(p.fragment, "frag")
|
||||
+ self.assertEqual(p.username, "User")
|
||||
+ self.assertEqual(p.password, "Pass")
|
||||
+ self.assertEqual(p.hostname, "www.python.org")
|
||||
+ self.assertEqual(p.port, 80)
|
||||
+ self.assertEqual(p.geturl(), base_url)
|
||||
+
|
||||
+ url = noise + base_url.encode("utf-8")
|
||||
+ p = urllib.parse.urlsplit(url)
|
||||
+ self.assertEqual(p.scheme, b"http")
|
||||
+ self.assertEqual(p.netloc, b"User:Pass@www.python.org:080")
|
||||
+ self.assertEqual(p.path, b"/doc/")
|
||||
+ self.assertEqual(p.query, b"query=yes")
|
||||
+ self.assertEqual(p.fragment, b"frag")
|
||||
+ self.assertEqual(p.username, b"User")
|
||||
+ self.assertEqual(p.password, b"Pass")
|
||||
+ self.assertEqual(p.hostname, b"www.python.org")
|
||||
+ self.assertEqual(p.port, 80)
|
||||
+ self.assertEqual(p.geturl(), base_url.encode("utf-8"))
|
||||
+
|
||||
+ # Test that trailing space is preserved as some applications rely on
|
||||
+ # this within query strings.
|
||||
+ query_spaces_url = "https://www.python.org:88/doc/?query= "
|
||||
+ p = urllib.parse.urlsplit(noise.decode("utf-8") + query_spaces_url)
|
||||
+ self.assertEqual(p.scheme, "https")
|
||||
+ self.assertEqual(p.netloc, "www.python.org:88")
|
||||
+ self.assertEqual(p.path, "/doc/")
|
||||
+ self.assertEqual(p.query, "query= ")
|
||||
+ self.assertEqual(p.port, 88)
|
||||
+ self.assertEqual(p.geturl(), query_spaces_url)
|
||||
+
|
||||
+ p = urllib.parse.urlsplit("www.pypi.org ")
|
||||
+ # That "hostname" gets considered a "path" due to the
|
||||
+ # trailing space and our existing logic... YUCK...
|
||||
+ # and re-assembles via geturl aka unurlsplit into the original.
|
||||
+ # django.core.validators.URLValidator (at least through v3.2) relies on
|
||||
+ # this, for better or worse, to catch it in a ValidationError via its
|
||||
+ # regular expressions.
|
||||
+ # Here we test the basic round trip concept of such a trailing space.
|
||||
+ self.assertEqual(urllib.parse.urlunsplit(p), "www.pypi.org ")
|
||||
+
|
||||
+ # with scheme as cache-key
|
||||
+ url = "//www.python.org/"
|
||||
+ scheme = noise.decode("utf-8") + "https" + noise.decode("utf-8")
|
||||
+ for _ in range(2):
|
||||
+ p = urllib.parse.urlsplit(url, scheme=scheme)
|
||||
+ self.assertEqual(p.scheme, "https")
|
||||
+ self.assertEqual(p.geturl(), "https://www.python.org/")
|
||||
+
|
||||
def test_attributes_bad_port(self):
|
||||
"""Check handling of invalid ports."""
|
||||
for bytes in (False, True):
|
||||
@@ -667,7 +726,7 @@ class UrlParseTestCase(unittest.TestCase):
|
||||
for port in ("foo", "1.5", "-1", "0x10"):
|
||||
with self.subTest(bytes=bytes, parse=parse, port=port):
|
||||
netloc = "www.example.net:" + port
|
||||
- url = "http://" + netloc
|
||||
+ url = "http://" + netloc + "/"
|
||||
if bytes:
|
||||
netloc = netloc.encode("ascii")
|
||||
url = url.encode("ascii")
|
||||
diff --git a/Lib/urllib/parse.py b/Lib/urllib/parse.py
|
||||
index ac6e7a9cee..717e990997 100644
|
||||
--- a/Lib/urllib/parse.py
|
||||
+++ b/Lib/urllib/parse.py
|
||||
@@ -25,6 +25,10 @@ currently not entirely compliant with this RFC due to defacto
|
||||
scenarios for parsing, and for backward compatibility purposes, some
|
||||
parsing quirks from older RFCs are retained. The testcases in
|
||||
test_urlparse.py provides a good indicator of parsing behavior.
|
||||
+
|
||||
+The WHATWG URL Parser spec should also be considered. We are not compliant with
|
||||
+it either due to existing user code API behavior expectations (Hyrum's Law).
|
||||
+It serves as a useful guide when making changes.
|
||||
"""
|
||||
|
||||
import re
|
||||
@@ -76,6 +80,10 @@ scheme_chars = ('abcdefghijklmnopqrstuvwxyz'
|
||||
'0123456789'
|
||||
'+-.')
|
||||
|
||||
+# Leading and trailing C0 control and space to be stripped per WHATWG spec.
|
||||
+# == "".join([chr(i) for i in range(0, 0x20 + 1)])
|
||||
+_WHATWG_C0_CONTROL_OR_SPACE = '\x00\x01\x02\x03\x04\x05\x06\x07\x08\t\n\x0b\x0c\r\x0e\x0f\x10\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f '
|
||||
+
|
||||
# Unsafe bytes to be removed per WHATWG spec
|
||||
_UNSAFE_URL_BYTES_TO_REMOVE = ['\t', '\r', '\n']
|
||||
|
||||
@@ -426,6 +434,10 @@ def urlsplit(url, scheme='', allow_fragments=True):
|
||||
url, scheme, _coerce_result = _coerce_args(url, scheme)
|
||||
url = _remove_unsafe_bytes_from_url(url)
|
||||
scheme = _remove_unsafe_bytes_from_url(scheme)
|
||||
+ # Only lstrip url as some applications rely on preserving trailing space.
|
||||
+ # (https://url.spec.whatwg.org/#concept-basic-url-parser would strip both)
|
||||
+ url = url.lstrip(_WHATWG_C0_CONTROL_OR_SPACE)
|
||||
+ scheme = scheme.strip(_WHATWG_C0_CONTROL_OR_SPACE)
|
||||
allow_fragments = bool(allow_fragments)
|
||||
key = url, scheme, allow_fragments, type(url), type(scheme)
|
||||
cached = _parse_cache.get(key, None)
|
||||
diff --git a/Misc/NEWS.d/next/Security/2023-03-07-20-59-17.gh-issue-102153.14CLSZ.rst b/Misc/NEWS.d/next/Security/2023-03-07-20-59-17.gh-issue-102153.14CLSZ.rst
|
||||
new file mode 100644
|
||||
index 0000000000..e57ac4ed3a
|
||||
--- /dev/null
|
||||
+++ b/Misc/NEWS.d/next/Security/2023-03-07-20-59-17.gh-issue-102153.14CLSZ.rst
|
||||
@@ -0,0 +1,3 @@
|
||||
+:func:`urllib.parse.urlsplit` now strips leading C0 control and space
|
||||
+characters following the specification for URLs defined by WHATWG in
|
||||
+response to CVE-2023-24329. Patch by Illia Volochii.
|
|
@ -0,0 +1,648 @@
|
|||
From 9f39318072b5775cf527f83daf8cb5d64678ac86 Mon Sep 17 00:00:00 2001
|
||||
From: =?UTF-8?q?=C5=81ukasz=20Langa?= <lukasz@langa.pl>
|
||||
Date: Tue, 22 Aug 2023 19:57:01 +0200
|
||||
Subject: [PATCH 1/4] gh-108310: Fix CVE-2023-40217: Check for & avoid the ssl
|
||||
pre-close flaw (#108321)
|
||||
MIME-Version: 1.0
|
||||
Content-Type: text/plain; charset=UTF-8
|
||||
Content-Transfer-Encoding: 8bit
|
||||
|
||||
gh-108310: Fix CVE-2023-40217: Check for & avoid the ssl pre-close flaw
|
||||
|
||||
Instances of `ssl.SSLSocket` were vulnerable to a bypass of the TLS handshake
|
||||
and included protections (like certificate verification) and treating sent
|
||||
unencrypted data as if it were post-handshake TLS encrypted data.
|
||||
|
||||
The vulnerability is caused when a socket is connected, data is sent by the
|
||||
malicious peer and stored in a buffer, and then the malicious peer closes the
|
||||
socket within a small timing window before the other peers’ TLS handshake can
|
||||
begin. After this sequence of events the closed socket will not immediately
|
||||
attempt a TLS handshake due to not being connected but will also allow the
|
||||
buffered data to be read as if a successful TLS handshake had occurred.
|
||||
|
||||
Co-authored-by: Gregory P. Smith [Google LLC] <greg@krypto.org>
|
||||
---
|
||||
Lib/ssl.py | 31 ++-
|
||||
Lib/test/test_ssl.py | 214 ++++++++++++++++++
|
||||
...-08-22-17-39-12.gh-issue-108310.fVM3sg.rst | 7 +
|
||||
3 files changed, 251 insertions(+), 1 deletion(-)
|
||||
create mode 100644 Misc/NEWS.d/next/Security/2023-08-22-17-39-12.gh-issue-108310.fVM3sg.rst
|
||||
|
||||
diff --git a/Lib/ssl.py b/Lib/ssl.py
|
||||
index c5c5529..288f237 100644
|
||||
--- a/Lib/ssl.py
|
||||
+++ b/Lib/ssl.py
|
||||
@@ -741,7 +741,7 @@ class SSLSocket(socket):
|
||||
type=sock.type,
|
||||
proto=sock.proto,
|
||||
fileno=sock.fileno())
|
||||
- self.settimeout(sock.gettimeout())
|
||||
+ sock_timeout = sock.gettimeout()
|
||||
sock.detach()
|
||||
elif fileno is not None:
|
||||
socket.__init__(self, fileno=fileno)
|
||||
@@ -755,9 +755,38 @@ class SSLSocket(socket):
|
||||
if e.errno != errno.ENOTCONN:
|
||||
raise
|
||||
connected = False
|
||||
+ blocking = self.getblocking()
|
||||
+ self.setblocking(False)
|
||||
+ try:
|
||||
+ # We are not connected so this is not supposed to block, but
|
||||
+ # testing revealed otherwise on macOS and Windows so we do
|
||||
+ # the non-blocking dance regardless. Our raise when any data
|
||||
+ # is found means consuming the data is harmless.
|
||||
+ notconn_pre_handshake_data = self.recv(1)
|
||||
+ except OSError as e:
|
||||
+ # EINVAL occurs for recv(1) on non-connected on unix sockets.
|
||||
+ if e.errno not in (errno.ENOTCONN, errno.EINVAL):
|
||||
+ raise
|
||||
+ notconn_pre_handshake_data = b''
|
||||
+ self.setblocking(blocking)
|
||||
+ if notconn_pre_handshake_data:
|
||||
+ # This prevents pending data sent to the socket before it was
|
||||
+ # closed from escaping to the caller who could otherwise
|
||||
+ # presume it came through a successful TLS connection.
|
||||
+ reason = "Closed before TLS handshake with data in recv buffer."
|
||||
+ notconn_pre_handshake_data_error = SSLError(e.errno, reason)
|
||||
+ # Add the SSLError attributes that _ssl.c always adds.
|
||||
+ notconn_pre_handshake_data_error.reason = reason
|
||||
+ notconn_pre_handshake_data_error.library = None
|
||||
+ try:
|
||||
+ self.close()
|
||||
+ except OSError:
|
||||
+ pass
|
||||
+ raise notconn_pre_handshake_data_error
|
||||
else:
|
||||
connected = True
|
||||
|
||||
+ self.settimeout(sock_timeout) # Must come after setblocking() calls.
|
||||
self._closed = False
|
||||
self._sslobj = None
|
||||
self._connected = connected
|
||||
diff --git a/Lib/test/test_ssl.py b/Lib/test/test_ssl.py
|
||||
index b35db25..e24d11b 100644
|
||||
--- a/Lib/test/test_ssl.py
|
||||
+++ b/Lib/test/test_ssl.py
|
||||
@@ -3,11 +3,14 @@
|
||||
import sys
|
||||
import unittest
|
||||
from test import support
|
||||
+import re
|
||||
import socket
|
||||
import select
|
||||
+import struct
|
||||
import time
|
||||
import datetime
|
||||
import gc
|
||||
+import http.client
|
||||
import os
|
||||
import errno
|
||||
import pprint
|
||||
@@ -3940,6 +3943,217 @@ class TestPostHandshakeAuth(unittest.TestCase):
|
||||
# server cert has not been validated
|
||||
self.assertEqual(s.getpeercert(), {})
|
||||
|
||||
+def set_socket_so_linger_on_with_zero_timeout(sock):
|
||||
+ sock.setsockopt(socket.SOL_SOCKET, socket.SO_LINGER, struct.pack('ii', 1, 0))
|
||||
+
|
||||
+
|
||||
+class TestPreHandshakeClose(unittest.TestCase):
|
||||
+ """Verify behavior of close sockets with received data before to the handshake.
|
||||
+ """
|
||||
+
|
||||
+ class SingleConnectionTestServerThread(threading.Thread):
|
||||
+
|
||||
+ def __init__(self, *, name, call_after_accept):
|
||||
+ self.call_after_accept = call_after_accept
|
||||
+ self.received_data = b'' # set by .run()
|
||||
+ self.wrap_error = None # set by .run()
|
||||
+ self.listener = None # set by .start()
|
||||
+ self.port = None # set by .start()
|
||||
+ super().__init__(name=name)
|
||||
+
|
||||
+ def __enter__(self):
|
||||
+ self.start()
|
||||
+ return self
|
||||
+
|
||||
+ def __exit__(self, *args):
|
||||
+ try:
|
||||
+ if self.listener:
|
||||
+ self.listener.close()
|
||||
+ except OSError:
|
||||
+ pass
|
||||
+ self.join()
|
||||
+ self.wrap_error = None # avoid dangling references
|
||||
+
|
||||
+ def start(self):
|
||||
+ self.ssl_ctx = ssl.create_default_context(ssl.Purpose.CLIENT_AUTH)
|
||||
+ self.ssl_ctx.verify_mode = ssl.CERT_REQUIRED
|
||||
+ self.ssl_ctx.load_verify_locations(cafile=ONLYCERT)
|
||||
+ self.ssl_ctx.load_cert_chain(certfile=ONLYCERT, keyfile=ONLYKEY)
|
||||
+ self.listener = socket.socket()
|
||||
+ self.port = support.bind_port(self.listener)
|
||||
+ self.listener.settimeout(2.0)
|
||||
+ self.listener.listen(1)
|
||||
+ super().start()
|
||||
+
|
||||
+ def run(self):
|
||||
+ conn, address = self.listener.accept()
|
||||
+ self.listener.close()
|
||||
+ with conn:
|
||||
+ if self.call_after_accept(conn):
|
||||
+ return
|
||||
+ try:
|
||||
+ tls_socket = self.ssl_ctx.wrap_socket(conn, server_side=True)
|
||||
+ except OSError as err: # ssl.SSLError inherits from OSError
|
||||
+ self.wrap_error = err
|
||||
+ else:
|
||||
+ try:
|
||||
+ self.received_data = tls_socket.recv(400)
|
||||
+ except OSError:
|
||||
+ pass # closed, protocol error, etc.
|
||||
+
|
||||
+ def non_linux_skip_if_other_okay_error(self, err):
|
||||
+ if sys.platform == "linux":
|
||||
+ return # Expect the full test setup to always work on Linux.
|
||||
+ if (isinstance(err, ConnectionResetError) or
|
||||
+ (isinstance(err, OSError) and err.errno == errno.EINVAL) or
|
||||
+ re.search('wrong.version.number', getattr(err, "reason", ""), re.I)):
|
||||
+ # On Windows the TCP RST leads to a ConnectionResetError
|
||||
+ # (ECONNRESET) which Linux doesn't appear to surface to userspace.
|
||||
+ # If wrap_socket() winds up on the "if connected:" path and doing
|
||||
+ # the actual wrapping... we get an SSLError from OpenSSL. Typically
|
||||
+ # WRONG_VERSION_NUMBER. While appropriate, neither is the scenario
|
||||
+ # we're specifically trying to test. The way this test is written
|
||||
+ # is known to work on Linux. We'll skip it anywhere else that it
|
||||
+ # does not present as doing so.
|
||||
+ self.skipTest("Could not recreate conditions on {}: \
|
||||
+ err={}".format(sys.platform,err))
|
||||
+ # If maintaining this conditional winds up being a problem.
|
||||
+ # just turn this into an unconditional skip anything but Linux.
|
||||
+ # The important thing is that our CI has the logic covered.
|
||||
+
|
||||
+ def test_preauth_data_to_tls_server(self):
|
||||
+ server_accept_called = threading.Event()
|
||||
+ ready_for_server_wrap_socket = threading.Event()
|
||||
+
|
||||
+ def call_after_accept(unused):
|
||||
+ server_accept_called.set()
|
||||
+ if not ready_for_server_wrap_socket.wait(2.0):
|
||||
+ raise RuntimeError("wrap_socket event never set, test may fail.")
|
||||
+ return False # Tell the server thread to continue.
|
||||
+
|
||||
+ server = self.SingleConnectionTestServerThread(
|
||||
+ call_after_accept=call_after_accept,
|
||||
+ name="preauth_data_to_tls_server")
|
||||
+ server.__enter__() # starts it
|
||||
+ self.addCleanup(server.__exit__) # ... & unittest.TestCase stops it.
|
||||
+
|
||||
+ with socket.socket() as client:
|
||||
+ client.connect(server.listener.getsockname())
|
||||
+ # This forces an immediate connection close via RST on .close().
|
||||
+ set_socket_so_linger_on_with_zero_timeout(client)
|
||||
+ client.setblocking(False)
|
||||
+
|
||||
+ server_accept_called.wait()
|
||||
+ client.send(b"DELETE /data HTTP/1.0\r\n\r\n")
|
||||
+ client.close() # RST
|
||||
+
|
||||
+ ready_for_server_wrap_socket.set()
|
||||
+ server.join()
|
||||
+ wrap_error = server.wrap_error
|
||||
+ self.assertEqual(b"", server.received_data)
|
||||
+ self.assertIsInstance(wrap_error, OSError) # All platforms.
|
||||
+ self.non_linux_skip_if_other_okay_error(wrap_error)
|
||||
+ self.assertIsInstance(wrap_error, ssl.SSLError)
|
||||
+ self.assertIn("before TLS handshake with data", wrap_error.args[1])
|
||||
+ self.assertIn("before TLS handshake with data", wrap_error.reason)
|
||||
+ self.assertNotEqual(0, wrap_error.args[0])
|
||||
+ self.assertIsNone(wrap_error.library, msg="attr must exist")
|
||||
+
|
||||
+ def test_preauth_data_to_tls_client(self):
|
||||
+ client_can_continue_with_wrap_socket = threading.Event()
|
||||
+
|
||||
+ def call_after_accept(conn_to_client):
|
||||
+ # This forces an immediate connection close via RST on .close().
|
||||
+ set_socket_so_linger_on_with_zero_timeout(conn_to_client)
|
||||
+ conn_to_client.send(
|
||||
+ b"HTTP/1.0 307 Temporary Redirect\r\n"
|
||||
+ b"Location: https://example.com/someone-elses-server\r\n"
|
||||
+ b"\r\n")
|
||||
+ conn_to_client.close() # RST
|
||||
+ client_can_continue_with_wrap_socket.set()
|
||||
+ return True # Tell the server to stop.
|
||||
+
|
||||
+ server = self.SingleConnectionTestServerThread(
|
||||
+ call_after_accept=call_after_accept,
|
||||
+ name="preauth_data_to_tls_client")
|
||||
+ server.__enter__() # starts it
|
||||
+ self.addCleanup(server.__exit__) # ... & unittest.TestCase stops it.
|
||||
+
|
||||
+ # Redundant; call_after_accept sets SO_LINGER on the accepted conn.
|
||||
+ set_socket_so_linger_on_with_zero_timeout(server.listener)
|
||||
+
|
||||
+ with socket.socket() as client:
|
||||
+ client.connect(server.listener.getsockname())
|
||||
+ if not client_can_continue_with_wrap_socket.wait(2.0):
|
||||
+ self.fail("test server took too long.")
|
||||
+ ssl_ctx = ssl.create_default_context()
|
||||
+ try:
|
||||
+ tls_client = ssl_ctx.wrap_socket(
|
||||
+ client, server_hostname="localhost")
|
||||
+ except OSError as err: # SSLError inherits from OSError
|
||||
+ wrap_error = err
|
||||
+ received_data = b""
|
||||
+ else:
|
||||
+ wrap_error = None
|
||||
+ received_data = tls_client.recv(400)
|
||||
+ tls_client.close()
|
||||
+
|
||||
+ server.join()
|
||||
+ self.assertEqual(b"", received_data)
|
||||
+ self.assertIsInstance(wrap_error, OSError) # All platforms.
|
||||
+ self.non_linux_skip_if_other_okay_error(wrap_error)
|
||||
+ self.assertIsInstance(wrap_error, ssl.SSLError)
|
||||
+ self.assertIn("before TLS handshake with data", wrap_error.args[1])
|
||||
+ self.assertIn("before TLS handshake with data", wrap_error.reason)
|
||||
+ self.assertNotEqual(0, wrap_error.args[0])
|
||||
+ self.assertIsNone(wrap_error.library, msg="attr must exist")
|
||||
+
|
||||
+ def test_https_client_non_tls_response_ignored(self):
|
||||
+
|
||||
+ server_responding = threading.Event()
|
||||
+
|
||||
+ class SynchronizedHTTPSConnection(http.client.HTTPSConnection):
|
||||
+ def connect(self):
|
||||
+ http.client.HTTPConnection.connect(self)
|
||||
+ # Wait for our fault injection server to have done its thing.
|
||||
+ if not server_responding.wait(1.0) and support.verbose:
|
||||
+ sys.stdout.write("server_responding event never set.")
|
||||
+ self.sock = self._context.wrap_socket(
|
||||
+ self.sock, server_hostname=self.host)
|
||||
+
|
||||
+ def call_after_accept(conn_to_client):
|
||||
+ # This forces an immediate connection close via RST on .close().
|
||||
+ set_socket_so_linger_on_with_zero_timeout(conn_to_client)
|
||||
+ conn_to_client.send(
|
||||
+ b"HTTP/1.0 402 Payment Required\r\n"
|
||||
+ b"\r\n")
|
||||
+ conn_to_client.close() # RST
|
||||
+ server_responding.set()
|
||||
+ return True # Tell the server to stop.
|
||||
+
|
||||
+ server = self.SingleConnectionTestServerThread(
|
||||
+ call_after_accept=call_after_accept,
|
||||
+ name="non_tls_http_RST_responder")
|
||||
+ server.__enter__() # starts it
|
||||
+ self.addCleanup(server.__exit__) # ... & unittest.TestCase stops it.
|
||||
+ # Redundant; call_after_accept sets SO_LINGER on the accepted conn.
|
||||
+ set_socket_so_linger_on_with_zero_timeout(server.listener)
|
||||
+
|
||||
+ connection = SynchronizedHTTPSConnection(
|
||||
+ f"localhost",
|
||||
+ port=server.port,
|
||||
+ context=ssl.create_default_context(),
|
||||
+ timeout=2.0,
|
||||
+ )
|
||||
+ # There are lots of reasons this raises as desired, long before this
|
||||
+ # test was added. Sending the request requires a successful TLS wrapped
|
||||
+ # socket; that fails if the connection is broken. It may seem pointless
|
||||
+ # to test this. It serves as an illustration of something that we never
|
||||
+ # want to happen... properly not happening.
|
||||
+ with self.assertRaises(OSError) as err_ctx:
|
||||
+ connection.request("HEAD", "/test", headers={"Host": "localhost"})
|
||||
+ response = connection.getresponse()
|
||||
+
|
||||
|
||||
def test_main(verbose=False):
|
||||
if support.verbose:
|
||||
diff --git a/Misc/NEWS.d/next/Security/2023-08-22-17-39-12.gh-issue-108310.fVM3sg.rst b/Misc/NEWS.d/next/Security/2023-08-22-17-39-12.gh-issue-108310.fVM3sg.rst
|
||||
new file mode 100644
|
||||
index 0000000..403c77a
|
||||
--- /dev/null
|
||||
+++ b/Misc/NEWS.d/next/Security/2023-08-22-17-39-12.gh-issue-108310.fVM3sg.rst
|
||||
@@ -0,0 +1,7 @@
|
||||
+Fixed an issue where instances of :class:`ssl.SSLSocket` were vulnerable to
|
||||
+a bypass of the TLS handshake and included protections (like certificate
|
||||
+verification) and treating sent unencrypted data as if it were
|
||||
+post-handshake TLS encrypted data. Security issue reported as
|
||||
+`CVE-2023-40217
|
||||
+<https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2023-40217>`_ by
|
||||
+Aapo Oksman. Patch by Gregory P. Smith.
|
||||
--
|
||||
2.41.0
|
||||
|
||||
|
||||
From 6fbb37c0b7ab8ce1db8e2e78df62d6e5c1e56766 Mon Sep 17 00:00:00 2001
|
||||
From: "Miss Islington (bot)"
|
||||
<31488909+miss-islington@users.noreply.github.com>
|
||||
Date: Wed, 23 Aug 2023 03:10:56 -0700
|
||||
Subject: [PATCH 2/4] gh-108342: Break ref cycle in SSLSocket._create() exc
|
||||
(GH-108344) (#108352)
|
||||
|
||||
Explicitly break a reference cycle when SSLSocket._create() raises an
|
||||
exception. Clear the variable storing the exception, since the
|
||||
exception traceback contains the variables and so creates a reference
|
||||
cycle.
|
||||
|
||||
This test leak was introduced by the test added for the fix of GH-108310.
|
||||
(cherry picked from commit 64f99350351bc46e016b2286f36ba7cd669b79e3)
|
||||
|
||||
Co-authored-by: Victor Stinner <vstinner@python.org>
|
||||
---
|
||||
Lib/ssl.py | 6 +++++-
|
||||
1 file changed, 5 insertions(+), 1 deletion(-)
|
||||
|
||||
diff --git a/Lib/ssl.py b/Lib/ssl.py
|
||||
index 288f237..67869c9 100644
|
||||
--- a/Lib/ssl.py
|
||||
+++ b/Lib/ssl.py
|
||||
@@ -782,7 +782,11 @@ class SSLSocket(socket):
|
||||
self.close()
|
||||
except OSError:
|
||||
pass
|
||||
- raise notconn_pre_handshake_data_error
|
||||
+ try:
|
||||
+ raise notconn_pre_handshake_data_error
|
||||
+ finally:
|
||||
+ # Explicitly break the reference cycle.
|
||||
+ notconn_pre_handshake_data_error = None
|
||||
else:
|
||||
connected = True
|
||||
|
||||
--
|
||||
2.41.0
|
||||
|
||||
|
||||
From 579d809e60e569f06c00844cc3a3f06a0e359603 Mon Sep 17 00:00:00 2001
|
||||
From: =?UTF-8?q?=C5=81ukasz=20Langa?= <lukasz@langa.pl>
|
||||
Date: Thu, 24 Aug 2023 12:09:30 +0200
|
||||
Subject: [PATCH 3/4] gh-108342: Make ssl TestPreHandshakeClose more reliable
|
||||
(GH-108370) (#108408)
|
||||
|
||||
* In preauth tests of test_ssl, explicitly break reference cycles
|
||||
invoving SingleConnectionTestServerThread to make sure that the
|
||||
thread is deleted. Otherwise, the test marks the environment as
|
||||
altered because the threading module sees a "dangling thread"
|
||||
(SingleConnectionTestServerThread). This test leak was introduced
|
||||
by the test added for the fix of issue gh-108310.
|
||||
* Use support.SHORT_TIMEOUT instead of hardcoded 1.0 or 2.0 seconds
|
||||
timeout.
|
||||
* SingleConnectionTestServerThread.run() catchs TimeoutError
|
||||
* Fix a race condition (missing synchronization) in
|
||||
test_preauth_data_to_tls_client(): the server now waits until the
|
||||
client connect() completed in call_after_accept().
|
||||
* test_https_client_non_tls_response_ignored() calls server.join()
|
||||
explicitly.
|
||||
* Replace "localhost" with server.listener.getsockname()[0].
|
||||
(cherry picked from commit 592bacb6fc0833336c0453e818e9b95016e9fd47)
|
||||
---
|
||||
Lib/test/test_ssl.py | 102 ++++++++++++++++++++++++++++++-------------
|
||||
1 file changed, 71 insertions(+), 31 deletions(-)
|
||||
|
||||
diff --git a/Lib/test/test_ssl.py b/Lib/test/test_ssl.py
|
||||
index e24d11b..6332330 100644
|
||||
--- a/Lib/test/test_ssl.py
|
||||
+++ b/Lib/test/test_ssl.py
|
||||
@@ -3953,12 +3953,16 @@ class TestPreHandshakeClose(unittest.TestCase):
|
||||
|
||||
class SingleConnectionTestServerThread(threading.Thread):
|
||||
|
||||
- def __init__(self, *, name, call_after_accept):
|
||||
+ def __init__(self, *, name, call_after_accept, timeout=None):
|
||||
self.call_after_accept = call_after_accept
|
||||
self.received_data = b'' # set by .run()
|
||||
self.wrap_error = None # set by .run()
|
||||
self.listener = None # set by .start()
|
||||
self.port = None # set by .start()
|
||||
+ if timeout is None:
|
||||
+ self.timeout = support.SHORT_TIMEOUT
|
||||
+ else:
|
||||
+ self.timeout = timeout
|
||||
super().__init__(name=name)
|
||||
|
||||
def __enter__(self):
|
||||
@@ -3981,13 +3985,19 @@ class TestPreHandshakeClose(unittest.TestCase):
|
||||
self.ssl_ctx.load_cert_chain(certfile=ONLYCERT, keyfile=ONLYKEY)
|
||||
self.listener = socket.socket()
|
||||
self.port = support.bind_port(self.listener)
|
||||
- self.listener.settimeout(2.0)
|
||||
+ self.listener.settimeout(self.timeout)
|
||||
self.listener.listen(1)
|
||||
super().start()
|
||||
|
||||
def run(self):
|
||||
- conn, address = self.listener.accept()
|
||||
- self.listener.close()
|
||||
+ try:
|
||||
+ conn, address = self.listener.accept()
|
||||
+ except TimeoutError:
|
||||
+ # on timeout, just close the listener
|
||||
+ return
|
||||
+ finally:
|
||||
+ self.listener.close()
|
||||
+
|
||||
with conn:
|
||||
if self.call_after_accept(conn):
|
||||
return
|
||||
@@ -4015,8 +4025,13 @@ class TestPreHandshakeClose(unittest.TestCase):
|
||||
# we're specifically trying to test. The way this test is written
|
||||
# is known to work on Linux. We'll skip it anywhere else that it
|
||||
# does not present as doing so.
|
||||
- self.skipTest("Could not recreate conditions on {}: \
|
||||
- err={}".format(sys.platform,err))
|
||||
+ try:
|
||||
+ self.skipTest("Could not recreate conditions on {}: \
|
||||
+ err={}".format(sys.platform,err))
|
||||
+ finally:
|
||||
+ # gh-108342: Explicitly break the reference cycle
|
||||
+ err = None
|
||||
+
|
||||
# If maintaining this conditional winds up being a problem.
|
||||
# just turn this into an unconditional skip anything but Linux.
|
||||
# The important thing is that our CI has the logic covered.
|
||||
@@ -4027,7 +4042,7 @@ class TestPreHandshakeClose(unittest.TestCase):
|
||||
|
||||
def call_after_accept(unused):
|
||||
server_accept_called.set()
|
||||
- if not ready_for_server_wrap_socket.wait(2.0):
|
||||
+ if not ready_for_server_wrap_socket.wait(support.SHORT_TIMEOUT):
|
||||
raise RuntimeError("wrap_socket event never set, test may fail.")
|
||||
return False # Tell the server thread to continue.
|
||||
|
||||
@@ -4049,20 +4064,31 @@ class TestPreHandshakeClose(unittest.TestCase):
|
||||
|
||||
ready_for_server_wrap_socket.set()
|
||||
server.join()
|
||||
+
|
||||
wrap_error = server.wrap_error
|
||||
- self.assertEqual(b"", server.received_data)
|
||||
- self.assertIsInstance(wrap_error, OSError) # All platforms.
|
||||
- self.non_linux_skip_if_other_okay_error(wrap_error)
|
||||
- self.assertIsInstance(wrap_error, ssl.SSLError)
|
||||
- self.assertIn("before TLS handshake with data", wrap_error.args[1])
|
||||
- self.assertIn("before TLS handshake with data", wrap_error.reason)
|
||||
- self.assertNotEqual(0, wrap_error.args[0])
|
||||
- self.assertIsNone(wrap_error.library, msg="attr must exist")
|
||||
+ server.wrap_error = None
|
||||
+ try:
|
||||
+ self.assertEqual(b"", server.received_data)
|
||||
+ self.assertIsInstance(wrap_error, OSError) # All platforms.
|
||||
+ self.non_linux_skip_if_other_okay_error(wrap_error)
|
||||
+ self.assertIsInstance(wrap_error, ssl.SSLError)
|
||||
+ self.assertIn("before TLS handshake with data", wrap_error.args[1])
|
||||
+ self.assertIn("before TLS handshake with data", wrap_error.reason)
|
||||
+ self.assertNotEqual(0, wrap_error.args[0])
|
||||
+ self.assertIsNone(wrap_error.library, msg="attr must exist")
|
||||
+ finally:
|
||||
+ # gh-108342: Explicitly break the reference cycle
|
||||
+ wrap_error = None
|
||||
+ server = None
|
||||
|
||||
def test_preauth_data_to_tls_client(self):
|
||||
+ server_can_continue_with_wrap_socket = threading.Event()
|
||||
client_can_continue_with_wrap_socket = threading.Event()
|
||||
|
||||
def call_after_accept(conn_to_client):
|
||||
+ if not server_can_continue_with_wrap_socket.wait(support.SHORT_TIMEOUT):
|
||||
+ print("ERROR: test client took too long")
|
||||
+
|
||||
# This forces an immediate connection close via RST on .close().
|
||||
set_socket_so_linger_on_with_zero_timeout(conn_to_client)
|
||||
conn_to_client.send(
|
||||
@@ -4084,8 +4110,10 @@ class TestPreHandshakeClose(unittest.TestCase):
|
||||
|
||||
with socket.socket() as client:
|
||||
client.connect(server.listener.getsockname())
|
||||
- if not client_can_continue_with_wrap_socket.wait(2.0):
|
||||
- self.fail("test server took too long.")
|
||||
+ server_can_continue_with_wrap_socket.set()
|
||||
+
|
||||
+ if not client_can_continue_with_wrap_socket.wait(support.SHORT_TIMEOUT):
|
||||
+ self.fail("test server took too long")
|
||||
ssl_ctx = ssl.create_default_context()
|
||||
try:
|
||||
tls_client = ssl_ctx.wrap_socket(
|
||||
@@ -4099,24 +4127,31 @@ class TestPreHandshakeClose(unittest.TestCase):
|
||||
tls_client.close()
|
||||
|
||||
server.join()
|
||||
- self.assertEqual(b"", received_data)
|
||||
- self.assertIsInstance(wrap_error, OSError) # All platforms.
|
||||
- self.non_linux_skip_if_other_okay_error(wrap_error)
|
||||
- self.assertIsInstance(wrap_error, ssl.SSLError)
|
||||
- self.assertIn("before TLS handshake with data", wrap_error.args[1])
|
||||
- self.assertIn("before TLS handshake with data", wrap_error.reason)
|
||||
- self.assertNotEqual(0, wrap_error.args[0])
|
||||
- self.assertIsNone(wrap_error.library, msg="attr must exist")
|
||||
+ try:
|
||||
+ self.assertEqual(b"", received_data)
|
||||
+ self.assertIsInstance(wrap_error, OSError) # All platforms.
|
||||
+ self.non_linux_skip_if_other_okay_error(wrap_error)
|
||||
+ self.assertIsInstance(wrap_error, ssl.SSLError)
|
||||
+ self.assertIn("before TLS handshake with data", wrap_error.args[1])
|
||||
+ self.assertIn("before TLS handshake with data", wrap_error.reason)
|
||||
+ self.assertNotEqual(0, wrap_error.args[0])
|
||||
+ self.assertIsNone(wrap_error.library, msg="attr must exist")
|
||||
+ finally:
|
||||
+ # gh-108342: Explicitly break the reference cycle
|
||||
+ wrap_error = None
|
||||
+ server = None
|
||||
|
||||
def test_https_client_non_tls_response_ignored(self):
|
||||
-
|
||||
server_responding = threading.Event()
|
||||
|
||||
class SynchronizedHTTPSConnection(http.client.HTTPSConnection):
|
||||
def connect(self):
|
||||
+ # Call clear text HTTP connect(), not the encrypted HTTPS (TLS)
|
||||
+ # connect(): wrap_socket() is called manually below.
|
||||
http.client.HTTPConnection.connect(self)
|
||||
+
|
||||
# Wait for our fault injection server to have done its thing.
|
||||
- if not server_responding.wait(1.0) and support.verbose:
|
||||
+ if not server_responding.wait(support.SHORT_TIMEOUT) and support.verbose:
|
||||
sys.stdout.write("server_responding event never set.")
|
||||
self.sock = self._context.wrap_socket(
|
||||
self.sock, server_hostname=self.host)
|
||||
@@ -4131,29 +4166,34 @@ class TestPreHandshakeClose(unittest.TestCase):
|
||||
server_responding.set()
|
||||
return True # Tell the server to stop.
|
||||
|
||||
+ timeout = 2.0
|
||||
server = self.SingleConnectionTestServerThread(
|
||||
call_after_accept=call_after_accept,
|
||||
- name="non_tls_http_RST_responder")
|
||||
+ name="non_tls_http_RST_responder",
|
||||
+ timeout=timeout)
|
||||
server.__enter__() # starts it
|
||||
self.addCleanup(server.__exit__) # ... & unittest.TestCase stops it.
|
||||
# Redundant; call_after_accept sets SO_LINGER on the accepted conn.
|
||||
set_socket_so_linger_on_with_zero_timeout(server.listener)
|
||||
|
||||
connection = SynchronizedHTTPSConnection(
|
||||
- f"localhost",
|
||||
+ server.listener.getsockname()[0],
|
||||
port=server.port,
|
||||
context=ssl.create_default_context(),
|
||||
- timeout=2.0,
|
||||
+ timeout=timeout,
|
||||
)
|
||||
+
|
||||
# There are lots of reasons this raises as desired, long before this
|
||||
# test was added. Sending the request requires a successful TLS wrapped
|
||||
# socket; that fails if the connection is broken. It may seem pointless
|
||||
# to test this. It serves as an illustration of something that we never
|
||||
# want to happen... properly not happening.
|
||||
- with self.assertRaises(OSError) as err_ctx:
|
||||
+ with self.assertRaises(OSError):
|
||||
connection.request("HEAD", "/test", headers={"Host": "localhost"})
|
||||
response = connection.getresponse()
|
||||
|
||||
+ server.join()
|
||||
+
|
||||
|
||||
def test_main(verbose=False):
|
||||
if support.verbose:
|
||||
--
|
||||
2.41.0
|
||||
|
||||
|
||||
From 2e8776245e9fb8ede784077df26684b4b53df0bc Mon Sep 17 00:00:00 2001
|
||||
From: Charalampos Stratakis <cstratak@redhat.com>
|
||||
Date: Mon, 25 Sep 2023 21:55:29 +0200
|
||||
Subject: [PATCH 4/4] Downstream: Additional fixup for 3.6:
|
||||
|
||||
Use alternative for self.getblocking(), which was added in Python 3.7
|
||||
see: https://docs.python.org/3/library/socket.html#socket.socket.getblocking
|
||||
|
||||
Set self._sslobj early to avoid AttributeError
|
||||
---
|
||||
Lib/ssl.py | 3 ++-
|
||||
1 file changed, 2 insertions(+), 1 deletion(-)
|
||||
|
||||
diff --git a/Lib/ssl.py b/Lib/ssl.py
|
||||
index 67869c9..daedc82 100644
|
||||
--- a/Lib/ssl.py
|
||||
+++ b/Lib/ssl.py
|
||||
@@ -690,6 +690,7 @@ class SSLSocket(socket):
|
||||
suppress_ragged_eofs=True, npn_protocols=None, ciphers=None,
|
||||
server_hostname=None,
|
||||
_context=None, _session=None):
|
||||
+ self._sslobj = None
|
||||
|
||||
if _context:
|
||||
self._context = _context
|
||||
@@ -755,7 +756,7 @@ class SSLSocket(socket):
|
||||
if e.errno != errno.ENOTCONN:
|
||||
raise
|
||||
connected = False
|
||||
- blocking = self.getblocking()
|
||||
+ blocking = (self.gettimeout() != 0)
|
||||
self.setblocking(False)
|
||||
try:
|
||||
# We are not connected so this is not supposed to block, but
|
||||
--
|
||||
2.41.0
|
||||
|
|
@ -0,0 +1,143 @@
|
|||
From c563f409ea30bcb0623d785428c9257917371b76 Mon Sep 17 00:00:00 2001
|
||||
From: "Miss Islington (bot)"
|
||||
<31488909+miss-islington@users.noreply.github.com>
|
||||
Date: Thu, 23 Jan 2020 06:49:19 -0800
|
||||
Subject: [PATCH] bpo-39421: Fix posible crash in heapq with custom comparison
|
||||
operators (GH-18118) (GH-18146)
|
||||
|
||||
(cherry picked from commit 79f89e6e5a659846d1068e8b1bd8e491ccdef861)
|
||||
|
||||
Co-authored-by: Pablo Galindo <Pablogsal@gmail.com>
|
||||
---
|
||||
Lib/test/test_heapq.py | 31 ++++++++++++++++
|
||||
.../2020-01-22-15-53-37.bpo-39421.O3nG7u.rst | 2 ++
|
||||
Modules/_heapqmodule.c | 35 ++++++++++++++-----
|
||||
3 files changed, 59 insertions(+), 9 deletions(-)
|
||||
create mode 100644 Misc/NEWS.d/next/Core and Builtins/2020-01-22-15-53-37.bpo-39421.O3nG7u.rst
|
||||
|
||||
diff --git a/Lib/test/test_heapq.py b/Lib/test/test_heapq.py
|
||||
index 2f8c648d84a58..7c3fb0210f69b 100644
|
||||
--- a/Lib/test/test_heapq.py
|
||||
+++ b/Lib/test/test_heapq.py
|
||||
@@ -414,6 +414,37 @@ def test_heappop_mutating_heap(self):
|
||||
with self.assertRaises((IndexError, RuntimeError)):
|
||||
self.module.heappop(heap)
|
||||
|
||||
+ def test_comparison_operator_modifiying_heap(self):
|
||||
+ # See bpo-39421: Strong references need to be taken
|
||||
+ # when comparing objects as they can alter the heap
|
||||
+ class EvilClass(int):
|
||||
+ def __lt__(self, o):
|
||||
+ heap.clear()
|
||||
+ return NotImplemented
|
||||
+
|
||||
+ heap = []
|
||||
+ self.module.heappush(heap, EvilClass(0))
|
||||
+ self.assertRaises(IndexError, self.module.heappushpop, heap, 1)
|
||||
+
|
||||
+ def test_comparison_operator_modifiying_heap_two_heaps(self):
|
||||
+
|
||||
+ class h(int):
|
||||
+ def __lt__(self, o):
|
||||
+ list2.clear()
|
||||
+ return NotImplemented
|
||||
+
|
||||
+ class g(int):
|
||||
+ def __lt__(self, o):
|
||||
+ list1.clear()
|
||||
+ return NotImplemented
|
||||
+
|
||||
+ list1, list2 = [], []
|
||||
+
|
||||
+ self.module.heappush(list1, h(0))
|
||||
+ self.module.heappush(list2, g(0))
|
||||
+
|
||||
+ self.assertRaises((IndexError, RuntimeError), self.module.heappush, list1, g(1))
|
||||
+ self.assertRaises((IndexError, RuntimeError), self.module.heappush, list2, h(1))
|
||||
|
||||
class TestErrorHandlingPython(TestErrorHandling, TestCase):
|
||||
module = py_heapq
|
||||
diff --git a/Misc/NEWS.d/next/Core and Builtins/2020-01-22-15-53-37.bpo-39421.O3nG7u.rst b/Misc/NEWS.d/next/Core and Builtins/2020-01-22-15-53-37.bpo-39421.O3nG7u.rst
|
||||
new file mode 100644
|
||||
index 0000000000000..bae008150ee12
|
||||
--- /dev/null
|
||||
+++ b/Misc/NEWS.d/next/Core and Builtins/2020-01-22-15-53-37.bpo-39421.O3nG7u.rst
|
||||
@@ -0,0 +1,2 @@
|
||||
+Fix possible crashes when operating with the functions in the :mod:`heapq`
|
||||
+module and custom comparison operators.
|
||||
diff --git a/Modules/_heapqmodule.c b/Modules/_heapqmodule.c
|
||||
index b499e1f668aae..0fb35ffe5ec48 100644
|
||||
--- a/Modules/_heapqmodule.c
|
||||
+++ b/Modules/_heapqmodule.c
|
||||
@@ -29,7 +29,11 @@ siftdown(PyListObject *heap, Py_ssize_t startpos, Py_ssize_t pos)
|
||||
while (pos > startpos) {
|
||||
parentpos = (pos - 1) >> 1;
|
||||
parent = arr[parentpos];
|
||||
+ Py_INCREF(newitem);
|
||||
+ Py_INCREF(parent);
|
||||
cmp = PyObject_RichCompareBool(newitem, parent, Py_LT);
|
||||
+ Py_DECREF(parent);
|
||||
+ Py_DECREF(newitem);
|
||||
if (cmp < 0)
|
||||
return -1;
|
||||
if (size != PyList_GET_SIZE(heap)) {
|
||||
@@ -71,10 +75,13 @@ siftup(PyListObject *heap, Py_ssize_t pos)
|
||||
/* Set childpos to index of smaller child. */
|
||||
childpos = 2*pos + 1; /* leftmost child position */
|
||||
if (childpos + 1 < endpos) {
|
||||
- cmp = PyObject_RichCompareBool(
|
||||
- arr[childpos],
|
||||
- arr[childpos + 1],
|
||||
- Py_LT);
|
||||
+ PyObject* a = arr[childpos];
|
||||
+ PyObject* b = arr[childpos + 1];
|
||||
+ Py_INCREF(a);
|
||||
+ Py_INCREF(b);
|
||||
+ cmp = PyObject_RichCompareBool(a, b, Py_LT);
|
||||
+ Py_DECREF(a);
|
||||
+ Py_DECREF(b);
|
||||
if (cmp < 0)
|
||||
return -1;
|
||||
childpos += ((unsigned)cmp ^ 1); /* increment when cmp==0 */
|
||||
@@ -229,7 +236,10 @@ heappushpop(PyObject *self, PyObject *args)
|
||||
return item;
|
||||
}
|
||||
|
||||
- cmp = PyObject_RichCompareBool(PyList_GET_ITEM(heap, 0), item, Py_LT);
|
||||
+ PyObject* top = PyList_GET_ITEM(heap, 0);
|
||||
+ Py_INCREF(top);
|
||||
+ cmp = PyObject_RichCompareBool(top, item, Py_LT);
|
||||
+ Py_DECREF(top);
|
||||
if (cmp < 0)
|
||||
return NULL;
|
||||
if (cmp == 0) {
|
||||
@@ -383,7 +393,11 @@ siftdown_max(PyListObject *heap, Py_ssize_t startpos, Py_ssize_t pos)
|
||||
while (pos > startpos) {
|
||||
parentpos = (pos - 1) >> 1;
|
||||
parent = arr[parentpos];
|
||||
+ Py_INCREF(parent);
|
||||
+ Py_INCREF(newitem);
|
||||
cmp = PyObject_RichCompareBool(parent, newitem, Py_LT);
|
||||
+ Py_DECREF(parent);
|
||||
+ Py_DECREF(newitem);
|
||||
if (cmp < 0)
|
||||
return -1;
|
||||
if (size != PyList_GET_SIZE(heap)) {
|
||||
@@ -425,10 +439,13 @@ siftup_max(PyListObject *heap, Py_ssize_t pos)
|
||||
/* Set childpos to index of smaller child. */
|
||||
childpos = 2*pos + 1; /* leftmost child position */
|
||||
if (childpos + 1 < endpos) {
|
||||
- cmp = PyObject_RichCompareBool(
|
||||
- arr[childpos + 1],
|
||||
- arr[childpos],
|
||||
- Py_LT);
|
||||
+ PyObject* a = arr[childpos + 1];
|
||||
+ PyObject* b = arr[childpos];
|
||||
+ Py_INCREF(a);
|
||||
+ Py_INCREF(b);
|
||||
+ cmp = PyObject_RichCompareBool(a, b, Py_LT);
|
||||
+ Py_DECREF(a);
|
||||
+ Py_DECREF(b);
|
||||
if (cmp < 0)
|
||||
return -1;
|
||||
childpos += ((unsigned)cmp ^ 1); /* increment when cmp==0 */
|
|
@ -0,0 +1,560 @@
|
|||
From cec66eefbee76ee95f252a52de0f5abc1b677f5b Mon Sep 17 00:00:00 2001
|
||||
From: Lumir Balhar <lbalhar@redhat.com>
|
||||
Date: Tue, 12 Dec 2023 13:03:42 +0100
|
||||
Subject: [PATCH] [3.6] bpo-42103: Improve validation of Plist files.
|
||||
(GH-22882) (GH-23118)
|
||||
|
||||
* Prevent some possible DoS attacks via providing invalid Plist files
|
||||
with extremely large number of objects or collection sizes.
|
||||
* Raise InvalidFileException for too large bytes and string size instead of returning garbage.
|
||||
* Raise InvalidFileException instead of ValueError for specific invalid datetime (NaN).
|
||||
* Raise InvalidFileException instead of TypeError for non-hashable dict keys.
|
||||
* Add more tests for invalid Plist files..
|
||||
(cherry picked from commit 34637a0ce21e7261b952fbd9d006474cc29b681f)
|
||||
|
||||
Co-authored-by: Serhiy Storchaka <storchaka@gmail.com>
|
||||
---
|
||||
Lib/plistlib.py | 34 +-
|
||||
Lib/test/test_plistlib.py | 394 +++++++++++++++---
|
||||
.../2020-10-23-19-20-14.bpo-42103.C5obK2.rst | 3 +
|
||||
.../2020-10-23-19-19-30.bpo-42103.cILT66.rst | 2 +
|
||||
4 files changed, 366 insertions(+), 67 deletions(-)
|
||||
create mode 100644 Misc/NEWS.d/next/Library/2020-10-23-19-20-14.bpo-42103.C5obK2.rst
|
||||
create mode 100644 Misc/NEWS.d/next/Security/2020-10-23-19-19-30.bpo-42103.cILT66.rst
|
||||
|
||||
diff --git a/Lib/plistlib.py b/Lib/plistlib.py
|
||||
index a918643..df1f346 100644
|
||||
--- a/Lib/plistlib.py
|
||||
+++ b/Lib/plistlib.py
|
||||
@@ -626,7 +626,7 @@ class _BinaryPlistParser:
|
||||
return self._read_object(top_object)
|
||||
|
||||
except (OSError, IndexError, struct.error, OverflowError,
|
||||
- UnicodeDecodeError):
|
||||
+ ValueError):
|
||||
raise InvalidFileException()
|
||||
|
||||
def _get_size(self, tokenL):
|
||||
@@ -642,7 +642,7 @@ class _BinaryPlistParser:
|
||||
def _read_ints(self, n, size):
|
||||
data = self._fp.read(size * n)
|
||||
if size in _BINARY_FORMAT:
|
||||
- return struct.unpack('>' + _BINARY_FORMAT[size] * n, data)
|
||||
+ return struct.unpack(f'>{n}{_BINARY_FORMAT[size]}', data)
|
||||
else:
|
||||
if not size or len(data) != size * n:
|
||||
raise InvalidFileException()
|
||||
@@ -701,19 +701,25 @@ class _BinaryPlistParser:
|
||||
|
||||
elif tokenH == 0x40: # data
|
||||
s = self._get_size(tokenL)
|
||||
- if self._use_builtin_types:
|
||||
- result = self._fp.read(s)
|
||||
- else:
|
||||
- result = Data(self._fp.read(s))
|
||||
+ result = self._fp.read(s)
|
||||
+ if len(result) != s:
|
||||
+ raise InvalidFileException()
|
||||
+ if not self._use_builtin_types:
|
||||
+ result = Data(result)
|
||||
|
||||
elif tokenH == 0x50: # ascii string
|
||||
s = self._get_size(tokenL)
|
||||
- result = self._fp.read(s).decode('ascii')
|
||||
- result = result
|
||||
+ data = self._fp.read(s)
|
||||
+ if len(data) != s:
|
||||
+ raise InvalidFileException()
|
||||
+ result = data.decode('ascii')
|
||||
|
||||
elif tokenH == 0x60: # unicode string
|
||||
- s = self._get_size(tokenL)
|
||||
- result = self._fp.read(s * 2).decode('utf-16be')
|
||||
+ s = self._get_size(tokenL) * 2
|
||||
+ data = self._fp.read(s)
|
||||
+ if len(data) != s:
|
||||
+ raise InvalidFileException()
|
||||
+ result = data.decode('utf-16be')
|
||||
|
||||
# tokenH == 0x80 is documented as 'UID' and appears to be used for
|
||||
# keyed-archiving, not in plists.
|
||||
@@ -737,9 +743,11 @@ class _BinaryPlistParser:
|
||||
obj_refs = self._read_refs(s)
|
||||
result = self._dict_type()
|
||||
self._objects[ref] = result
|
||||
- for k, o in zip(key_refs, obj_refs):
|
||||
- result[self._read_object(k)] = self._read_object(o)
|
||||
-
|
||||
+ try:
|
||||
+ for k, o in zip(key_refs, obj_refs):
|
||||
+ result[self._read_object(k)] = self._read_object(o)
|
||||
+ except TypeError:
|
||||
+ raise InvalidFileException()
|
||||
else:
|
||||
raise InvalidFileException()
|
||||
|
||||
diff --git a/Lib/test/test_plistlib.py b/Lib/test/test_plistlib.py
|
||||
index d47c607..f71245d 100644
|
||||
--- a/Lib/test/test_plistlib.py
|
||||
+++ b/Lib/test/test_plistlib.py
|
||||
@@ -1,5 +1,6 @@
|
||||
# Copyright (C) 2003-2013 Python Software Foundation
|
||||
|
||||
+import struct
|
||||
import unittest
|
||||
import plistlib
|
||||
import os
|
||||
@@ -90,6 +91,284 @@ TESTDATA={
|
||||
xQHHAsQC0gAAAAAAAAIBAAAAAAAAADkAAAAAAAAAAAAAAAAAAALs'''),
|
||||
}
|
||||
|
||||
+INVALID_BINARY_PLISTS = [
|
||||
+ ('too short data',
|
||||
+ b''
|
||||
+ ),
|
||||
+ ('too large offset_table_offset and offset_size = 1',
|
||||
+ b'\x00\x08'
|
||||
+ b'\x00\x00\x00\x00\x00\x00\x01\x01'
|
||||
+ b'\x00\x00\x00\x00\x00\x00\x00\x01'
|
||||
+ b'\x00\x00\x00\x00\x00\x00\x00\x00'
|
||||
+ b'\x00\x00\x00\x00\x00\x00\x00\x2a'
|
||||
+ ),
|
||||
+ ('too large offset_table_offset and nonstandard offset_size',
|
||||
+ b'\x00\x00\x00\x08'
|
||||
+ b'\x00\x00\x00\x00\x00\x00\x03\x01'
|
||||
+ b'\x00\x00\x00\x00\x00\x00\x00\x01'
|
||||
+ b'\x00\x00\x00\x00\x00\x00\x00\x00'
|
||||
+ b'\x00\x00\x00\x00\x00\x00\x00\x2c'
|
||||
+ ),
|
||||
+ ('integer overflow in offset_table_offset',
|
||||
+ b'\x00\x08'
|
||||
+ b'\x00\x00\x00\x00\x00\x00\x01\x01'
|
||||
+ b'\x00\x00\x00\x00\x00\x00\x00\x01'
|
||||
+ b'\x00\x00\x00\x00\x00\x00\x00\x00'
|
||||
+ b'\xff\xff\xff\xff\xff\xff\xff\xff'
|
||||
+ ),
|
||||
+ ('too large top_object',
|
||||
+ b'\x00\x08'
|
||||
+ b'\x00\x00\x00\x00\x00\x00\x01\x01'
|
||||
+ b'\x00\x00\x00\x00\x00\x00\x00\x01'
|
||||
+ b'\x00\x00\x00\x00\x00\x00\x00\x01'
|
||||
+ b'\x00\x00\x00\x00\x00\x00\x00\x09'
|
||||
+ ),
|
||||
+ ('integer overflow in top_object',
|
||||
+ b'\x00\x08'
|
||||
+ b'\x00\x00\x00\x00\x00\x00\x01\x01'
|
||||
+ b'\x00\x00\x00\x00\x00\x00\x00\x01'
|
||||
+ b'\xff\xff\xff\xff\xff\xff\xff\xff'
|
||||
+ b'\x00\x00\x00\x00\x00\x00\x00\x09'
|
||||
+ ),
|
||||
+ ('too large num_objects and offset_size = 1',
|
||||
+ b'\x00\x08'
|
||||
+ b'\x00\x00\x00\x00\x00\x00\x01\x01'
|
||||
+ b'\x00\x00\x00\x00\x00\x00\x00\xff'
|
||||
+ b'\x00\x00\x00\x00\x00\x00\x00\x00'
|
||||
+ b'\x00\x00\x00\x00\x00\x00\x00\x09'
|
||||
+ ),
|
||||
+ ('too large num_objects and nonstandard offset_size',
|
||||
+ b'\x00\x00\x00\x08'
|
||||
+ b'\x00\x00\x00\x00\x00\x00\x03\x01'
|
||||
+ b'\x00\x00\x00\x00\x00\x00\x00\xff'
|
||||
+ b'\x00\x00\x00\x00\x00\x00\x00\x00'
|
||||
+ b'\x00\x00\x00\x00\x00\x00\x00\x09'
|
||||
+ ),
|
||||
+ ('extremally large num_objects (32 bit)',
|
||||
+ b'\x00\x08'
|
||||
+ b'\x00\x00\x00\x00\x00\x00\x01\x01'
|
||||
+ b'\x00\x00\x00\x00\x7f\xff\xff\xff'
|
||||
+ b'\x00\x00\x00\x00\x00\x00\x00\x00'
|
||||
+ b'\x00\x00\x00\x00\x00\x00\x00\x09'
|
||||
+ ),
|
||||
+ ('extremally large num_objects (64 bit)',
|
||||
+ b'\x00\x08'
|
||||
+ b'\x00\x00\x00\x00\x00\x00\x01\x01'
|
||||
+ b'\x00\x00\x00\xff\xff\xff\xff\xff'
|
||||
+ b'\x00\x00\x00\x00\x00\x00\x00\x00'
|
||||
+ b'\x00\x00\x00\x00\x00\x00\x00\x09'
|
||||
+ ),
|
||||
+ ('integer overflow in num_objects',
|
||||
+ b'\x00\x08'
|
||||
+ b'\x00\x00\x00\x00\x00\x00\x01\x01'
|
||||
+ b'\xff\xff\xff\xff\xff\xff\xff\xff'
|
||||
+ b'\x00\x00\x00\x00\x00\x00\x00\x00'
|
||||
+ b'\x00\x00\x00\x00\x00\x00\x00\x09'
|
||||
+ ),
|
||||
+ ('offset_size = 0',
|
||||
+ b'\x00\x08'
|
||||
+ b'\x00\x00\x00\x00\x00\x00\x00\x01'
|
||||
+ b'\x00\x00\x00\x00\x00\x00\x00\x01'
|
||||
+ b'\x00\x00\x00\x00\x00\x00\x00\x00'
|
||||
+ b'\x00\x00\x00\x00\x00\x00\x00\x09'
|
||||
+ ),
|
||||
+ ('ref_size = 0',
|
||||
+ b'\xa1\x01\x00\x08\x0a'
|
||||
+ b'\x00\x00\x00\x00\x00\x00\x01\x00'
|
||||
+ b'\x00\x00\x00\x00\x00\x00\x00\x02'
|
||||
+ b'\x00\x00\x00\x00\x00\x00\x00\x00'
|
||||
+ b'\x00\x00\x00\x00\x00\x00\x00\x0b'
|
||||
+ ),
|
||||
+ ('too large offset',
|
||||
+ b'\x00\x2a'
|
||||
+ b'\x00\x00\x00\x00\x00\x00\x01\x01'
|
||||
+ b'\x00\x00\x00\x00\x00\x00\x00\x01'
|
||||
+ b'\x00\x00\x00\x00\x00\x00\x00\x00'
|
||||
+ b'\x00\x00\x00\x00\x00\x00\x00\x09'
|
||||
+ ),
|
||||
+ ('integer overflow in offset',
|
||||
+ b'\x00\xff\xff\xff\xff\xff\xff\xff\xff'
|
||||
+ b'\x00\x00\x00\x00\x00\x00\x08\x01'
|
||||
+ b'\x00\x00\x00\x00\x00\x00\x00\x01'
|
||||
+ b'\x00\x00\x00\x00\x00\x00\x00\x00'
|
||||
+ b'\x00\x00\x00\x00\x00\x00\x00\x09'
|
||||
+ ),
|
||||
+ ('too large array size',
|
||||
+ b'\xaf\x00\x01\xff\x00\x08\x0c'
|
||||
+ b'\x00\x00\x00\x00\x00\x00\x01\x01'
|
||||
+ b'\x00\x00\x00\x00\x00\x00\x00\x02'
|
||||
+ b'\x00\x00\x00\x00\x00\x00\x00\x00'
|
||||
+ b'\x00\x00\x00\x00\x00\x00\x00\x0d'
|
||||
+ ),
|
||||
+ ('extremally large array size (32-bit)',
|
||||
+ b'\xaf\x02\x7f\xff\xff\xff\x01\x00\x08\x0f'
|
||||
+ b'\x00\x00\x00\x00\x00\x00\x01\x01'
|
||||
+ b'\x00\x00\x00\x00\x00\x00\x00\x02'
|
||||
+ b'\x00\x00\x00\x00\x00\x00\x00\x00'
|
||||
+ b'\x00\x00\x00\x00\x00\x00\x00\x10'
|
||||
+ ),
|
||||
+ ('extremally large array size (64-bit)',
|
||||
+ b'\xaf\x03\x00\x00\x00\xff\xff\xff\xff\xff\x01\x00\x08\x13'
|
||||
+ b'\x00\x00\x00\x00\x00\x00\x01\x01'
|
||||
+ b'\x00\x00\x00\x00\x00\x00\x00\x02'
|
||||
+ b'\x00\x00\x00\x00\x00\x00\x00\x00'
|
||||
+ b'\x00\x00\x00\x00\x00\x00\x00\x14'
|
||||
+ ),
|
||||
+ ('integer overflow in array size',
|
||||
+ b'\xaf\x03\xff\xff\xff\xff\xff\xff\xff\xff\x01\x00\x08\x13'
|
||||
+ b'\x00\x00\x00\x00\x00\x00\x01\x01'
|
||||
+ b'\x00\x00\x00\x00\x00\x00\x00\x02'
|
||||
+ b'\x00\x00\x00\x00\x00\x00\x00\x00'
|
||||
+ b'\x00\x00\x00\x00\x00\x00\x00\x14'
|
||||
+ ),
|
||||
+ ('too large reference index',
|
||||
+ b'\xa1\x02\x00\x08\x0a'
|
||||
+ b'\x00\x00\x00\x00\x00\x00\x01\x01'
|
||||
+ b'\x00\x00\x00\x00\x00\x00\x00\x02'
|
||||
+ b'\x00\x00\x00\x00\x00\x00\x00\x00'
|
||||
+ b'\x00\x00\x00\x00\x00\x00\x00\x0b'
|
||||
+ ),
|
||||
+ ('integer overflow in reference index',
|
||||
+ b'\xa1\xff\xff\xff\xff\xff\xff\xff\xff\x00\x08\x11'
|
||||
+ b'\x00\x00\x00\x00\x00\x00\x01\x08'
|
||||
+ b'\x00\x00\x00\x00\x00\x00\x00\x02'
|
||||
+ b'\x00\x00\x00\x00\x00\x00\x00\x00'
|
||||
+ b'\x00\x00\x00\x00\x00\x00\x00\x12'
|
||||
+ ),
|
||||
+ ('too large bytes size',
|
||||
+ b'\x4f\x00\x23\x41\x08'
|
||||
+ b'\x00\x00\x00\x00\x00\x00\x01\x01'
|
||||
+ b'\x00\x00\x00\x00\x00\x00\x00\x01'
|
||||
+ b'\x00\x00\x00\x00\x00\x00\x00\x00'
|
||||
+ b'\x00\x00\x00\x00\x00\x00\x00\x0c'
|
||||
+ ),
|
||||
+ ('extremally large bytes size (32-bit)',
|
||||
+ b'\x4f\x02\x7f\xff\xff\xff\x41\x08'
|
||||
+ b'\x00\x00\x00\x00\x00\x00\x01\x01'
|
||||
+ b'\x00\x00\x00\x00\x00\x00\x00\x01'
|
||||
+ b'\x00\x00\x00\x00\x00\x00\x00\x00'
|
||||
+ b'\x00\x00\x00\x00\x00\x00\x00\x0f'
|
||||
+ ),
|
||||
+ ('extremally large bytes size (64-bit)',
|
||||
+ b'\x4f\x03\x00\x00\x00\xff\xff\xff\xff\xff\x41\x08'
|
||||
+ b'\x00\x00\x00\x00\x00\x00\x01\x01'
|
||||
+ b'\x00\x00\x00\x00\x00\x00\x00\x01'
|
||||
+ b'\x00\x00\x00\x00\x00\x00\x00\x00'
|
||||
+ b'\x00\x00\x00\x00\x00\x00\x00\x13'
|
||||
+ ),
|
||||
+ ('integer overflow in bytes size',
|
||||
+ b'\x4f\x03\xff\xff\xff\xff\xff\xff\xff\xff\x41\x08'
|
||||
+ b'\x00\x00\x00\x00\x00\x00\x01\x01'
|
||||
+ b'\x00\x00\x00\x00\x00\x00\x00\x01'
|
||||
+ b'\x00\x00\x00\x00\x00\x00\x00\x00'
|
||||
+ b'\x00\x00\x00\x00\x00\x00\x00\x13'
|
||||
+ ),
|
||||
+ ('too large ASCII size',
|
||||
+ b'\x5f\x00\x23\x41\x08'
|
||||
+ b'\x00\x00\x00\x00\x00\x00\x01\x01'
|
||||
+ b'\x00\x00\x00\x00\x00\x00\x00\x01'
|
||||
+ b'\x00\x00\x00\x00\x00\x00\x00\x00'
|
||||
+ b'\x00\x00\x00\x00\x00\x00\x00\x0c'
|
||||
+ ),
|
||||
+ ('extremally large ASCII size (32-bit)',
|
||||
+ b'\x5f\x02\x7f\xff\xff\xff\x41\x08'
|
||||
+ b'\x00\x00\x00\x00\x00\x00\x01\x01'
|
||||
+ b'\x00\x00\x00\x00\x00\x00\x00\x01'
|
||||
+ b'\x00\x00\x00\x00\x00\x00\x00\x00'
|
||||
+ b'\x00\x00\x00\x00\x00\x00\x00\x0f'
|
||||
+ ),
|
||||
+ ('extremally large ASCII size (64-bit)',
|
||||
+ b'\x5f\x03\x00\x00\x00\xff\xff\xff\xff\xff\x41\x08'
|
||||
+ b'\x00\x00\x00\x00\x00\x00\x01\x01'
|
||||
+ b'\x00\x00\x00\x00\x00\x00\x00\x01'
|
||||
+ b'\x00\x00\x00\x00\x00\x00\x00\x00'
|
||||
+ b'\x00\x00\x00\x00\x00\x00\x00\x13'
|
||||
+ ),
|
||||
+ ('integer overflow in ASCII size',
|
||||
+ b'\x5f\x03\xff\xff\xff\xff\xff\xff\xff\xff\x41\x08'
|
||||
+ b'\x00\x00\x00\x00\x00\x00\x01\x01'
|
||||
+ b'\x00\x00\x00\x00\x00\x00\x00\x01'
|
||||
+ b'\x00\x00\x00\x00\x00\x00\x00\x00'
|
||||
+ b'\x00\x00\x00\x00\x00\x00\x00\x13'
|
||||
+ ),
|
||||
+ ('invalid ASCII',
|
||||
+ b'\x51\xff\x08'
|
||||
+ b'\x00\x00\x00\x00\x00\x00\x01\x01'
|
||||
+ b'\x00\x00\x00\x00\x00\x00\x00\x01'
|
||||
+ b'\x00\x00\x00\x00\x00\x00\x00\x00'
|
||||
+ b'\x00\x00\x00\x00\x00\x00\x00\x0a'
|
||||
+ ),
|
||||
+ ('too large UTF-16 size',
|
||||
+ b'\x6f\x00\x13\x20\xac\x00\x08'
|
||||
+ b'\x00\x00\x00\x00\x00\x00\x01\x01'
|
||||
+ b'\x00\x00\x00\x00\x00\x00\x00\x01'
|
||||
+ b'\x00\x00\x00\x00\x00\x00\x00\x00'
|
||||
+ b'\x00\x00\x00\x00\x00\x00\x00\x0e'
|
||||
+ ),
|
||||
+ ('extremally large UTF-16 size (32-bit)',
|
||||
+ b'\x6f\x02\x4f\xff\xff\xff\x20\xac\x00\x08'
|
||||
+ b'\x00\x00\x00\x00\x00\x00\x01\x01'
|
||||
+ b'\x00\x00\x00\x00\x00\x00\x00\x01'
|
||||
+ b'\x00\x00\x00\x00\x00\x00\x00\x00'
|
||||
+ b'\x00\x00\x00\x00\x00\x00\x00\x11'
|
||||
+ ),
|
||||
+ ('extremally large UTF-16 size (64-bit)',
|
||||
+ b'\x6f\x03\x00\x00\x00\xff\xff\xff\xff\xff\x20\xac\x00\x08'
|
||||
+ b'\x00\x00\x00\x00\x00\x00\x01\x01'
|
||||
+ b'\x00\x00\x00\x00\x00\x00\x00\x01'
|
||||
+ b'\x00\x00\x00\x00\x00\x00\x00\x00'
|
||||
+ b'\x00\x00\x00\x00\x00\x00\x00\x15'
|
||||
+ ),
|
||||
+ ('integer overflow in UTF-16 size',
|
||||
+ b'\x6f\x03\xff\xff\xff\xff\xff\xff\xff\xff\x20\xac\x00\x08'
|
||||
+ b'\x00\x00\x00\x00\x00\x00\x01\x01'
|
||||
+ b'\x00\x00\x00\x00\x00\x00\x00\x01'
|
||||
+ b'\x00\x00\x00\x00\x00\x00\x00\x00'
|
||||
+ b'\x00\x00\x00\x00\x00\x00\x00\x15'
|
||||
+ ),
|
||||
+ ('invalid UTF-16',
|
||||
+ b'\x61\xd8\x00\x08'
|
||||
+ b'\x00\x00\x00\x00\x00\x00\x01\x01'
|
||||
+ b'\x00\x00\x00\x00\x00\x00\x00\x01'
|
||||
+ b'\x00\x00\x00\x00\x00\x00\x00\x00'
|
||||
+ b'\x00\x00\x00\x00\x00\x00\x00\x0b'
|
||||
+ ),
|
||||
+ ('non-hashable key',
|
||||
+ b'\xd1\x01\x01\xa0\x08\x0b'
|
||||
+ b'\x00\x00\x00\x00\x00\x00\x01\x01'
|
||||
+ b'\x00\x00\x00\x00\x00\x00\x00\x02'
|
||||
+ b'\x00\x00\x00\x00\x00\x00\x00\x00'
|
||||
+ b'\x00\x00\x00\x00\x00\x00\x00\x0c'
|
||||
+ ),
|
||||
+ ('too large datetime (datetime overflow)',
|
||||
+ b'\x33\x42\x50\x00\x00\x00\x00\x00\x00\x08'
|
||||
+ b'\x00\x00\x00\x00\x00\x00\x01\x01'
|
||||
+ b'\x00\x00\x00\x00\x00\x00\x00\x01'
|
||||
+ b'\x00\x00\x00\x00\x00\x00\x00\x00'
|
||||
+ b'\x00\x00\x00\x00\x00\x00\x00\x11'
|
||||
+ ),
|
||||
+ ('too large datetime (timedelta overflow)',
|
||||
+ b'\x33\x42\xe0\x00\x00\x00\x00\x00\x00\x08'
|
||||
+ b'\x00\x00\x00\x00\x00\x00\x01\x01'
|
||||
+ b'\x00\x00\x00\x00\x00\x00\x00\x01'
|
||||
+ b'\x00\x00\x00\x00\x00\x00\x00\x00'
|
||||
+ b'\x00\x00\x00\x00\x00\x00\x00\x11'
|
||||
+ ),
|
||||
+ ('invalid datetime (Infinity)',
|
||||
+ b'\x33\x7f\xf0\x00\x00\x00\x00\x00\x00\x08'
|
||||
+ b'\x00\x00\x00\x00\x00\x00\x01\x01'
|
||||
+ b'\x00\x00\x00\x00\x00\x00\x00\x01'
|
||||
+ b'\x00\x00\x00\x00\x00\x00\x00\x00'
|
||||
+ b'\x00\x00\x00\x00\x00\x00\x00\x11'
|
||||
+ ),
|
||||
+ ('invalid datetime (NaN)',
|
||||
+ b'\x33\x7f\xf8\x00\x00\x00\x00\x00\x00\x08'
|
||||
+ b'\x00\x00\x00\x00\x00\x00\x01\x01'
|
||||
+ b'\x00\x00\x00\x00\x00\x00\x00\x01'
|
||||
+ b'\x00\x00\x00\x00\x00\x00\x00\x00'
|
||||
+ b'\x00\x00\x00\x00\x00\x00\x00\x11'
|
||||
+ ),
|
||||
+]
|
||||
|
||||
class TestPlistlib(unittest.TestCase):
|
||||
|
||||
@@ -447,6 +726,21 @@ class TestPlistlib(unittest.TestCase):
|
||||
|
||||
class TestBinaryPlistlib(unittest.TestCase):
|
||||
|
||||
+ @staticmethod
|
||||
+ def decode(*objects, offset_size=1, ref_size=1):
|
||||
+ data = [b'bplist00']
|
||||
+ offset = 8
|
||||
+ offsets = []
|
||||
+ for x in objects:
|
||||
+ offsets.append(offset.to_bytes(offset_size, 'big'))
|
||||
+ data.append(x)
|
||||
+ offset += len(x)
|
||||
+ tail = struct.pack('>6xBBQQQ', offset_size, ref_size,
|
||||
+ len(objects), 0, offset)
|
||||
+ data.extend(offsets)
|
||||
+ data.append(tail)
|
||||
+ return plistlib.loads(b''.join(data), fmt=plistlib.FMT_BINARY)
|
||||
+
|
||||
def test_nonstandard_refs_size(self):
|
||||
# Issue #21538: Refs and offsets are 24-bit integers
|
||||
data = (b'bplist00'
|
||||
@@ -461,7 +755,7 @@ class TestBinaryPlistlib(unittest.TestCase):
|
||||
|
||||
def test_dump_duplicates(self):
|
||||
# Test effectiveness of saving duplicated objects
|
||||
- for x in (None, False, True, 12345, 123.45, 'abcde', b'abcde',
|
||||
+ for x in (None, False, True, 12345, 123.45, 'abcde', 'абвгд', b'abcde',
|
||||
datetime.datetime(2004, 10, 26, 10, 33, 33),
|
||||
plistlib.Data(b'abcde'), bytearray(b'abcde'),
|
||||
[12, 345], (12, 345), {'12': 345}):
|
||||
@@ -500,6 +794,20 @@ class TestBinaryPlistlib(unittest.TestCase):
|
||||
b = plistlib.loads(plistlib.dumps(a, fmt=plistlib.FMT_BINARY))
|
||||
self.assertIs(b['x'], b)
|
||||
|
||||
+ def test_deep_nesting(self):
|
||||
+ for N in [300, 100000]:
|
||||
+ chunks = [b'\xa1' + (i + 1).to_bytes(4, 'big') for i in range(N)]
|
||||
+ try:
|
||||
+ result = self.decode(*chunks, b'\x54seed', offset_size=4, ref_size=4)
|
||||
+ except RecursionError:
|
||||
+ pass
|
||||
+ else:
|
||||
+ for i in range(N):
|
||||
+ self.assertIsInstance(result, list)
|
||||
+ self.assertEqual(len(result), 1)
|
||||
+ result = result[0]
|
||||
+ self.assertEqual(result, 'seed')
|
||||
+
|
||||
def test_large_timestamp(self):
|
||||
# Issue #26709: 32-bit timestamp out of range
|
||||
for ts in -2**31-1, 2**31:
|
||||
@@ -509,55 +817,37 @@ class TestBinaryPlistlib(unittest.TestCase):
|
||||
data = plistlib.dumps(d, fmt=plistlib.FMT_BINARY)
|
||||
self.assertEqual(plistlib.loads(data), d)
|
||||
|
||||
+ def test_load_singletons(self):
|
||||
+ self.assertIs(self.decode(b'\x00'), None)
|
||||
+ self.assertIs(self.decode(b'\x08'), False)
|
||||
+ self.assertIs(self.decode(b'\x09'), True)
|
||||
+ self.assertEqual(self.decode(b'\x0f'), b'')
|
||||
+
|
||||
+ def test_load_int(self):
|
||||
+ self.assertEqual(self.decode(b'\x10\x00'), 0)
|
||||
+ self.assertEqual(self.decode(b'\x10\xfe'), 0xfe)
|
||||
+ self.assertEqual(self.decode(b'\x11\xfe\xdc'), 0xfedc)
|
||||
+ self.assertEqual(self.decode(b'\x12\xfe\xdc\xba\x98'), 0xfedcba98)
|
||||
+ self.assertEqual(self.decode(b'\x13\x01\x23\x45\x67\x89\xab\xcd\xef'),
|
||||
+ 0x0123456789abcdef)
|
||||
+ self.assertEqual(self.decode(b'\x13\xfe\xdc\xba\x98\x76\x54\x32\x10'),
|
||||
+ -0x123456789abcdf0)
|
||||
+
|
||||
+ def test_unsupported(self):
|
||||
+ unsupported = [*range(1, 8), *range(10, 15),
|
||||
+ 0x20, 0x21, *range(0x24, 0x33), *range(0x34, 0x40)]
|
||||
+ for i in [0x70, 0x90, 0xb0, 0xc0, 0xe0, 0xf0]:
|
||||
+ unsupported.extend(i + j for j in range(16))
|
||||
+ for token in unsupported:
|
||||
+ with self.subTest(f'token {token:02x}'):
|
||||
+ with self.assertRaises(plistlib.InvalidFileException):
|
||||
+ self.decode(bytes([token]) + b'\x00'*16)
|
||||
+
|
||||
def test_invalid_binary(self):
|
||||
- for data in [
|
||||
- # too short data
|
||||
- b'',
|
||||
- # too large offset_table_offset and nonstandard offset_size
|
||||
- b'\x00\x08'
|
||||
- b'\x00\x00\x00\x00\x00\x00\x03\x01'
|
||||
- b'\x00\x00\x00\x00\x00\x00\x00\x01'
|
||||
- b'\x00\x00\x00\x00\x00\x00\x00\x00'
|
||||
- b'\x00\x00\x00\x00\x00\x00\x00\x2a',
|
||||
- # integer overflow in offset_table_offset
|
||||
- b'\x00\x08'
|
||||
- b'\x00\x00\x00\x00\x00\x00\x01\x01'
|
||||
- b'\x00\x00\x00\x00\x00\x00\x00\x01'
|
||||
- b'\x00\x00\x00\x00\x00\x00\x00\x00'
|
||||
- b'\xff\xff\xff\xff\xff\xff\xff\xff',
|
||||
- # offset_size = 0
|
||||
- b'\x00\x08'
|
||||
- b'\x00\x00\x00\x00\x00\x00\x00\x01'
|
||||
- b'\x00\x00\x00\x00\x00\x00\x00\x01'
|
||||
- b'\x00\x00\x00\x00\x00\x00\x00\x00'
|
||||
- b'\x00\x00\x00\x00\x00\x00\x00\x09',
|
||||
- # ref_size = 0
|
||||
- b'\xa1\x01\x00\x08\x0a'
|
||||
- b'\x00\x00\x00\x00\x00\x00\x01\x00'
|
||||
- b'\x00\x00\x00\x00\x00\x00\x00\x02'
|
||||
- b'\x00\x00\x00\x00\x00\x00\x00\x00'
|
||||
- b'\x00\x00\x00\x00\x00\x00\x00\x0b',
|
||||
- # integer overflow in offset
|
||||
- b'\x00\xff\xff\xff\xff\xff\xff\xff\xff'
|
||||
- b'\x00\x00\x00\x00\x00\x00\x08\x01'
|
||||
- b'\x00\x00\x00\x00\x00\x00\x00\x01'
|
||||
- b'\x00\x00\x00\x00\x00\x00\x00\x00'
|
||||
- b'\x00\x00\x00\x00\x00\x00\x00\x09',
|
||||
- # invalid ASCII
|
||||
- b'\x51\xff\x08'
|
||||
- b'\x00\x00\x00\x00\x00\x00\x01\x01'
|
||||
- b'\x00\x00\x00\x00\x00\x00\x00\x01'
|
||||
- b'\x00\x00\x00\x00\x00\x00\x00\x00'
|
||||
- b'\x00\x00\x00\x00\x00\x00\x00\x0a',
|
||||
- # invalid UTF-16
|
||||
- b'\x61\xd8\x00\x08'
|
||||
- b'\x00\x00\x00\x00\x00\x00\x01\x01'
|
||||
- b'\x00\x00\x00\x00\x00\x00\x00\x01'
|
||||
- b'\x00\x00\x00\x00\x00\x00\x00\x00'
|
||||
- b'\x00\x00\x00\x00\x00\x00\x00\x0b',
|
||||
- ]:
|
||||
- with self.assertRaises(plistlib.InvalidFileException):
|
||||
- plistlib.loads(b'bplist00' + data, fmt=plistlib.FMT_BINARY)
|
||||
+ for name, data in INVALID_BINARY_PLISTS:
|
||||
+ with self.subTest(name):
|
||||
+ with self.assertRaises(plistlib.InvalidFileException):
|
||||
+ plistlib.loads(b'bplist00' + data, fmt=plistlib.FMT_BINARY)
|
||||
|
||||
|
||||
class TestPlistlibDeprecated(unittest.TestCase):
|
||||
@@ -655,9 +945,5 @@ class MiscTestCase(unittest.TestCase):
|
||||
support.check__all__(self, plistlib, blacklist=blacklist)
|
||||
|
||||
|
||||
-def test_main():
|
||||
- support.run_unittest(TestPlistlib, TestPlistlibDeprecated, MiscTestCase)
|
||||
-
|
||||
-
|
||||
if __name__ == '__main__':
|
||||
- test_main()
|
||||
+ unittest.main()
|
||||
diff --git a/Misc/NEWS.d/next/Library/2020-10-23-19-20-14.bpo-42103.C5obK2.rst b/Misc/NEWS.d/next/Library/2020-10-23-19-20-14.bpo-42103.C5obK2.rst
|
||||
new file mode 100644
|
||||
index 0000000..4eb694c
|
||||
--- /dev/null
|
||||
+++ b/Misc/NEWS.d/next/Library/2020-10-23-19-20-14.bpo-42103.C5obK2.rst
|
||||
@@ -0,0 +1,3 @@
|
||||
+:exc:`~plistlib.InvalidFileException` and :exc:`RecursionError` are now
|
||||
+the only errors caused by loading malformed binary Plist file (previously
|
||||
+ValueError and TypeError could be raised in some specific cases).
|
||||
diff --git a/Misc/NEWS.d/next/Security/2020-10-23-19-19-30.bpo-42103.cILT66.rst b/Misc/NEWS.d/next/Security/2020-10-23-19-19-30.bpo-42103.cILT66.rst
|
||||
new file mode 100644
|
||||
index 0000000..15d7b65
|
||||
--- /dev/null
|
||||
+++ b/Misc/NEWS.d/next/Security/2020-10-23-19-19-30.bpo-42103.cILT66.rst
|
||||
@@ -0,0 +1,2 @@
|
||||
+Prevented potential DoS attack via CPU and RAM exhaustion when processing
|
||||
+malformed Apple Property List files in binary format.
|
||||
--
|
||||
2.43.0
|
||||
|
|
@ -0,0 +1,88 @@
|
|||
From 0d02ff99721f7650e39ba4c7d8fe06f412bbb591 Mon Sep 17 00:00:00 2001
|
||||
From: Victor Stinner <vstinner@python.org>
|
||||
Date: Wed, 13 Dec 2023 11:50:26 +0100
|
||||
Subject: [PATCH] bpo-46623: Skip two test_zlib tests on s390x (GH-31096)
|
||||
|
||||
Skip test_pair() and test_speech128() of test_zlib on s390x since
|
||||
they fail if zlib uses the s390x hardware accelerator.
|
||||
---
|
||||
Lib/test/test_zlib.py | 32 +++++++++++++++++++
|
||||
.../2022-02-03-09-45-26.bpo-46623.vxzuhV.rst | 2 ++
|
||||
2 files changed, 34 insertions(+)
|
||||
create mode 100644 Misc/NEWS.d/next/Tests/2022-02-03-09-45-26.bpo-46623.vxzuhV.rst
|
||||
|
||||
diff --git a/Lib/test/test_zlib.py b/Lib/test/test_zlib.py
|
||||
index b7170b4..770a425 100644
|
||||
--- a/Lib/test/test_zlib.py
|
||||
+++ b/Lib/test/test_zlib.py
|
||||
@@ -1,6 +1,7 @@
|
||||
import unittest
|
||||
from test import support
|
||||
import binascii
|
||||
+import os
|
||||
import pickle
|
||||
import random
|
||||
import sys
|
||||
@@ -15,6 +16,35 @@ requires_Decompress_copy = unittest.skipUnless(
|
||||
hasattr(zlib.decompressobj(), "copy"),
|
||||
'requires Decompress.copy()')
|
||||
|
||||
+# bpo-46623: On s390x, when a hardware accelerator is used, using different
|
||||
+# ways to compress data with zlib can produce different compressed data.
|
||||
+# Simplified test_pair() code:
|
||||
+#
|
||||
+# def func1(data):
|
||||
+# return zlib.compress(data)
|
||||
+#
|
||||
+# def func2(data)
|
||||
+# co = zlib.compressobj()
|
||||
+# x1 = co.compress(data)
|
||||
+# x2 = co.flush()
|
||||
+# return x1 + x2
|
||||
+#
|
||||
+# On s390x if zlib uses a hardware accelerator, func1() creates a single
|
||||
+# "final" compressed block whereas func2() produces 3 compressed blocks (the
|
||||
+# last one is a final block). On other platforms with no accelerator, func1()
|
||||
+# and func2() produce the same compressed data made of a single (final)
|
||||
+# compressed block.
|
||||
+#
|
||||
+# Only the compressed data is different, the decompression returns the original
|
||||
+# data:
|
||||
+#
|
||||
+# zlib.decompress(func1(data)) == zlib.decompress(func2(data)) == data
|
||||
+#
|
||||
+# Make the assumption that s390x always has an accelerator to simplify the skip
|
||||
+# condition. Windows doesn't have os.uname() but it doesn't support s390x.
|
||||
+skip_on_s390x = unittest.skipIf(hasattr(os, 'uname') and os.uname().machine == 's390x',
|
||||
+ 'skipped on s390x')
|
||||
+
|
||||
|
||||
class VersionTestCase(unittest.TestCase):
|
||||
|
||||
@@ -174,6 +204,7 @@ class CompressTestCase(BaseCompressTestCase, unittest.TestCase):
|
||||
bufsize=zlib.DEF_BUF_SIZE),
|
||||
HAMLET_SCENE)
|
||||
|
||||
+ @skip_on_s390x
|
||||
def test_speech128(self):
|
||||
# compress more data
|
||||
data = HAMLET_SCENE * 128
|
||||
@@ -225,6 +256,7 @@ class CompressTestCase(BaseCompressTestCase, unittest.TestCase):
|
||||
|
||||
class CompressObjectTestCase(BaseCompressTestCase, unittest.TestCase):
|
||||
# Test compression object
|
||||
+ @skip_on_s390x
|
||||
def test_pair(self):
|
||||
# straightforward compress/decompress objects
|
||||
datasrc = HAMLET_SCENE * 128
|
||||
diff --git a/Misc/NEWS.d/next/Tests/2022-02-03-09-45-26.bpo-46623.vxzuhV.rst b/Misc/NEWS.d/next/Tests/2022-02-03-09-45-26.bpo-46623.vxzuhV.rst
|
||||
new file mode 100644
|
||||
index 0000000..be085c0
|
||||
--- /dev/null
|
||||
+++ b/Misc/NEWS.d/next/Tests/2022-02-03-09-45-26.bpo-46623.vxzuhV.rst
|
||||
@@ -0,0 +1,2 @@
|
||||
+Skip test_pair() and test_speech128() of test_zlib on s390x since they fail
|
||||
+if zlib uses the s390x hardware accelerator. Patch by Victor Stinner.
|
||||
--
|
||||
2.43.0
|
||||
|
|
@ -0,0 +1,750 @@
|
|||
From 0000000000000000000000000000000000000000 Mon Sep 17 00:00:00 2001
|
||||
From: Victor Stinner <vstinner@python.org>
|
||||
Date: Fri, 15 Dec 2023 16:10:40 +0100
|
||||
Subject: [PATCH] 00415: [CVE-2023-27043] gh-102988: Reject malformed addresses
|
||||
in email.parseaddr() (#111116)
|
||||
|
||||
Detect email address parsing errors and return empty tuple to
|
||||
indicate the parsing error (old API). Add an optional 'strict'
|
||||
parameter to getaddresses() and parseaddr() functions. Patch by
|
||||
Thomas Dwyer.
|
||||
|
||||
Co-Authored-By: Thomas Dwyer <github@tomd.tel>
|
||||
---
|
||||
Doc/library/email.utils.rst | 19 +-
|
||||
Lib/email/utils.py | 151 ++++++++++++-
|
||||
Lib/test/test_email/test_email.py | 204 +++++++++++++++++-
|
||||
...-10-20-15-28-08.gh-issue-102988.dStNO7.rst | 8 +
|
||||
4 files changed, 361 insertions(+), 21 deletions(-)
|
||||
create mode 100644 Misc/NEWS.d/next/Library/2023-10-20-15-28-08.gh-issue-102988.dStNO7.rst
|
||||
|
||||
diff --git a/Doc/library/email.utils.rst b/Doc/library/email.utils.rst
|
||||
index 63fae2ab84..d1e1898591 100644
|
||||
--- a/Doc/library/email.utils.rst
|
||||
+++ b/Doc/library/email.utils.rst
|
||||
@@ -60,13 +60,18 @@ of the new API.
|
||||
begins with angle brackets, they are stripped off.
|
||||
|
||||
|
||||
-.. function:: parseaddr(address)
|
||||
+.. function:: parseaddr(address, *, strict=True)
|
||||
|
||||
Parse address -- which should be the value of some address-containing field such
|
||||
as :mailheader:`To` or :mailheader:`Cc` -- into its constituent *realname* and
|
||||
*email address* parts. Returns a tuple of that information, unless the parse
|
||||
fails, in which case a 2-tuple of ``('', '')`` is returned.
|
||||
|
||||
+ If *strict* is true, use a strict parser which rejects malformed inputs.
|
||||
+
|
||||
+ .. versionchanged:: 3.13
|
||||
+ Add *strict* optional parameter and reject malformed inputs by default.
|
||||
+
|
||||
|
||||
.. function:: formataddr(pair, charset='utf-8')
|
||||
|
||||
@@ -84,12 +89,15 @@ of the new API.
|
||||
Added the *charset* option.
|
||||
|
||||
|
||||
-.. function:: getaddresses(fieldvalues)
|
||||
+.. function:: getaddresses(fieldvalues, *, strict=True)
|
||||
|
||||
This method returns a list of 2-tuples of the form returned by ``parseaddr()``.
|
||||
*fieldvalues* is a sequence of header field values as might be returned by
|
||||
- :meth:`Message.get_all <email.message.Message.get_all>`. Here's a simple
|
||||
- example that gets all the recipients of a message::
|
||||
+ :meth:`Message.get_all <email.message.Message.get_all>`.
|
||||
+
|
||||
+ If *strict* is true, use a strict parser which rejects malformed inputs.
|
||||
+
|
||||
+ Here's a simple example that gets all the recipients of a message::
|
||||
|
||||
from email.utils import getaddresses
|
||||
|
||||
@@ -99,6 +107,9 @@ of the new API.
|
||||
resent_ccs = msg.get_all('resent-cc', [])
|
||||
all_recipients = getaddresses(tos + ccs + resent_tos + resent_ccs)
|
||||
|
||||
+ .. versionchanged:: 3.13
|
||||
+ Add *strict* optional parameter and reject malformed inputs by default.
|
||||
+
|
||||
|
||||
.. function:: parsedate(date)
|
||||
|
||||
diff --git a/Lib/email/utils.py b/Lib/email/utils.py
|
||||
index 39c2240607..f83b7e5d7e 100644
|
||||
--- a/Lib/email/utils.py
|
||||
+++ b/Lib/email/utils.py
|
||||
@@ -48,6 +48,7 @@ TICK = "'"
|
||||
specialsre = re.compile(r'[][\\()<>@,:;".]')
|
||||
escapesre = re.compile(r'[\\"]')
|
||||
|
||||
+
|
||||
def _has_surrogates(s):
|
||||
"""Return True if s contains surrogate-escaped binary data."""
|
||||
# This check is based on the fact that unless there are surrogates, utf8
|
||||
@@ -106,12 +107,127 @@ def formataddr(pair, charset='utf-8'):
|
||||
return address
|
||||
|
||||
|
||||
+def _iter_escaped_chars(addr):
|
||||
+ pos = 0
|
||||
+ escape = False
|
||||
+ for pos, ch in enumerate(addr):
|
||||
+ if escape:
|
||||
+ yield (pos, '\\' + ch)
|
||||
+ escape = False
|
||||
+ elif ch == '\\':
|
||||
+ escape = True
|
||||
+ else:
|
||||
+ yield (pos, ch)
|
||||
+ if escape:
|
||||
+ yield (pos, '\\')
|
||||
|
||||
-def getaddresses(fieldvalues):
|
||||
- """Return a list of (REALNAME, EMAIL) for each fieldvalue."""
|
||||
- all = COMMASPACE.join(fieldvalues)
|
||||
- a = _AddressList(all)
|
||||
- return a.addresslist
|
||||
+
|
||||
+def _strip_quoted_realnames(addr):
|
||||
+ """Strip real names between quotes."""
|
||||
+ if '"' not in addr:
|
||||
+ # Fast path
|
||||
+ return addr
|
||||
+
|
||||
+ start = 0
|
||||
+ open_pos = None
|
||||
+ result = []
|
||||
+ for pos, ch in _iter_escaped_chars(addr):
|
||||
+ if ch == '"':
|
||||
+ if open_pos is None:
|
||||
+ open_pos = pos
|
||||
+ else:
|
||||
+ if start != open_pos:
|
||||
+ result.append(addr[start:open_pos])
|
||||
+ start = pos + 1
|
||||
+ open_pos = None
|
||||
+
|
||||
+ if start < len(addr):
|
||||
+ result.append(addr[start:])
|
||||
+
|
||||
+ return ''.join(result)
|
||||
+
|
||||
+
|
||||
+supports_strict_parsing = True
|
||||
+
|
||||
+def getaddresses(fieldvalues, *, strict=True):
|
||||
+ """Return a list of (REALNAME, EMAIL) or ('','') for each fieldvalue.
|
||||
+
|
||||
+ When parsing fails for a fieldvalue, a 2-tuple of ('', '') is returned in
|
||||
+ its place.
|
||||
+
|
||||
+ If strict is true, use a strict parser which rejects malformed inputs.
|
||||
+ """
|
||||
+
|
||||
+ # If strict is true, if the resulting list of parsed addresses is greater
|
||||
+ # than the number of fieldvalues in the input list, a parsing error has
|
||||
+ # occurred and consequently a list containing a single empty 2-tuple [('',
|
||||
+ # '')] is returned in its place. This is done to avoid invalid output.
|
||||
+ #
|
||||
+ # Malformed input: getaddresses(['alice@example.com <bob@example.com>'])
|
||||
+ # Invalid output: [('', 'alice@example.com'), ('', 'bob@example.com')]
|
||||
+ # Safe output: [('', '')]
|
||||
+
|
||||
+ if not strict:
|
||||
+ all = COMMASPACE.join(str(v) for v in fieldvalues)
|
||||
+ a = _AddressList(all)
|
||||
+ return a.addresslist
|
||||
+
|
||||
+ fieldvalues = [str(v) for v in fieldvalues]
|
||||
+ fieldvalues = _pre_parse_validation(fieldvalues)
|
||||
+ addr = COMMASPACE.join(fieldvalues)
|
||||
+ a = _AddressList(addr)
|
||||
+ result = _post_parse_validation(a.addresslist)
|
||||
+
|
||||
+ # Treat output as invalid if the number of addresses is not equal to the
|
||||
+ # expected number of addresses.
|
||||
+ n = 0
|
||||
+ for v in fieldvalues:
|
||||
+ # When a comma is used in the Real Name part it is not a deliminator.
|
||||
+ # So strip those out before counting the commas.
|
||||
+ v = _strip_quoted_realnames(v)
|
||||
+ # Expected number of addresses: 1 + number of commas
|
||||
+ n += 1 + v.count(',')
|
||||
+ if len(result) != n:
|
||||
+ return [('', '')]
|
||||
+
|
||||
+ return result
|
||||
+
|
||||
+
|
||||
+def _check_parenthesis(addr):
|
||||
+ # Ignore parenthesis in quoted real names.
|
||||
+ addr = _strip_quoted_realnames(addr)
|
||||
+
|
||||
+ opens = 0
|
||||
+ for pos, ch in _iter_escaped_chars(addr):
|
||||
+ if ch == '(':
|
||||
+ opens += 1
|
||||
+ elif ch == ')':
|
||||
+ opens -= 1
|
||||
+ if opens < 0:
|
||||
+ return False
|
||||
+ return (opens == 0)
|
||||
+
|
||||
+
|
||||
+def _pre_parse_validation(email_header_fields):
|
||||
+ accepted_values = []
|
||||
+ for v in email_header_fields:
|
||||
+ if not _check_parenthesis(v):
|
||||
+ v = "('', '')"
|
||||
+ accepted_values.append(v)
|
||||
+
|
||||
+ return accepted_values
|
||||
+
|
||||
+
|
||||
+def _post_parse_validation(parsed_email_header_tuples):
|
||||
+ accepted_values = []
|
||||
+ # The parser would have parsed a correctly formatted domain-literal
|
||||
+ # The existence of an [ after parsing indicates a parsing failure
|
||||
+ for v in parsed_email_header_tuples:
|
||||
+ if '[' in v[1]:
|
||||
+ v = ('', '')
|
||||
+ accepted_values.append(v)
|
||||
+
|
||||
+ return accepted_values
|
||||
|
||||
|
||||
|
||||
@@ -214,16 +330,33 @@ def parsedate_to_datetime(data):
|
||||
tzinfo=datetime.timezone(datetime.timedelta(seconds=tz)))
|
||||
|
||||
|
||||
-def parseaddr(addr):
|
||||
+def parseaddr(addr, *, strict=True):
|
||||
"""
|
||||
Parse addr into its constituent realname and email address parts.
|
||||
|
||||
Return a tuple of realname and email address, unless the parse fails, in
|
||||
which case return a 2-tuple of ('', '').
|
||||
+
|
||||
+ If strict is True, use a strict parser which rejects malformed inputs.
|
||||
"""
|
||||
- addrs = _AddressList(addr).addresslist
|
||||
- if not addrs:
|
||||
- return '', ''
|
||||
+ if not strict:
|
||||
+ addrs = _AddressList(addr).addresslist
|
||||
+ if not addrs:
|
||||
+ return ('', '')
|
||||
+ return addrs[0]
|
||||
+
|
||||
+ if isinstance(addr, list):
|
||||
+ addr = addr[0]
|
||||
+
|
||||
+ if not isinstance(addr, str):
|
||||
+ return ('', '')
|
||||
+
|
||||
+ addr = _pre_parse_validation([addr])[0]
|
||||
+ addrs = _post_parse_validation(_AddressList(addr).addresslist)
|
||||
+
|
||||
+ if not addrs or len(addrs) > 1:
|
||||
+ return ('', '')
|
||||
+
|
||||
return addrs[0]
|
||||
|
||||
|
||||
diff --git a/Lib/test/test_email/test_email.py b/Lib/test/test_email/test_email.py
|
||||
index e4e40b612f..ce36efc1b1 100644
|
||||
--- a/Lib/test/test_email/test_email.py
|
||||
+++ b/Lib/test/test_email/test_email.py
|
||||
@@ -19,6 +19,7 @@ except ImportError:
|
||||
|
||||
import email
|
||||
import email.policy
|
||||
+import email.utils
|
||||
|
||||
from email.charset import Charset
|
||||
from email.header import Header, decode_header, make_header
|
||||
@@ -3207,15 +3208,154 @@ Foo
|
||||
[('Al Person', 'aperson@dom.ain'),
|
||||
('Bud Person', 'bperson@dom.ain')])
|
||||
|
||||
+ def test_getaddresses_comma_in_name(self):
|
||||
+ """GH-106669 regression test."""
|
||||
+ self.assertEqual(
|
||||
+ utils.getaddresses(
|
||||
+ [
|
||||
+ '"Bud, Person" <bperson@dom.ain>',
|
||||
+ 'aperson@dom.ain (Al Person)',
|
||||
+ '"Mariusz Felisiak" <to@example.com>',
|
||||
+ ]
|
||||
+ ),
|
||||
+ [
|
||||
+ ('Bud, Person', 'bperson@dom.ain'),
|
||||
+ ('Al Person', 'aperson@dom.ain'),
|
||||
+ ('Mariusz Felisiak', 'to@example.com'),
|
||||
+ ],
|
||||
+ )
|
||||
+
|
||||
+ def test_parsing_errors(self):
|
||||
+ """Test for parsing errors from CVE-2023-27043 and CVE-2019-16056"""
|
||||
+ alice = 'alice@example.org'
|
||||
+ bob = 'bob@example.com'
|
||||
+ empty = ('', '')
|
||||
+
|
||||
+ # Test utils.getaddresses() and utils.parseaddr() on malformed email
|
||||
+ # addresses: default behavior (strict=True) rejects malformed address,
|
||||
+ # and strict=False which tolerates malformed address.
|
||||
+ for invalid_separator, expected_non_strict in (
|
||||
+ ('(', [(f'<{bob}>', alice)]),
|
||||
+ (')', [('', alice), empty, ('', bob)]),
|
||||
+ ('<', [('', alice), empty, ('', bob), empty]),
|
||||
+ ('>', [('', alice), empty, ('', bob)]),
|
||||
+ ('[', [('', f'{alice}[<{bob}>]')]),
|
||||
+ (']', [('', alice), empty, ('', bob)]),
|
||||
+ ('@', [empty, empty, ('', bob)]),
|
||||
+ (';', [('', alice), empty, ('', bob)]),
|
||||
+ (':', [('', alice), ('', bob)]),
|
||||
+ ('.', [('', alice + '.'), ('', bob)]),
|
||||
+ ('"', [('', alice), ('', f'<{bob}>')]),
|
||||
+ ):
|
||||
+ address = f'{alice}{invalid_separator}<{bob}>'
|
||||
+ with self.subTest(address=address):
|
||||
+ self.assertEqual(utils.getaddresses([address]),
|
||||
+ [empty])
|
||||
+ self.assertEqual(utils.getaddresses([address], strict=False),
|
||||
+ expected_non_strict)
|
||||
+
|
||||
+ self.assertEqual(utils.parseaddr([address]),
|
||||
+ empty)
|
||||
+ self.assertEqual(utils.parseaddr([address], strict=False),
|
||||
+ ('', address))
|
||||
+
|
||||
+ # Comma (',') is treated differently depending on strict parameter.
|
||||
+ # Comma without quotes.
|
||||
+ address = f'{alice},<{bob}>'
|
||||
+ self.assertEqual(utils.getaddresses([address]),
|
||||
+ [('', alice), ('', bob)])
|
||||
+ self.assertEqual(utils.getaddresses([address], strict=False),
|
||||
+ [('', alice), ('', bob)])
|
||||
+ self.assertEqual(utils.parseaddr([address]),
|
||||
+ empty)
|
||||
+ self.assertEqual(utils.parseaddr([address], strict=False),
|
||||
+ ('', address))
|
||||
+
|
||||
+ # Real name between quotes containing comma.
|
||||
+ address = '"Alice, alice@example.org" <bob@example.com>'
|
||||
+ expected_strict = ('Alice, alice@example.org', 'bob@example.com')
|
||||
+ self.assertEqual(utils.getaddresses([address]), [expected_strict])
|
||||
+ self.assertEqual(utils.getaddresses([address], strict=False), [expected_strict])
|
||||
+ self.assertEqual(utils.parseaddr([address]), expected_strict)
|
||||
+ self.assertEqual(utils.parseaddr([address], strict=False),
|
||||
+ ('', address))
|
||||
+
|
||||
+ # Valid parenthesis in comments.
|
||||
+ address = 'alice@example.org (Alice)'
|
||||
+ expected_strict = ('Alice', 'alice@example.org')
|
||||
+ self.assertEqual(utils.getaddresses([address]), [expected_strict])
|
||||
+ self.assertEqual(utils.getaddresses([address], strict=False), [expected_strict])
|
||||
+ self.assertEqual(utils.parseaddr([address]), expected_strict)
|
||||
+ self.assertEqual(utils.parseaddr([address], strict=False),
|
||||
+ ('', address))
|
||||
+
|
||||
+ # Invalid parenthesis in comments.
|
||||
+ address = 'alice@example.org )Alice('
|
||||
+ self.assertEqual(utils.getaddresses([address]), [empty])
|
||||
+ self.assertEqual(utils.getaddresses([address], strict=False),
|
||||
+ [('', 'alice@example.org'), ('', ''), ('', 'Alice')])
|
||||
+ self.assertEqual(utils.parseaddr([address]), empty)
|
||||
+ self.assertEqual(utils.parseaddr([address], strict=False),
|
||||
+ ('', address))
|
||||
+
|
||||
+ # Two addresses with quotes separated by comma.
|
||||
+ address = '"Jane Doe" <jane@example.net>, "John Doe" <john@example.net>'
|
||||
+ self.assertEqual(utils.getaddresses([address]),
|
||||
+ [('Jane Doe', 'jane@example.net'),
|
||||
+ ('John Doe', 'john@example.net')])
|
||||
+ self.assertEqual(utils.getaddresses([address], strict=False),
|
||||
+ [('Jane Doe', 'jane@example.net'),
|
||||
+ ('John Doe', 'john@example.net')])
|
||||
+ self.assertEqual(utils.parseaddr([address]), empty)
|
||||
+ self.assertEqual(utils.parseaddr([address], strict=False),
|
||||
+ ('', address))
|
||||
+
|
||||
+ # Test email.utils.supports_strict_parsing attribute
|
||||
+ self.assertEqual(email.utils.supports_strict_parsing, True)
|
||||
+
|
||||
def test_getaddresses_nasty(self):
|
||||
- eq = self.assertEqual
|
||||
- eq(utils.getaddresses(['foo: ;']), [('', '')])
|
||||
- eq(utils.getaddresses(
|
||||
- ['[]*-- =~$']),
|
||||
- [('', ''), ('', ''), ('', '*--')])
|
||||
- eq(utils.getaddresses(
|
||||
- ['foo: ;', '"Jason R. Mastaler" <jason@dom.ain>']),
|
||||
- [('', ''), ('Jason R. Mastaler', 'jason@dom.ain')])
|
||||
+ for addresses, expected in (
|
||||
+ (['"Sürname, Firstname" <to@example.com>'],
|
||||
+ [('Sürname, Firstname', 'to@example.com')]),
|
||||
+
|
||||
+ (['foo: ;'],
|
||||
+ [('', '')]),
|
||||
+
|
||||
+ (['foo: ;', '"Jason R. Mastaler" <jason@dom.ain>'],
|
||||
+ [('', ''), ('Jason R. Mastaler', 'jason@dom.ain')]),
|
||||
+
|
||||
+ ([r'Pete(A nice \) chap) <pete(his account)@silly.test(his host)>'],
|
||||
+ [('Pete (A nice ) chap his account his host)', 'pete@silly.test')]),
|
||||
+
|
||||
+ (['(Empty list)(start)Undisclosed recipients :(nobody(I know))'],
|
||||
+ [('', '')]),
|
||||
+
|
||||
+ (['Mary <@machine.tld:mary@example.net>, , jdoe@test . example'],
|
||||
+ [('Mary', 'mary@example.net'), ('', ''), ('', 'jdoe@test.example')]),
|
||||
+
|
||||
+ (['John Doe <jdoe@machine(comment). example>'],
|
||||
+ [('John Doe (comment)', 'jdoe@machine.example')]),
|
||||
+
|
||||
+ (['"Mary Smith: Personal Account" <smith@home.example>'],
|
||||
+ [('Mary Smith: Personal Account', 'smith@home.example')]),
|
||||
+
|
||||
+ (['Undisclosed recipients:;'],
|
||||
+ [('', '')]),
|
||||
+
|
||||
+ ([r'<boss@nil.test>, "Giant; \"Big\" Box" <bob@example.net>'],
|
||||
+ [('', 'boss@nil.test'), ('Giant; "Big" Box', 'bob@example.net')]),
|
||||
+ ):
|
||||
+ with self.subTest(addresses=addresses):
|
||||
+ self.assertEqual(utils.getaddresses(addresses),
|
||||
+ expected)
|
||||
+ self.assertEqual(utils.getaddresses(addresses, strict=False),
|
||||
+ expected)
|
||||
+
|
||||
+ addresses = ['[]*-- =~$']
|
||||
+ self.assertEqual(utils.getaddresses(addresses),
|
||||
+ [('', '')])
|
||||
+ self.assertEqual(utils.getaddresses(addresses, strict=False),
|
||||
+ [('', ''), ('', ''), ('', '*--')])
|
||||
|
||||
def test_getaddresses_embedded_comment(self):
|
||||
"""Test proper handling of a nested comment"""
|
||||
@@ -3397,6 +3537,54 @@ multipart/report
|
||||
m = cls(*constructor, policy=email.policy.default)
|
||||
self.assertIs(m.policy, email.policy.default)
|
||||
|
||||
+ def test_iter_escaped_chars(self):
|
||||
+ self.assertEqual(list(utils._iter_escaped_chars(r'a\\b\"c\\"d')),
|
||||
+ [(0, 'a'),
|
||||
+ (2, '\\\\'),
|
||||
+ (3, 'b'),
|
||||
+ (5, '\\"'),
|
||||
+ (6, 'c'),
|
||||
+ (8, '\\\\'),
|
||||
+ (9, '"'),
|
||||
+ (10, 'd')])
|
||||
+ self.assertEqual(list(utils._iter_escaped_chars('a\\')),
|
||||
+ [(0, 'a'), (1, '\\')])
|
||||
+
|
||||
+ def test_strip_quoted_realnames(self):
|
||||
+ def check(addr, expected):
|
||||
+ self.assertEqual(utils._strip_quoted_realnames(addr), expected)
|
||||
+
|
||||
+ check('"Jane Doe" <jane@example.net>, "John Doe" <john@example.net>',
|
||||
+ ' <jane@example.net>, <john@example.net>')
|
||||
+ check(r'"Jane \"Doe\"." <jane@example.net>',
|
||||
+ ' <jane@example.net>')
|
||||
+
|
||||
+ # special cases
|
||||
+ check(r'before"name"after', 'beforeafter')
|
||||
+ check(r'before"name"', 'before')
|
||||
+ check(r'b"name"', 'b') # single char
|
||||
+ check(r'"name"after', 'after')
|
||||
+ check(r'"name"a', 'a') # single char
|
||||
+ check(r'"name"', '')
|
||||
+
|
||||
+ # no change
|
||||
+ for addr in (
|
||||
+ 'Jane Doe <jane@example.net>, John Doe <john@example.net>',
|
||||
+ 'lone " quote',
|
||||
+ ):
|
||||
+ self.assertEqual(utils._strip_quoted_realnames(addr), addr)
|
||||
+
|
||||
+
|
||||
+ def test_check_parenthesis(self):
|
||||
+ addr = 'alice@example.net'
|
||||
+ self.assertTrue(utils._check_parenthesis(f'{addr} (Alice)'))
|
||||
+ self.assertFalse(utils._check_parenthesis(f'{addr} )Alice('))
|
||||
+ self.assertFalse(utils._check_parenthesis(f'{addr} (Alice))'))
|
||||
+ self.assertFalse(utils._check_parenthesis(f'{addr} ((Alice)'))
|
||||
+
|
||||
+ # Ignore real name between quotes
|
||||
+ self.assertTrue(utils._check_parenthesis(f'")Alice((" {addr}'))
|
||||
+
|
||||
|
||||
# Test the iterator/generators
|
||||
class TestIterators(TestEmailBase):
|
||||
diff --git a/Misc/NEWS.d/next/Library/2023-10-20-15-28-08.gh-issue-102988.dStNO7.rst b/Misc/NEWS.d/next/Library/2023-10-20-15-28-08.gh-issue-102988.dStNO7.rst
|
||||
new file mode 100644
|
||||
index 0000000000..3d0e9e4078
|
||||
--- /dev/null
|
||||
+++ b/Misc/NEWS.d/next/Library/2023-10-20-15-28-08.gh-issue-102988.dStNO7.rst
|
||||
@@ -0,0 +1,8 @@
|
||||
+:func:`email.utils.getaddresses` and :func:`email.utils.parseaddr` now
|
||||
+return ``('', '')`` 2-tuples in more situations where invalid email
|
||||
+addresses are encountered instead of potentially inaccurate values. Add
|
||||
+optional *strict* parameter to these two functions: use ``strict=False`` to
|
||||
+get the old behavior, accept malformed inputs.
|
||||
+``getattr(email.utils, 'supports_strict_parsing', False)`` can be use to check
|
||||
+if the *strict* paramater is available. Patch by Thomas Dwyer and Victor
|
||||
+Stinner to improve the CVE-2023-27043 fix.
|
||||
|
||||
|
||||
From 4df4fad359c280f2328b98ea9b4414f244624a58 Mon Sep 17 00:00:00 2001
|
||||
From: Lumir Balhar <lbalhar@redhat.com>
|
||||
Date: Mon, 18 Dec 2023 20:15:33 +0100
|
||||
Subject: [PATCH] Make it possible to disable strict parsing in email module
|
||||
|
||||
---
|
||||
Doc/library/email.utils.rst | 26 +++++++++++
|
||||
Lib/email/utils.py | 54 ++++++++++++++++++++++-
|
||||
Lib/test/test_email/test_email.py | 72 ++++++++++++++++++++++++++++++-
|
||||
3 files changed, 149 insertions(+), 3 deletions(-)
|
||||
|
||||
diff --git a/Doc/library/email.utils.rst b/Doc/library/email.utils.rst
|
||||
index d1e1898591..7aef773b5f 100644
|
||||
--- a/Doc/library/email.utils.rst
|
||||
+++ b/Doc/library/email.utils.rst
|
||||
@@ -69,6 +69,19 @@ of the new API.
|
||||
|
||||
If *strict* is true, use a strict parser which rejects malformed inputs.
|
||||
|
||||
+ The default setting for *strict* is set to ``True``, but you can override
|
||||
+ it by setting the environment variable ``PYTHON_EMAIL_DISABLE_STRICT_ADDR_PARSING``
|
||||
+ to non-empty string.
|
||||
+
|
||||
+ Additionally, you can permanently set the default value for *strict* to
|
||||
+ ``False`` by creating the configuration file ``/etc/python/email.cfg``
|
||||
+ with the following content:
|
||||
+
|
||||
+ .. code-block:: ini
|
||||
+
|
||||
+ [email_addr_parsing]
|
||||
+ PYTHON_EMAIL_DISABLE_STRICT_ADDR_PARSING = true
|
||||
+
|
||||
.. versionchanged:: 3.13
|
||||
Add *strict* optional parameter and reject malformed inputs by default.
|
||||
|
||||
@@ -97,6 +110,19 @@ of the new API.
|
||||
|
||||
If *strict* is true, use a strict parser which rejects malformed inputs.
|
||||
|
||||
+ The default setting for *strict* is set to ``True``, but you can override
|
||||
+ it by setting the environment variable ``PYTHON_EMAIL_DISABLE_STRICT_ADDR_PARSING``
|
||||
+ to non-empty string.
|
||||
+
|
||||
+ Additionally, you can permanently set the default value for *strict* to
|
||||
+ ``False`` by creating the configuration file ``/etc/python/email.cfg``
|
||||
+ with the following content:
|
||||
+
|
||||
+ .. code-block:: ini
|
||||
+
|
||||
+ [email_addr_parsing]
|
||||
+ PYTHON_EMAIL_DISABLE_STRICT_ADDR_PARSING = true
|
||||
+
|
||||
Here's a simple example that gets all the recipients of a message::
|
||||
|
||||
from email.utils import getaddresses
|
||||
diff --git a/Lib/email/utils.py b/Lib/email/utils.py
|
||||
index f83b7e5d7e..b8e90ceb8e 100644
|
||||
--- a/Lib/email/utils.py
|
||||
+++ b/Lib/email/utils.py
|
||||
@@ -48,6 +48,46 @@ TICK = "'"
|
||||
specialsre = re.compile(r'[][\\()<>@,:;".]')
|
||||
escapesre = re.compile(r'[\\"]')
|
||||
|
||||
+_EMAIL_CONFIG_FILE = "/etc/python/email.cfg"
|
||||
+_cached_strict_addr_parsing = None
|
||||
+
|
||||
+
|
||||
+def _use_strict_email_parsing():
|
||||
+ """"Cache implementation for _cached_strict_addr_parsing"""
|
||||
+ global _cached_strict_addr_parsing
|
||||
+ if _cached_strict_addr_parsing is None:
|
||||
+ _cached_strict_addr_parsing = _use_strict_email_parsing_impl()
|
||||
+ return _cached_strict_addr_parsing
|
||||
+
|
||||
+
|
||||
+def _use_strict_email_parsing_impl():
|
||||
+ """Returns True if strict email parsing is not disabled by
|
||||
+ config file or env variable.
|
||||
+ """
|
||||
+ disabled = bool(os.environ.get("PYTHON_EMAIL_DISABLE_STRICT_ADDR_PARSING"))
|
||||
+ if disabled:
|
||||
+ return False
|
||||
+
|
||||
+ try:
|
||||
+ file = open(_EMAIL_CONFIG_FILE)
|
||||
+ except FileNotFoundError:
|
||||
+ pass
|
||||
+ else:
|
||||
+ with file:
|
||||
+ import configparser
|
||||
+ config = configparser.ConfigParser(
|
||||
+ interpolation=None,
|
||||
+ comment_prefixes=('#', ),
|
||||
+
|
||||
+ )
|
||||
+ config.read_file(file)
|
||||
+ disabled = config.getboolean('email_addr_parsing', "PYTHON_EMAIL_DISABLE_STRICT_ADDR_PARSING", fallback=None)
|
||||
+
|
||||
+ if disabled:
|
||||
+ return False
|
||||
+
|
||||
+ return True
|
||||
+
|
||||
|
||||
def _has_surrogates(s):
|
||||
"""Return True if s contains surrogate-escaped binary data."""
|
||||
@@ -149,7 +189,7 @@ def _strip_quoted_realnames(addr):
|
||||
|
||||
supports_strict_parsing = True
|
||||
|
||||
-def getaddresses(fieldvalues, *, strict=True):
|
||||
+def getaddresses(fieldvalues, *, strict=None):
|
||||
"""Return a list of (REALNAME, EMAIL) or ('','') for each fieldvalue.
|
||||
|
||||
When parsing fails for a fieldvalue, a 2-tuple of ('', '') is returned in
|
||||
@@ -158,6 +198,11 @@ def getaddresses(fieldvalues, *, strict=True):
|
||||
If strict is true, use a strict parser which rejects malformed inputs.
|
||||
"""
|
||||
|
||||
+ # If default is used, it's True unless disabled
|
||||
+ # by env variable or config file.
|
||||
+ if strict == None:
|
||||
+ strict = _use_strict_email_parsing()
|
||||
+
|
||||
# If strict is true, if the resulting list of parsed addresses is greater
|
||||
# than the number of fieldvalues in the input list, a parsing error has
|
||||
# occurred and consequently a list containing a single empty 2-tuple [('',
|
||||
@@ -330,7 +375,7 @@ def parsedate_to_datetime(data):
|
||||
tzinfo=datetime.timezone(datetime.timedelta(seconds=tz)))
|
||||
|
||||
|
||||
-def parseaddr(addr, *, strict=True):
|
||||
+def parseaddr(addr, *, strict=None):
|
||||
"""
|
||||
Parse addr into its constituent realname and email address parts.
|
||||
|
||||
@@ -339,6 +384,11 @@ def parseaddr(addr, *, strict=True):
|
||||
|
||||
If strict is True, use a strict parser which rejects malformed inputs.
|
||||
"""
|
||||
+ # If default is used, it's True unless disabled
|
||||
+ # by env variable or config file.
|
||||
+ if strict == None:
|
||||
+ strict = _use_strict_email_parsing()
|
||||
+
|
||||
if not strict:
|
||||
addrs = _AddressList(addr).addresslist
|
||||
if not addrs:
|
||||
diff --git a/Lib/test/test_email/test_email.py b/Lib/test/test_email/test_email.py
|
||||
index ce36efc1b1..05ea201b68 100644
|
||||
--- a/Lib/test/test_email/test_email.py
|
||||
+++ b/Lib/test/test_email/test_email.py
|
||||
@@ -7,6 +7,9 @@ import time
|
||||
import base64
|
||||
import unittest
|
||||
import textwrap
|
||||
+import contextlib
|
||||
+import tempfile
|
||||
+import os
|
||||
|
||||
from io import StringIO, BytesIO
|
||||
from itertools import chain
|
||||
@@ -41,7 +44,7 @@ from email import iterators
|
||||
from email import base64mime
|
||||
from email import quoprimime
|
||||
|
||||
-from test.support import unlink, start_threads
|
||||
+from test.support import unlink, start_threads, EnvironmentVarGuard, swap_attr
|
||||
from test.test_email import openfile, TestEmailBase
|
||||
|
||||
# These imports are documented to work, but we are testing them using a
|
||||
@@ -3313,6 +3316,73 @@ Foo
|
||||
# Test email.utils.supports_strict_parsing attribute
|
||||
self.assertEqual(email.utils.supports_strict_parsing, True)
|
||||
|
||||
+ def test_parsing_errors_strict_set_via_env_var(self):
|
||||
+ address = 'alice@example.org )Alice('
|
||||
+ empty = ('', '')
|
||||
+
|
||||
+ # Reset cached default value to make the function
|
||||
+ # reload the config file provided below.
|
||||
+ utils._cached_strict_addr_parsing = None
|
||||
+
|
||||
+ # Strict disabled via env variable, old behavior expected
|
||||
+ with EnvironmentVarGuard() as environ:
|
||||
+ environ["PYTHON_EMAIL_DISABLE_STRICT_ADDR_PARSING"] = "1"
|
||||
+
|
||||
+ self.assertEqual(utils.getaddresses([address]),
|
||||
+ [('', 'alice@example.org'), ('', ''), ('', 'Alice')])
|
||||
+ self.assertEqual(utils.parseaddr([address]), ('', address))
|
||||
+
|
||||
+ # Clear cache again
|
||||
+ utils._cached_strict_addr_parsing = None
|
||||
+
|
||||
+ # Default strict=True, empty result expected
|
||||
+ self.assertEqual(utils.getaddresses([address]), [empty])
|
||||
+ self.assertEqual(utils.parseaddr([address]), empty)
|
||||
+
|
||||
+ # Clear cache again
|
||||
+ utils._cached_strict_addr_parsing = None
|
||||
+
|
||||
+ # Empty string in env variable = strict parsing enabled (default)
|
||||
+ with EnvironmentVarGuard() as environ:
|
||||
+ environ["PYTHON_EMAIL_DISABLE_STRICT_ADDR_PARSING"] = ""
|
||||
+
|
||||
+ # Default strict=True, empty result expected
|
||||
+ self.assertEqual(utils.getaddresses([address]), [empty])
|
||||
+ self.assertEqual(utils.parseaddr([address]), empty)
|
||||
+
|
||||
+ @contextlib.contextmanager
|
||||
+ def _email_strict_parsing_conf(self):
|
||||
+ """Context for the given email strict parsing configured in config file"""
|
||||
+ with tempfile.TemporaryDirectory() as tmpdirname:
|
||||
+ filename = os.path.join(tmpdirname, 'conf.cfg')
|
||||
+ with swap_attr(utils, "_EMAIL_CONFIG_FILE", filename):
|
||||
+ with open(filename, 'w') as file:
|
||||
+ file.write('[email_addr_parsing]\n')
|
||||
+ file.write('PYTHON_EMAIL_DISABLE_STRICT_ADDR_PARSING = true')
|
||||
+ utils._EMAIL_CONFIG_FILE = filename
|
||||
+ yield
|
||||
+
|
||||
+ def test_parsing_errors_strict_disabled_via_config_file(self):
|
||||
+ address = 'alice@example.org )Alice('
|
||||
+ empty = ('', '')
|
||||
+
|
||||
+ # Reset cached default value to make the function
|
||||
+ # reload the config file provided below.
|
||||
+ utils._cached_strict_addr_parsing = None
|
||||
+
|
||||
+ # Strict disabled via config file, old results expected
|
||||
+ with self._email_strict_parsing_conf():
|
||||
+ self.assertEqual(utils.getaddresses([address]),
|
||||
+ [('', 'alice@example.org'), ('', ''), ('', 'Alice')])
|
||||
+ self.assertEqual(utils.parseaddr([address]), ('', address))
|
||||
+
|
||||
+ # Clear cache again
|
||||
+ utils._cached_strict_addr_parsing = None
|
||||
+
|
||||
+ # Default strict=True, empty result expected
|
||||
+ self.assertEqual(utils.getaddresses([address]), [empty])
|
||||
+ self.assertEqual(utils.parseaddr([address]), empty)
|
||||
+
|
||||
def test_getaddresses_nasty(self):
|
||||
for addresses, expected in (
|
||||
(['"Sürname, Firstname" <to@example.com>'],
|
||||
--
|
||||
2.43.0
|
||||
|
|
@ -0,0 +1,90 @@
|
|||
From 87acab66e124912549fbc3151f27ca7fae76386c Mon Sep 17 00:00:00 2001
|
||||
From: Serhiy Storchaka <storchaka@gmail.com>
|
||||
Date: Tue, 23 Apr 2024 19:54:00 +0200
|
||||
Subject: [PATCH] gh-115133: Fix tests for XMLPullParser with Expat 2.6.0
|
||||
|
||||
Feeding the parser by too small chunks defers parsing to prevent
|
||||
CVE-2023-52425. Future versions of Expat may be more reactive.
|
||||
|
||||
(cherry picked from commit 4a08e7b3431cd32a0daf22a33421cd3035343dc4)
|
||||
---
|
||||
Lib/test/test_xml_etree.py | 53 +++++++++++--------
|
||||
...-02-08-14-21-28.gh-issue-115133.ycl4ko.rst | 2 +
|
||||
2 files changed, 33 insertions(+), 22 deletions(-)
|
||||
create mode 100644 Misc/NEWS.d/next/Library/2024-02-08-14-21-28.gh-issue-115133.ycl4ko.rst
|
||||
|
||||
diff --git a/Lib/test/test_xml_etree.py b/Lib/test/test_xml_etree.py
|
||||
index acaa519..c01af47 100644
|
||||
--- a/Lib/test/test_xml_etree.py
|
||||
+++ b/Lib/test/test_xml_etree.py
|
||||
@@ -1044,28 +1044,37 @@ class XMLPullParserTest(unittest.TestCase):
|
||||
self.assertEqual([(action, elem.tag) for action, elem in events],
|
||||
expected)
|
||||
|
||||
- def test_simple_xml(self):
|
||||
- for chunk_size in (None, 1, 5):
|
||||
- with self.subTest(chunk_size=chunk_size):
|
||||
- parser = ET.XMLPullParser()
|
||||
- self.assert_event_tags(parser, [])
|
||||
- self._feed(parser, "<!-- comment -->\n", chunk_size)
|
||||
- self.assert_event_tags(parser, [])
|
||||
- self._feed(parser,
|
||||
- "<root>\n <element key='value'>text</element",
|
||||
- chunk_size)
|
||||
- self.assert_event_tags(parser, [])
|
||||
- self._feed(parser, ">\n", chunk_size)
|
||||
- self.assert_event_tags(parser, [('end', 'element')])
|
||||
- self._feed(parser, "<element>text</element>tail\n", chunk_size)
|
||||
- self._feed(parser, "<empty-element/>\n", chunk_size)
|
||||
- self.assert_event_tags(parser, [
|
||||
- ('end', 'element'),
|
||||
- ('end', 'empty-element'),
|
||||
- ])
|
||||
- self._feed(parser, "</root>\n", chunk_size)
|
||||
- self.assert_event_tags(parser, [('end', 'root')])
|
||||
- self.assertIsNone(parser.close())
|
||||
+ def test_simple_xml(self, chunk_size=None):
|
||||
+ parser = ET.XMLPullParser()
|
||||
+ self.assert_event_tags(parser, [])
|
||||
+ self._feed(parser, "<!-- comment -->\n", chunk_size)
|
||||
+ self.assert_event_tags(parser, [])
|
||||
+ self._feed(parser,
|
||||
+ "<root>\n <element key='value'>text</element",
|
||||
+ chunk_size)
|
||||
+ self.assert_event_tags(parser, [])
|
||||
+ self._feed(parser, ">\n", chunk_size)
|
||||
+ self.assert_event_tags(parser, [('end', 'element')])
|
||||
+ self._feed(parser, "<element>text</element>tail\n", chunk_size)
|
||||
+ self._feed(parser, "<empty-element/>\n", chunk_size)
|
||||
+ self.assert_event_tags(parser, [
|
||||
+ ('end', 'element'),
|
||||
+ ('end', 'empty-element'),
|
||||
+ ])
|
||||
+ self._feed(parser, "</root>\n", chunk_size)
|
||||
+ self.assert_event_tags(parser, [('end', 'root')])
|
||||
+ self.assertIsNone(parser.close())
|
||||
+
|
||||
+ @unittest.expectedFailure
|
||||
+ def test_simple_xml_chunk_1(self):
|
||||
+ self.test_simple_xml(chunk_size=1)
|
||||
+
|
||||
+ @unittest.expectedFailure
|
||||
+ def test_simple_xml_chunk_5(self):
|
||||
+ self.test_simple_xml(chunk_size=5)
|
||||
+
|
||||
+ def test_simple_xml_chunk_22(self):
|
||||
+ self.test_simple_xml(chunk_size=22)
|
||||
|
||||
def test_feed_while_iterating(self):
|
||||
parser = ET.XMLPullParser()
|
||||
diff --git a/Misc/NEWS.d/next/Library/2024-02-08-14-21-28.gh-issue-115133.ycl4ko.rst b/Misc/NEWS.d/next/Library/2024-02-08-14-21-28.gh-issue-115133.ycl4ko.rst
|
||||
new file mode 100644
|
||||
index 0000000..6f10152
|
||||
--- /dev/null
|
||||
+++ b/Misc/NEWS.d/next/Library/2024-02-08-14-21-28.gh-issue-115133.ycl4ko.rst
|
||||
@@ -0,0 +1,2 @@
|
||||
+Fix tests for :class:`~xml.etree.ElementTree.XMLPullParser` with Expat
|
||||
+2.6.0.
|
||||
--
|
||||
2.44.0
|
||||
|
|
@ -0,0 +1,291 @@
|
|||
From 82f1ea4b72be40f58fd0a9a37f8d8d2f7d16f9e0 Mon Sep 17 00:00:00 2001
|
||||
From: Lumir Balhar <lbalhar@redhat.com>
|
||||
Date: Wed, 24 Apr 2024 00:19:23 +0200
|
||||
Subject: [PATCH] CVE-2023-6597
|
||||
|
||||
Co-authored-by: Søren Løvborg <sorenl@unity3d.com>
|
||||
Co-authored-by: Serhiy Storchaka <storchaka@gmail.com>
|
||||
---
|
||||
Lib/tempfile.py | 44 +++++++++-
|
||||
Lib/test/test_tempfile.py | 166 +++++++++++++++++++++++++++++++++++---
|
||||
2 files changed, 199 insertions(+), 11 deletions(-)
|
||||
|
||||
diff --git a/Lib/tempfile.py b/Lib/tempfile.py
|
||||
index 2cb5434..d79b70c 100644
|
||||
--- a/Lib/tempfile.py
|
||||
+++ b/Lib/tempfile.py
|
||||
@@ -276,6 +276,23 @@ def _mkstemp_inner(dir, pre, suf, flags, output_type):
|
||||
"No usable temporary file name found")
|
||||
|
||||
|
||||
+def _dont_follow_symlinks(func, path, *args):
|
||||
+ # Pass follow_symlinks=False, unless not supported on this platform.
|
||||
+ if func in _os.supports_follow_symlinks:
|
||||
+ func(path, *args, follow_symlinks=False)
|
||||
+ elif _os.name == 'nt' or not _os.path.islink(path):
|
||||
+ func(path, *args)
|
||||
+
|
||||
+
|
||||
+def _resetperms(path):
|
||||
+ try:
|
||||
+ chflags = _os.chflags
|
||||
+ except AttributeError:
|
||||
+ pass
|
||||
+ else:
|
||||
+ _dont_follow_symlinks(chflags, path, 0)
|
||||
+ _dont_follow_symlinks(_os.chmod, path, 0o700)
|
||||
+
|
||||
# User visible interfaces.
|
||||
|
||||
def gettempprefix():
|
||||
@@ -794,9 +811,32 @@ class TemporaryDirectory(object):
|
||||
self, self._cleanup, self.name,
|
||||
warn_message="Implicitly cleaning up {!r}".format(self))
|
||||
|
||||
+ @classmethod
|
||||
+ def _rmtree(cls, name):
|
||||
+ def onerror(func, path, exc_info):
|
||||
+ if issubclass(exc_info[0], PermissionError):
|
||||
+ try:
|
||||
+ if path != name:
|
||||
+ _resetperms(_os.path.dirname(path))
|
||||
+ _resetperms(path)
|
||||
+
|
||||
+ try:
|
||||
+ _os.unlink(path)
|
||||
+ # PermissionError is raised on FreeBSD for directories
|
||||
+ except (IsADirectoryError, PermissionError):
|
||||
+ cls._rmtree(path)
|
||||
+ except FileNotFoundError:
|
||||
+ pass
|
||||
+ elif issubclass(exc_info[0], FileNotFoundError):
|
||||
+ pass
|
||||
+ else:
|
||||
+ raise
|
||||
+
|
||||
+ _shutil.rmtree(name, onerror=onerror)
|
||||
+
|
||||
@classmethod
|
||||
def _cleanup(cls, name, warn_message):
|
||||
- _shutil.rmtree(name)
|
||||
+ cls._rmtree(name)
|
||||
_warnings.warn(warn_message, ResourceWarning)
|
||||
|
||||
def __repr__(self):
|
||||
@@ -810,4 +850,4 @@ class TemporaryDirectory(object):
|
||||
|
||||
def cleanup(self):
|
||||
if self._finalizer.detach():
|
||||
- _shutil.rmtree(self.name)
|
||||
+ self._rmtree(self.name)
|
||||
diff --git a/Lib/test/test_tempfile.py b/Lib/test/test_tempfile.py
|
||||
index 710756b..c5560e1 100644
|
||||
--- a/Lib/test/test_tempfile.py
|
||||
+++ b/Lib/test/test_tempfile.py
|
||||
@@ -1298,19 +1298,25 @@ class NulledModules:
|
||||
class TestTemporaryDirectory(BaseTestCase):
|
||||
"""Test TemporaryDirectory()."""
|
||||
|
||||
- def do_create(self, dir=None, pre="", suf="", recurse=1):
|
||||
+ def do_create(self, dir=None, pre="", suf="", recurse=1, dirs=1, files=1):
|
||||
if dir is None:
|
||||
dir = tempfile.gettempdir()
|
||||
tmp = tempfile.TemporaryDirectory(dir=dir, prefix=pre, suffix=suf)
|
||||
self.nameCheck(tmp.name, dir, pre, suf)
|
||||
- # Create a subdirectory and some files
|
||||
- if recurse:
|
||||
- d1 = self.do_create(tmp.name, pre, suf, recurse-1)
|
||||
- d1.name = None
|
||||
- with open(os.path.join(tmp.name, "test.txt"), "wb") as f:
|
||||
- f.write(b"Hello world!")
|
||||
+ self.do_create2(tmp.name, recurse, dirs, files)
|
||||
return tmp
|
||||
|
||||
+ def do_create2(self, path, recurse=1, dirs=1, files=1):
|
||||
+ # Create subdirectories and some files
|
||||
+ if recurse:
|
||||
+ for i in range(dirs):
|
||||
+ name = os.path.join(path, "dir%d" % i)
|
||||
+ os.mkdir(name)
|
||||
+ self.do_create2(name, recurse-1, dirs, files)
|
||||
+ for i in range(files):
|
||||
+ with open(os.path.join(path, "test%d.txt" % i), "wb") as f:
|
||||
+ f.write(b"Hello world!")
|
||||
+
|
||||
def test_mkdtemp_failure(self):
|
||||
# Check no additional exception if mkdtemp fails
|
||||
# Previously would raise AttributeError instead
|
||||
@@ -1350,11 +1356,108 @@ class TestTemporaryDirectory(BaseTestCase):
|
||||
"TemporaryDirectory %s exists after cleanup" % d1.name)
|
||||
self.assertTrue(os.path.exists(d2.name),
|
||||
"Directory pointed to by a symlink was deleted")
|
||||
- self.assertEqual(os.listdir(d2.name), ['test.txt'],
|
||||
+ self.assertEqual(os.listdir(d2.name), ['test0.txt'],
|
||||
"Contents of the directory pointed to by a symlink "
|
||||
"were deleted")
|
||||
d2.cleanup()
|
||||
|
||||
+ @support.skip_unless_symlink
|
||||
+ def test_cleanup_with_symlink_modes(self):
|
||||
+ # cleanup() should not follow symlinks when fixing mode bits (#91133)
|
||||
+ with self.do_create(recurse=0) as d2:
|
||||
+ file1 = os.path.join(d2, 'file1')
|
||||
+ open(file1, 'wb').close()
|
||||
+ dir1 = os.path.join(d2, 'dir1')
|
||||
+ os.mkdir(dir1)
|
||||
+ for mode in range(8):
|
||||
+ mode <<= 6
|
||||
+ with self.subTest(mode=format(mode, '03o')):
|
||||
+ def test(target, target_is_directory):
|
||||
+ d1 = self.do_create(recurse=0)
|
||||
+ symlink = os.path.join(d1.name, 'symlink')
|
||||
+ os.symlink(target, symlink,
|
||||
+ target_is_directory=target_is_directory)
|
||||
+ try:
|
||||
+ os.chmod(symlink, mode, follow_symlinks=False)
|
||||
+ except NotImplementedError:
|
||||
+ pass
|
||||
+ try:
|
||||
+ os.chmod(symlink, mode)
|
||||
+ except FileNotFoundError:
|
||||
+ pass
|
||||
+ os.chmod(d1.name, mode)
|
||||
+ d1.cleanup()
|
||||
+ self.assertFalse(os.path.exists(d1.name))
|
||||
+
|
||||
+ with self.subTest('nonexisting file'):
|
||||
+ test('nonexisting', target_is_directory=False)
|
||||
+ with self.subTest('nonexisting dir'):
|
||||
+ test('nonexisting', target_is_directory=True)
|
||||
+
|
||||
+ with self.subTest('existing file'):
|
||||
+ os.chmod(file1, mode)
|
||||
+ old_mode = os.stat(file1).st_mode
|
||||
+ test(file1, target_is_directory=False)
|
||||
+ new_mode = os.stat(file1).st_mode
|
||||
+ self.assertEqual(new_mode, old_mode,
|
||||
+ '%03o != %03o' % (new_mode, old_mode))
|
||||
+
|
||||
+ with self.subTest('existing dir'):
|
||||
+ os.chmod(dir1, mode)
|
||||
+ old_mode = os.stat(dir1).st_mode
|
||||
+ test(dir1, target_is_directory=True)
|
||||
+ new_mode = os.stat(dir1).st_mode
|
||||
+ self.assertEqual(new_mode, old_mode,
|
||||
+ '%03o != %03o' % (new_mode, old_mode))
|
||||
+
|
||||
+ @unittest.skipUnless(hasattr(os, 'chflags'), 'requires os.chflags')
|
||||
+ @support.skip_unless_symlink
|
||||
+ def test_cleanup_with_symlink_flags(self):
|
||||
+ # cleanup() should not follow symlinks when fixing flags (#91133)
|
||||
+ flags = stat.UF_IMMUTABLE | stat.UF_NOUNLINK
|
||||
+ self.check_flags(flags)
|
||||
+
|
||||
+ with self.do_create(recurse=0) as d2:
|
||||
+ file1 = os.path.join(d2, 'file1')
|
||||
+ open(file1, 'wb').close()
|
||||
+ dir1 = os.path.join(d2, 'dir1')
|
||||
+ os.mkdir(dir1)
|
||||
+ def test(target, target_is_directory):
|
||||
+ d1 = self.do_create(recurse=0)
|
||||
+ symlink = os.path.join(d1.name, 'symlink')
|
||||
+ os.symlink(target, symlink,
|
||||
+ target_is_directory=target_is_directory)
|
||||
+ try:
|
||||
+ os.chflags(symlink, flags, follow_symlinks=False)
|
||||
+ except NotImplementedError:
|
||||
+ pass
|
||||
+ try:
|
||||
+ os.chflags(symlink, flags)
|
||||
+ except FileNotFoundError:
|
||||
+ pass
|
||||
+ os.chflags(d1.name, flags)
|
||||
+ d1.cleanup()
|
||||
+ self.assertFalse(os.path.exists(d1.name))
|
||||
+
|
||||
+ with self.subTest('nonexisting file'):
|
||||
+ test('nonexisting', target_is_directory=False)
|
||||
+ with self.subTest('nonexisting dir'):
|
||||
+ test('nonexisting', target_is_directory=True)
|
||||
+
|
||||
+ with self.subTest('existing file'):
|
||||
+ os.chflags(file1, flags)
|
||||
+ old_flags = os.stat(file1).st_flags
|
||||
+ test(file1, target_is_directory=False)
|
||||
+ new_flags = os.stat(file1).st_flags
|
||||
+ self.assertEqual(new_flags, old_flags)
|
||||
+
|
||||
+ with self.subTest('existing dir'):
|
||||
+ os.chflags(dir1, flags)
|
||||
+ old_flags = os.stat(dir1).st_flags
|
||||
+ test(dir1, target_is_directory=True)
|
||||
+ new_flags = os.stat(dir1).st_flags
|
||||
+ self.assertEqual(new_flags, old_flags)
|
||||
+
|
||||
@support.cpython_only
|
||||
def test_del_on_collection(self):
|
||||
# A TemporaryDirectory is deleted when garbage collected
|
||||
@@ -1385,7 +1488,7 @@ class TestTemporaryDirectory(BaseTestCase):
|
||||
|
||||
tmp2 = os.path.join(tmp.name, 'test_dir')
|
||||
os.mkdir(tmp2)
|
||||
- with open(os.path.join(tmp2, "test.txt"), "w") as f:
|
||||
+ with open(os.path.join(tmp2, "test0.txt"), "w") as f:
|
||||
f.write("Hello world!")
|
||||
|
||||
{mod}.tmp = tmp
|
||||
@@ -1453,6 +1556,51 @@ class TestTemporaryDirectory(BaseTestCase):
|
||||
self.assertEqual(name, d.name)
|
||||
self.assertFalse(os.path.exists(name))
|
||||
|
||||
+ def test_modes(self):
|
||||
+ for mode in range(8):
|
||||
+ mode <<= 6
|
||||
+ with self.subTest(mode=format(mode, '03o')):
|
||||
+ d = self.do_create(recurse=3, dirs=2, files=2)
|
||||
+ with d:
|
||||
+ # Change files and directories mode recursively.
|
||||
+ for root, dirs, files in os.walk(d.name, topdown=False):
|
||||
+ for name in files:
|
||||
+ os.chmod(os.path.join(root, name), mode)
|
||||
+ os.chmod(root, mode)
|
||||
+ d.cleanup()
|
||||
+ self.assertFalse(os.path.exists(d.name))
|
||||
+
|
||||
+ def check_flags(self, flags):
|
||||
+ # skip the test if these flags are not supported (ex: FreeBSD 13)
|
||||
+ filename = support.TESTFN
|
||||
+ try:
|
||||
+ open(filename, "w").close()
|
||||
+ try:
|
||||
+ os.chflags(filename, flags)
|
||||
+ except OSError as exc:
|
||||
+ # "OSError: [Errno 45] Operation not supported"
|
||||
+ self.skipTest(f"chflags() doesn't support flags "
|
||||
+ f"{flags:#b}: {exc}")
|
||||
+ else:
|
||||
+ os.chflags(filename, 0)
|
||||
+ finally:
|
||||
+ support.unlink(filename)
|
||||
+
|
||||
+ @unittest.skipUnless(hasattr(os, 'chflags'), 'requires os.lchflags')
|
||||
+ def test_flags(self):
|
||||
+ flags = stat.UF_IMMUTABLE | stat.UF_NOUNLINK
|
||||
+ self.check_flags(flags)
|
||||
+
|
||||
+ d = self.do_create(recurse=3, dirs=2, files=2)
|
||||
+ with d:
|
||||
+ # Change files and directories flags recursively.
|
||||
+ for root, dirs, files in os.walk(d.name, topdown=False):
|
||||
+ for name in files:
|
||||
+ os.chflags(os.path.join(root, name), flags)
|
||||
+ os.chflags(root, flags)
|
||||
+ d.cleanup()
|
||||
+ self.assertFalse(os.path.exists(d.name))
|
||||
+
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
--
|
||||
2.44.0
|
||||
|
|
@ -0,0 +1,346 @@
|
|||
From 066df4fd454d6ff9be66e80b2a65995b10af174f Mon Sep 17 00:00:00 2001
|
||||
From: John Jolly <john.jolly@gmail.com>
|
||||
Date: Tue, 30 Jan 2018 01:51:35 -0700
|
||||
Subject: [PATCH] bpo-22908: Add seek and tell functionality to ZipExtFile
|
||||
(GH-4966)
|
||||
|
||||
This allows for nested zip files, tar files within zip files, zip files within tar files, etc.
|
||||
|
||||
Contributed by: John Jolly
|
||||
---
|
||||
Doc/library/zipfile.rst | 6 +-
|
||||
Lib/test/test_zipfile.py | 34 ++++++++
|
||||
Lib/zipfile.py | 82 +++++++++++++++++++
|
||||
.../2017-12-21-22-00-11.bpo-22908.cVm89I.rst | 2 +
|
||||
4 files changed, 121 insertions(+), 3 deletions(-)
|
||||
create mode 100644 Misc/NEWS.d/next/Library/2017-12-21-22-00-11.bpo-22908.cVm89I.rst
|
||||
|
||||
diff --git a/Doc/library/zipfile.rst b/Doc/library/zipfile.rst
|
||||
index d58efe0b417516..7c9a8c80225491 100644
|
||||
--- a/Doc/library/zipfile.rst
|
||||
+++ b/Doc/library/zipfile.rst
|
||||
@@ -246,9 +246,9 @@ ZipFile Objects
|
||||
With *mode* ``'r'`` the file-like object
|
||||
(``ZipExtFile``) is read-only and provides the following methods:
|
||||
:meth:`~io.BufferedIOBase.read`, :meth:`~io.IOBase.readline`,
|
||||
- :meth:`~io.IOBase.readlines`, :meth:`__iter__`,
|
||||
- :meth:`~iterator.__next__`. These objects can operate independently of
|
||||
- the ZipFile.
|
||||
+ :meth:`~io.IOBase.readlines`, :meth:`~io.IOBase.seek`,
|
||||
+ :meth:`~io.IOBase.tell`, :meth:`__iter__`, :meth:`~iterator.__next__`.
|
||||
+ These objects can operate independently of the ZipFile.
|
||||
|
||||
With ``mode='w'``, a writable file handle is returned, which supports the
|
||||
:meth:`~io.BufferedIOBase.write` method. While a writable file handle is open,
|
||||
diff --git a/Lib/test/test_zipfile.py b/Lib/test/test_zipfile.py
|
||||
index 94db858a1517c4..61c3e349a69ef4 100644
|
||||
--- a/Lib/test/test_zipfile.py
|
||||
+++ b/Lib/test/test_zipfile.py
|
||||
@@ -1628,6 +1628,40 @@ def test_open_conflicting_handles(self):
|
||||
self.assertEqual(zipf.read('baz'), msg3)
|
||||
self.assertEqual(zipf.namelist(), ['foo', 'bar', 'baz'])
|
||||
|
||||
+ def test_seek_tell(self):
|
||||
+ # Test seek functionality
|
||||
+ txt = b"Where's Bruce?"
|
||||
+ bloc = txt.find(b"Bruce")
|
||||
+ # Check seek on a file
|
||||
+ with zipfile.ZipFile(TESTFN, "w") as zipf:
|
||||
+ zipf.writestr("foo.txt", txt)
|
||||
+ with zipfile.ZipFile(TESTFN, "r") as zipf:
|
||||
+ with zipf.open("foo.txt", "r") as fp:
|
||||
+ fp.seek(bloc, os.SEEK_SET)
|
||||
+ self.assertEqual(fp.tell(), bloc)
|
||||
+ fp.seek(-bloc, os.SEEK_CUR)
|
||||
+ self.assertEqual(fp.tell(), 0)
|
||||
+ fp.seek(bloc, os.SEEK_CUR)
|
||||
+ self.assertEqual(fp.tell(), bloc)
|
||||
+ self.assertEqual(fp.read(5), txt[bloc:bloc+5])
|
||||
+ fp.seek(0, os.SEEK_END)
|
||||
+ self.assertEqual(fp.tell(), len(txt))
|
||||
+ # Check seek on memory file
|
||||
+ data = io.BytesIO()
|
||||
+ with zipfile.ZipFile(data, mode="w") as zipf:
|
||||
+ zipf.writestr("foo.txt", txt)
|
||||
+ with zipfile.ZipFile(data, mode="r") as zipf:
|
||||
+ with zipf.open("foo.txt", "r") as fp:
|
||||
+ fp.seek(bloc, os.SEEK_SET)
|
||||
+ self.assertEqual(fp.tell(), bloc)
|
||||
+ fp.seek(-bloc, os.SEEK_CUR)
|
||||
+ self.assertEqual(fp.tell(), 0)
|
||||
+ fp.seek(bloc, os.SEEK_CUR)
|
||||
+ self.assertEqual(fp.tell(), bloc)
|
||||
+ self.assertEqual(fp.read(5), txt[bloc:bloc+5])
|
||||
+ fp.seek(0, os.SEEK_END)
|
||||
+ self.assertEqual(fp.tell(), len(txt))
|
||||
+
|
||||
def tearDown(self):
|
||||
unlink(TESTFN)
|
||||
unlink(TESTFN2)
|
||||
diff --git a/Lib/zipfile.py b/Lib/zipfile.py
|
||||
index f9db45f58a2bde..5df7b1bf75b9d9 100644
|
||||
--- a/Lib/zipfile.py
|
||||
+++ b/Lib/zipfile.py
|
||||
@@ -696,6 +696,18 @@ def __init__(self, file, pos, close, lock, writing):
|
||||
self._close = close
|
||||
self._lock = lock
|
||||
self._writing = writing
|
||||
+ self.seekable = file.seekable
|
||||
+ self.tell = file.tell
|
||||
+
|
||||
+ def seek(self, offset, whence=0):
|
||||
+ with self._lock:
|
||||
+ if self.writing():
|
||||
+ raise ValueError("Can't reposition in the ZIP file while "
|
||||
+ "there is an open writing handle on it. "
|
||||
+ "Close the writing handle before trying to read.")
|
||||
+ self._file.seek(self._pos)
|
||||
+ self._pos = self._file.tell()
|
||||
+ return self._pos
|
||||
|
||||
def read(self, n=-1):
|
||||
with self._lock:
|
||||
@@ -746,6 +758,9 @@ class ZipExtFile(io.BufferedIOBase):
|
||||
# Read from compressed files in 4k blocks.
|
||||
MIN_READ_SIZE = 4096
|
||||
|
||||
+ # Chunk size to read during seek
|
||||
+ MAX_SEEK_READ = 1 << 24
|
||||
+
|
||||
def __init__(self, fileobj, mode, zipinfo, decrypter=None,
|
||||
close_fileobj=False):
|
||||
self._fileobj = fileobj
|
||||
@@ -778,6 +793,17 @@ def __init__(self, fileobj, mode, zipinfo, decrypter=None,
|
||||
else:
|
||||
self._expected_crc = None
|
||||
|
||||
+ self._seekable = False
|
||||
+ try:
|
||||
+ if fileobj.seekable():
|
||||
+ self._orig_compress_start = fileobj.tell()
|
||||
+ self._orig_compress_size = zipinfo.compress_size
|
||||
+ self._orig_file_size = zipinfo.file_size
|
||||
+ self._orig_start_crc = self._running_crc
|
||||
+ self._seekable = True
|
||||
+ except AttributeError:
|
||||
+ pass
|
||||
+
|
||||
def __repr__(self):
|
||||
result = ['<%s.%s' % (self.__class__.__module__,
|
||||
self.__class__.__qualname__)]
|
||||
@@ -963,6 +989,62 @@ def close(self):
|
||||
finally:
|
||||
super().close()
|
||||
|
||||
+ def seekable(self):
|
||||
+ return self._seekable
|
||||
+
|
||||
+ def seek(self, offset, whence=0):
|
||||
+ if not self._seekable:
|
||||
+ raise io.UnsupportedOperation("underlying stream is not seekable")
|
||||
+ curr_pos = self.tell()
|
||||
+ if whence == 0: # Seek from start of file
|
||||
+ new_pos = offset
|
||||
+ elif whence == 1: # Seek from current position
|
||||
+ new_pos = curr_pos + offset
|
||||
+ elif whence == 2: # Seek from EOF
|
||||
+ new_pos = self._orig_file_size + offset
|
||||
+ else:
|
||||
+ raise ValueError("whence must be os.SEEK_SET (0), "
|
||||
+ "os.SEEK_CUR (1), or os.SEEK_END (2)")
|
||||
+
|
||||
+ if new_pos > self._orig_file_size:
|
||||
+ new_pos = self._orig_file_size
|
||||
+
|
||||
+ if new_pos < 0:
|
||||
+ new_pos = 0
|
||||
+
|
||||
+ read_offset = new_pos - curr_pos
|
||||
+ buff_offset = read_offset + self._offset
|
||||
+
|
||||
+ if buff_offset >= 0 and buff_offset < len(self._readbuffer):
|
||||
+ # Just move the _offset index if the new position is in the _readbuffer
|
||||
+ self._offset = buff_offset
|
||||
+ read_offset = 0
|
||||
+ elif read_offset < 0:
|
||||
+ # Position is before the current position. Reset the ZipExtFile
|
||||
+
|
||||
+ self._fileobj.seek(self._orig_compress_start)
|
||||
+ self._running_crc = self._orig_start_crc
|
||||
+ self._compress_left = self._orig_compress_size
|
||||
+ self._left = self._orig_file_size
|
||||
+ self._readbuffer = b''
|
||||
+ self._offset = 0
|
||||
+ self._decompressor = zipfile._get_decompressor(self._compress_type)
|
||||
+ self._eof = False
|
||||
+ read_offset = new_pos
|
||||
+
|
||||
+ while read_offset > 0:
|
||||
+ read_len = min(self.MAX_SEEK_READ, read_offset)
|
||||
+ self.read(read_len)
|
||||
+ read_offset -= read_len
|
||||
+
|
||||
+ return self.tell()
|
||||
+
|
||||
+ def tell(self):
|
||||
+ if not self._seekable:
|
||||
+ raise io.UnsupportedOperation("underlying stream is not seekable")
|
||||
+ filepos = self._orig_file_size - self._left - len(self._readbuffer) + self._offset
|
||||
+ return filepos
|
||||
+
|
||||
|
||||
class _ZipWriteFile(io.BufferedIOBase):
|
||||
def __init__(self, zf, zinfo, zip64):
|
||||
diff --git a/Misc/NEWS.d/next/Library/2017-12-21-22-00-11.bpo-22908.cVm89I.rst b/Misc/NEWS.d/next/Library/2017-12-21-22-00-11.bpo-22908.cVm89I.rst
|
||||
new file mode 100644
|
||||
index 00000000000000..4f3cc0166019f1
|
||||
--- /dev/null
|
||||
+++ b/Misc/NEWS.d/next/Library/2017-12-21-22-00-11.bpo-22908.cVm89I.rst
|
||||
@@ -0,0 +1,2 @@
|
||||
+Added seek and tell to the ZipExtFile class. This only works if the file
|
||||
+object used to open the zipfile is seekable.
|
||||
|
||||
|
||||
From 55beb125db2942b5362454e05542e9661e964a65 Mon Sep 17 00:00:00 2001
|
||||
From: Serhiy Storchaka <storchaka@gmail.com>
|
||||
Date: Tue, 23 Apr 2024 14:29:31 +0200
|
||||
Subject: [PATCH] gh-109858: Protect zipfile from "quoted-overlap" zipbomb
|
||||
(GH-110016) (GH-113916)
|
||||
|
||||
Raise BadZipFile when try to read an entry that overlaps with other entry or
|
||||
central directory.
|
||||
(cherry picked from commit 66363b9a7b9fe7c99eba3a185b74c5fdbf842eba)
|
||||
---
|
||||
Lib/test/test_zipfile.py | 60 +++++++++++++++++++
|
||||
Lib/zipfile.py | 12 ++++
|
||||
...-09-28-13-15-51.gh-issue-109858.43e2dg.rst | 3 +
|
||||
3 files changed, 75 insertions(+)
|
||||
create mode 100644 Misc/NEWS.d/next/Library/2023-09-28-13-15-51.gh-issue-109858.43e2dg.rst
|
||||
|
||||
diff --git a/Lib/test/test_zipfile.py b/Lib/test/test_zipfile.py
|
||||
index 7f82586..0379909 100644
|
||||
--- a/Lib/test/test_zipfile.py
|
||||
+++ b/Lib/test/test_zipfile.py
|
||||
@@ -1644,6 +1644,66 @@ class OtherTests(unittest.TestCase):
|
||||
fp.seek(0, os.SEEK_END)
|
||||
self.assertEqual(fp.tell(), len(txt))
|
||||
|
||||
+ @requires_zlib
|
||||
+ def test_full_overlap(self):
|
||||
+ data = (
|
||||
+ b'PK\x03\x04\x14\x00\x00\x00\x08\x00\xa0lH\x05\xe2\x1e'
|
||||
+ b'8\xbb\x10\x00\x00\x00\t\x04\x00\x00\x01\x00\x00\x00a\xed'
|
||||
+ b'\xc0\x81\x08\x00\x00\x00\xc00\xd6\xfbK\\d\x0b`P'
|
||||
+ b'K\x01\x02\x14\x00\x14\x00\x00\x00\x08\x00\xa0lH\x05\xe2'
|
||||
+ b'\x1e8\xbb\x10\x00\x00\x00\t\x04\x00\x00\x01\x00\x00\x00\x00'
|
||||
+ b'\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00aPK'
|
||||
+ b'\x01\x02\x14\x00\x14\x00\x00\x00\x08\x00\xa0lH\x05\xe2\x1e'
|
||||
+ b'8\xbb\x10\x00\x00\x00\t\x04\x00\x00\x01\x00\x00\x00\x00\x00'
|
||||
+ b'\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00bPK\x05'
|
||||
+ b'\x06\x00\x00\x00\x00\x02\x00\x02\x00^\x00\x00\x00/\x00\x00'
|
||||
+ b'\x00\x00\x00'
|
||||
+ )
|
||||
+ with zipfile.ZipFile(io.BytesIO(data), 'r') as zipf:
|
||||
+ self.assertEqual(zipf.namelist(), ['a', 'b'])
|
||||
+ zi = zipf.getinfo('a')
|
||||
+ self.assertEqual(zi.header_offset, 0)
|
||||
+ self.assertEqual(zi.compress_size, 16)
|
||||
+ self.assertEqual(zi.file_size, 1033)
|
||||
+ zi = zipf.getinfo('b')
|
||||
+ self.assertEqual(zi.header_offset, 0)
|
||||
+ self.assertEqual(zi.compress_size, 16)
|
||||
+ self.assertEqual(zi.file_size, 1033)
|
||||
+ self.assertEqual(len(zipf.read('a')), 1033)
|
||||
+ with self.assertRaisesRegex(zipfile.BadZipFile, 'File name.*differ'):
|
||||
+ zipf.read('b')
|
||||
+
|
||||
+ @requires_zlib
|
||||
+ def test_quoted_overlap(self):
|
||||
+ data = (
|
||||
+ b'PK\x03\x04\x14\x00\x00\x00\x08\x00\xa0lH\x05Y\xfc'
|
||||
+ b'8\x044\x00\x00\x00(\x04\x00\x00\x01\x00\x00\x00a\x00'
|
||||
+ b'\x1f\x00\xe0\xffPK\x03\x04\x14\x00\x00\x00\x08\x00\xa0l'
|
||||
+ b'H\x05\xe2\x1e8\xbb\x10\x00\x00\x00\t\x04\x00\x00\x01\x00'
|
||||
+ b'\x00\x00b\xed\xc0\x81\x08\x00\x00\x00\xc00\xd6\xfbK\\'
|
||||
+ b'd\x0b`PK\x01\x02\x14\x00\x14\x00\x00\x00\x08\x00\xa0'
|
||||
+ b'lH\x05Y\xfc8\x044\x00\x00\x00(\x04\x00\x00\x01'
|
||||
+ b'\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00'
|
||||
+ b'\x00aPK\x01\x02\x14\x00\x14\x00\x00\x00\x08\x00\xa0l'
|
||||
+ b'H\x05\xe2\x1e8\xbb\x10\x00\x00\x00\t\x04\x00\x00\x01\x00'
|
||||
+ b'\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00$\x00\x00\x00'
|
||||
+ b'bPK\x05\x06\x00\x00\x00\x00\x02\x00\x02\x00^\x00\x00'
|
||||
+ b'\x00S\x00\x00\x00\x00\x00'
|
||||
+ )
|
||||
+ with zipfile.ZipFile(io.BytesIO(data), 'r') as zipf:
|
||||
+ self.assertEqual(zipf.namelist(), ['a', 'b'])
|
||||
+ zi = zipf.getinfo('a')
|
||||
+ self.assertEqual(zi.header_offset, 0)
|
||||
+ self.assertEqual(zi.compress_size, 52)
|
||||
+ self.assertEqual(zi.file_size, 1064)
|
||||
+ zi = zipf.getinfo('b')
|
||||
+ self.assertEqual(zi.header_offset, 36)
|
||||
+ self.assertEqual(zi.compress_size, 16)
|
||||
+ self.assertEqual(zi.file_size, 1033)
|
||||
+ with self.assertRaisesRegex(zipfile.BadZipFile, 'Overlapped entries'):
|
||||
+ zipf.read('a')
|
||||
+ self.assertEqual(len(zipf.read('b')), 1033)
|
||||
+
|
||||
def tearDown(self):
|
||||
unlink(TESTFN)
|
||||
unlink(TESTFN2)
|
||||
diff --git a/Lib/zipfile.py b/Lib/zipfile.py
|
||||
index 0ab9fac..e6d7676 100644
|
||||
--- a/Lib/zipfile.py
|
||||
+++ b/Lib/zipfile.py
|
||||
@@ -338,6 +338,7 @@ class ZipInfo (object):
|
||||
'compress_size',
|
||||
'file_size',
|
||||
'_raw_time',
|
||||
+ '_end_offset',
|
||||
)
|
||||
|
||||
def __init__(self, filename="NoName", date_time=(1980,1,1,0,0,0)):
|
||||
@@ -376,6 +377,7 @@ class ZipInfo (object):
|
||||
self.volume = 0 # Volume number of file header
|
||||
self.internal_attr = 0 # Internal attributes
|
||||
self.external_attr = 0 # External file attributes
|
||||
+ self._end_offset = None # Start of the next local header or central directory
|
||||
# Other attributes are set by class ZipFile:
|
||||
# header_offset Byte offset to the file header
|
||||
# CRC CRC-32 of the uncompressed file
|
||||
@@ -1346,6 +1348,12 @@ class ZipFile:
|
||||
if self.debug > 2:
|
||||
print("total", total)
|
||||
|
||||
+ end_offset = self.start_dir
|
||||
+ for zinfo in sorted(self.filelist,
|
||||
+ key=lambda zinfo: zinfo.header_offset,
|
||||
+ reverse=True):
|
||||
+ zinfo._end_offset = end_offset
|
||||
+ end_offset = zinfo.header_offset
|
||||
|
||||
def namelist(self):
|
||||
"""Return a list of file names in the archive."""
|
||||
@@ -1500,6 +1508,10 @@ class ZipFile:
|
||||
'File name in directory %r and header %r differ.'
|
||||
% (zinfo.orig_filename, fname))
|
||||
|
||||
+ if (zinfo._end_offset is not None and
|
||||
+ zef_file.tell() + zinfo.compress_size > zinfo._end_offset):
|
||||
+ raise BadZipFile(f"Overlapped entries: {zinfo.orig_filename!r} (possible zip bomb)")
|
||||
+
|
||||
# check for encrypted flag & handle password
|
||||
is_encrypted = zinfo.flag_bits & 0x1
|
||||
zd = None
|
||||
diff --git a/Misc/NEWS.d/next/Library/2023-09-28-13-15-51.gh-issue-109858.43e2dg.rst b/Misc/NEWS.d/next/Library/2023-09-28-13-15-51.gh-issue-109858.43e2dg.rst
|
||||
new file mode 100644
|
||||
index 0000000..be279ca
|
||||
--- /dev/null
|
||||
+++ b/Misc/NEWS.d/next/Library/2023-09-28-13-15-51.gh-issue-109858.43e2dg.rst
|
||||
@@ -0,0 +1,3 @@
|
||||
+Protect :mod:`zipfile` from "quoted-overlap" zipbomb. It now raises
|
||||
+BadZipFile when try to read an entry that overlaps with other entry or
|
||||
+central directory.
|
||||
--
|
||||
2.44.0
|
||||
|
|
@ -14,7 +14,7 @@ URL: https://www.python.org/
|
|||
# WARNING When rebasing to a new Python version,
|
||||
# remember to update the python3-docs package as well
|
||||
Version: %{pybasever}.8
|
||||
Release: 31%{?dist}
|
||||
Release: 62%{?dist}
|
||||
License: Python
|
||||
|
||||
|
||||
|
@ -56,6 +56,15 @@ License: Python
|
|||
%bcond_with valgrind
|
||||
%endif
|
||||
|
||||
# https://fedoraproject.org/wiki/Changes/Python_Upstream_Architecture_Names
|
||||
# For a very long time we have converted "upstream architecture names" to "Fedora names".
|
||||
# This made sense at the time, see https://github.com/pypa/manylinux/issues/687#issuecomment-666362947
|
||||
# However, with manylinux wheels popularity growth, this is now a problem.
|
||||
# Wheels built on a Linux that doesn't do this were not compatible with ours and vice versa.
|
||||
# We now have a compatibility layer to workaround a problem,
|
||||
# but we also no longer use the legacy arch names in Fedora 34+.
|
||||
# This bcond controls the behavior. The defaults should be good for anybody.
|
||||
%bcond_without legacy_archnames
|
||||
|
||||
# ==================================
|
||||
# Notes from bootstraping Python 3.6
|
||||
|
@ -111,8 +120,21 @@ License: Python
|
|||
%global LDVERSION_optimized %{pybasever}%{ABIFLAGS_optimized}
|
||||
%global LDVERSION_debug %{pybasever}%{ABIFLAGS_debug}
|
||||
|
||||
%global SOABI_optimized cpython-%{pyshortver}%{ABIFLAGS_optimized}-%{_arch}-linux%{_gnu}
|
||||
%global SOABI_debug cpython-%{pyshortver}%{ABIFLAGS_debug}-%{_arch}-linux%{_gnu}
|
||||
# When we use the upstream arch triplets, we convert them from the legacy ones
|
||||
# This is reversed in prep when %%with legacy_archnames, so we keep both macros
|
||||
%global platform_triplet_legacy %{_arch}-linux%{_gnu}
|
||||
%global platform_triplet_upstream %{expand:%(echo %{platform_triplet_legacy} | sed -E \\
|
||||
-e 's/^arm(eb)?-linux-gnueabi$/arm\\1-linux-gnueabihf/' \\
|
||||
-e 's/^mips64(el)?-linux-gnu$/mips64\\1-linux-gnuabi64/' \\
|
||||
-e 's/^ppc(64)?(le)?-linux-gnu$/powerpc\\1\\2-linux-gnu/')}
|
||||
%if %{with legacy_archnames}
|
||||
%global platform_triplet %{platform_triplet_legacy}
|
||||
%else
|
||||
%global platform_triplet %{platform_triplet_upstream}
|
||||
%endif
|
||||
|
||||
%global SOABI_optimized cpython-%{pyshortver}%{ABIFLAGS_optimized}-%{platform_triplet}
|
||||
%global SOABI_debug cpython-%{pyshortver}%{ABIFLAGS_debug}-%{platform_triplet}
|
||||
|
||||
# All bytecode files are in a __pycache__ subdirectory, with a name
|
||||
# reflecting the version of the bytecode.
|
||||
|
@ -322,6 +344,16 @@ Patch189: 00189-use-rpm-wheels.patch
|
|||
# Fedora Change: https://fedoraproject.org/wiki/Changes/Making_sudo_pip_safe
|
||||
Patch251: 00251-change-user-install-location.patch
|
||||
|
||||
# 00257 #
|
||||
# Use the monotonic clock for threading.Condition.wait() as to not be affected by
|
||||
# the system clock changes.
|
||||
# This patch works around the issue.
|
||||
# Implemented by backporting the respective python2 code:
|
||||
# https://github.com/python/cpython/blob/v2.7.18/Lib/threading.py#L331
|
||||
# along with our downstream patch for python2 fixing the same issue.
|
||||
# Downstream only.
|
||||
Patch257: 00257-threading-condition-wait.patch
|
||||
|
||||
# 00262 #
|
||||
# Backport of PEP 538: Coercing the legacy C locale to a UTF-8 based locale
|
||||
# https://www.python.org/dev/peps/pep-0538/
|
||||
|
@ -329,10 +361,6 @@ Patch251: 00251-change-user-install-location.patch
|
|||
# Original proposal: https://bugzilla.redhat.com/show_bug.cgi?id=1404918
|
||||
Patch262: 00262-pep538_coerce_legacy_c_locale.patch
|
||||
|
||||
# 00274 #
|
||||
# Upstream uses Debian-style architecture naming. Change to match Fedora.
|
||||
Patch274: 00274-fix-arch-names.patch
|
||||
|
||||
# 00294 #
|
||||
# Define TLS cipher suite on build time depending
|
||||
# on the OpenSSL default cipher suite selection.
|
||||
|
@ -519,6 +547,333 @@ Patch351: 00351-avoid-infinite-loop-in-the-tarfile-module.patch
|
|||
# Fixed upstream: https://bugs.python.org/issue41004
|
||||
Patch352: 00352-resolve-hash-collisions-for-ipv4interface-and-ipv6interface.patch
|
||||
|
||||
# 00353 #
|
||||
# Original names for architectures with different names downstream
|
||||
#
|
||||
# https://fedoraproject.org/wiki/Changes/Python_Upstream_Architecture_Names
|
||||
#
|
||||
# Pythons in RHEL/Fedora used different names for some architectures
|
||||
# than upstream and other distros (for example ppc64 vs. powerpc64).
|
||||
# This was patched in patch 274, now it is sedded if %%with legacy_archnames.
|
||||
#
|
||||
# That meant that an extension built with the default upstream settings
|
||||
# (on other distro or as an manylinux wheel) could not been found by Python
|
||||
# on RHEL/Fedora because it had a different suffix.
|
||||
# This patch adds the legacy names to importlib so Python is able
|
||||
# to import extensions with a legacy architecture name in its
|
||||
# file name.
|
||||
# It work both ways, so it support both %%with and %%without legacy_archnames.
|
||||
#
|
||||
# WARNING: This patch has no effect on Python built with bootstrap
|
||||
# enabled because Python/importlib_external.h is not regenerated
|
||||
# and therefore Python during bootstrap contains importlib from
|
||||
# upstream without this feature. It's possible to include
|
||||
# Python/importlib_external.h to this patch but it'd make rebasing
|
||||
# a nightmare because it's basically a binary file.
|
||||
Patch353: 00353-architecture-names-upstream-downstream.patch
|
||||
|
||||
# 00354 #
|
||||
# Reject control chars in HTTP method in http.client to prevent
|
||||
# HTTP header injection
|
||||
# Fixed ustream: https://bugs.python.org/issue39603
|
||||
Patch354: 00354-cve-2020-26116-http-request-method-crlf-injection-in-httplib.patch
|
||||
|
||||
# 00355 #
|
||||
# No longer call eval() on content received via HTTP in the CJK codec tests
|
||||
# Fixed upstream: https://bugs.python.org/issue41944
|
||||
Patch355: 00355-CVE-2020-27619.patch
|
||||
|
||||
# 00356 #
|
||||
# options -a and -k for pathfix.py used in %%py3_shebang_fix
|
||||
# Upstream: https://github.com/python/cpython/commit/c71c54c62600fd721baed3c96709e3d6e9c33817
|
||||
Patch356: 00356-k_and_a_options_for_pathfix.patch
|
||||
|
||||
# 00357 #
|
||||
# CVE-2021-3177 stack-based buffer overflow in PyCArg_repr in _ctypes/callproc.c
|
||||
# Upstream: https://bugs.python.org/issue42938
|
||||
# Main BZ: https://bugzilla.redhat.com/show_bug.cgi?id=1918168
|
||||
Patch357: 00357-CVE-2021-3177.patch
|
||||
|
||||
# 00359 #
|
||||
# CVE-2021-23336 python: Web Cache Poisoning via urllib.parse.parse_qsl and
|
||||
# urllib.parse.parse_qs by using a semicolon in query parameters
|
||||
# Upstream: https://bugs.python.org/issue42967
|
||||
# Main BZ: https://bugzilla.redhat.com/show_bug.cgi?id=1928904
|
||||
Patch359: 00359-CVE-2021-23336.patch
|
||||
|
||||
# 00360 #
|
||||
# CVE-2021-3426: information disclosure via pydoc
|
||||
# Upstream: https://bugs.python.org/issue42988
|
||||
# Main BZ: https://bugzilla.redhat.com/show_bug.cgi?id=1935913
|
||||
Patch360: 00360-CVE-2021-3426.patch
|
||||
|
||||
# 00362 #
|
||||
# The threading.enumerate() function now uses a reentrant lock to
|
||||
# prevent a hang on reentrant call.
|
||||
# Upstream: https://bugs.python.org/issue44422
|
||||
# Main BZ: https://bugzilla.redhat.com/show_bug.cgi?id=1959459
|
||||
Patch362: 00362-threading-enumerate-rlock.patch
|
||||
|
||||
# 00364 #
|
||||
# Don't call PyThread_exit_thread() explicitly.
|
||||
# Upstream: https://bugs.python.org/issue44434
|
||||
# Main BZ: https://bugzilla.redhat.com/show_bug.cgi?id=1972293
|
||||
Patch364: 00364-thread-exit.patch
|
||||
|
||||
# 00366 #
|
||||
# CVE-2021-3733: Denial of service when identifying crafted invalid RFCs
|
||||
# Upstream: https://bugs.python.org/issue43075
|
||||
# Tracking bug: https://bugzilla.redhat.com/show_bug.cgi?id=1995234
|
||||
Patch366: 00366-CVE-2021-3733.patch
|
||||
|
||||
# 00368 #
|
||||
# CVE-2021-3737: client can enter an infinite loop on a 100 Continue response from the server
|
||||
# Upstream: https://bugs.python.org/issue44022
|
||||
# Tracking bug: https://bugzilla.redhat.com/show_bug.cgi?id=1995162
|
||||
Patch368: 00368-CVE-2021-3737.patch
|
||||
|
||||
# 00369 #
|
||||
# Change shouldRollover() methods of logging.handlers to only rollover regular files and not devices
|
||||
# Upstream: https://bugs.python.org/issue45401
|
||||
# Bugzilla: https://bugzilla.redhat.com/show_bug.cgi?id=2009200
|
||||
Patch369: 00369-rollover-only-regular-files-in-logging-handlers.patch
|
||||
|
||||
# 00370 #
|
||||
# Utilize the monotonic clock for the global interpreter lock instead of the real-time clock
|
||||
# to avoid issues when the system time changes
|
||||
# Upstream: https://bugs.python.org/issue12822
|
||||
# Bugzilla: https://bugzilla.redhat.com/show_bug.cgi?id=2003758
|
||||
Patch370: 00370-GIL-monotonic-clock.patch
|
||||
|
||||
# 00372 #
|
||||
# CVE-2021-4189: ftplib should not use the host from the PASV response
|
||||
# Upstream: https://bugs.python.org/issue43285
|
||||
# Tracking bug: https://bugzilla.redhat.com/show_bug.cgi?id=2036020
|
||||
Patch372: 00372-CVE-2021-4189.patch
|
||||
|
||||
# 00377 #
|
||||
# CVE-2022-0391: urlparse does not sanitize URLs containing ASCII newline and tabs
|
||||
#
|
||||
# ASCII newline and tab characters are stripped from the URL.
|
||||
#
|
||||
# Upstream: https://bugs.python.org/issue43882
|
||||
# Tracking bug: https://bugzilla.redhat.com/show_bug.cgi?id=2047376
|
||||
Patch377: 00377-CVE-2022-0391.patch
|
||||
|
||||
# 00378 #
|
||||
# Support expat 2.4.5
|
||||
#
|
||||
# Curly brackets were never allowed in namespace URIs
|
||||
# according to RFC 3986, and so-called namespace-validating
|
||||
# XML parsers have the right to reject them a invalid URIs.
|
||||
#
|
||||
# libexpat >=2.4.5 has become strcter in that regard due to
|
||||
# related security issues; with ET.XML instantiating a
|
||||
# namespace-aware parser under the hood, this test has no
|
||||
# future in CPython.
|
||||
#
|
||||
# References:
|
||||
# - https://datatracker.ietf.org/doc/html/rfc3968
|
||||
# - https://www.w3.org/TR/xml-names/
|
||||
#
|
||||
# Also, test_minidom.py: Support Expat >=2.4.5
|
||||
#
|
||||
# The patch has diverged from upstream as the python test
|
||||
# suite was relying on checking the expat version, whereas
|
||||
# in RHEL fixes get backported instead of rebasing packages.
|
||||
#
|
||||
# Upstream: https://bugs.python.org/issue46811
|
||||
Patch378: 00378-support-expat-2-4-5.patch
|
||||
|
||||
# 00382 #
|
||||
# CVE-2015-20107
|
||||
#
|
||||
# Make mailcap refuse to match unsafe filenames/types/params (GH-91993)
|
||||
#
|
||||
# Upstream: https://github.com/python/cpython/issues/68966
|
||||
#
|
||||
# Tracker bug: https://bugzilla.redhat.com/show_bug.cgi?id=2075390
|
||||
Patch382: 00382-cve-2015-20107.patch
|
||||
|
||||
# 00386 #
|
||||
# CVE-2021-28861
|
||||
#
|
||||
# Fix an open redirection vulnerability in the `http.server` module when
|
||||
# an URI path starts with `//` that could produce a 301 Location header
|
||||
# with a misleading target. Vulnerability discovered, and logic fix
|
||||
# proposed, by Hamza Avvan (@hamzaavvan).
|
||||
#
|
||||
# Test and comments authored by Gregory P. Smith [Google].
|
||||
#
|
||||
# Upstream: https://github.com/python/cpython/pull/93879
|
||||
# Tracking bugzilla: https://bugzilla.redhat.com/show_bug.cgi?id=2120642
|
||||
Patch386: 00386-cve-2021-28861.patch
|
||||
|
||||
# 00387 #
|
||||
# CVE-2020-10735: Prevent DoS by very large int()
|
||||
#
|
||||
# gh-95778: CVE-2020-10735: Prevent DoS by very large int() (GH-96504)
|
||||
#
|
||||
# Converting between `int` and `str` in bases other than 2
|
||||
# (binary), 4, 8 (octal), 16 (hexadecimal), or 32 such as base 10 (decimal) now
|
||||
# raises a `ValueError` if the number of digits in string form is above a
|
||||
# limit to avoid potential denial of service attacks due to the algorithmic
|
||||
# complexity. This is a mitigation for CVE-2020-10735
|
||||
# (https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2020-10735).
|
||||
#
|
||||
# This new limit can be configured or disabled by environment variable, command
|
||||
# line flag, or :mod:`sys` APIs. See the `Integer String Conversion Length
|
||||
# Limitation` documentation. The default limit is 4300
|
||||
# digits in string form.
|
||||
#
|
||||
# Patch by Gregory P. Smith [Google] and Christian Heimes [Red Hat] with feedback
|
||||
# from Victor Stinner, Thomas Wouters, Steve Dower, Ned Deily, and Mark Dickinson.
|
||||
#
|
||||
# Notes on the backport to Python 3.6 in RHEL:
|
||||
#
|
||||
# * Use "Python 3.6.8-48" version in the documentation, whereas this
|
||||
# version will never be released
|
||||
# * Only add _Py_global_config_int_max_str_digits global variable:
|
||||
# Python 3.6 doesn't have PyConfig API (PEP 597) nor _PyRuntime.
|
||||
# * sys.flags.int_max_str_digits cannot be -1 on Python 3.6: it is
|
||||
# set to the default limit. Adapt test_int_max_str_digits() for that.
|
||||
# * Declare _PY_LONG_DEFAULT_MAX_STR_DIGITS and
|
||||
# _PY_LONG_MAX_STR_DIGITS_THRESHOLD macros in longobject.h but only
|
||||
# if the Py_BUILD_CORE macro is defined.
|
||||
# * Declare _Py_global_config_int_max_str_digits in pydebug.h.
|
||||
#
|
||||
#
|
||||
# gh-95778: Mention sys.set_int_max_str_digits() in error message (#96874)
|
||||
#
|
||||
# When ValueError is raised if an integer is larger than the limit,
|
||||
# mention sys.set_int_max_str_digits() in the error message.
|
||||
#
|
||||
#
|
||||
# gh-96848: Fix -X int_max_str_digits option parsing (#96988)
|
||||
#
|
||||
# Fix command line parsing: reject "-X int_max_str_digits" option with
|
||||
# no value (invalid) when the PYTHONINTMAXSTRDIGITS environment
|
||||
# variable is set to a valid limit.
|
||||
Patch387: 00387-cve-2020-10735-prevent-dos-by-very-large-int.patch
|
||||
|
||||
# 00394 #
|
||||
# CVE-2022-45061: CPU denial of service via inefficient IDNA decoder
|
||||
#
|
||||
# gh-98433: Fix quadratic time idna decoding.
|
||||
#
|
||||
# There was an unnecessary quadratic loop in idna decoding. This restores
|
||||
# the behavior to linear.
|
||||
Patch394: 00394-cve-2022-45061-cpu-denial-of-service-via-inefficient-idna-decoder.patch
|
||||
|
||||
# 00397 #
|
||||
# Add filters for tarfile extraction (CVE-2007-4559, PEP-706)
|
||||
# The first patches in the file backport the upstream fix:
|
||||
# - https://github.com/python/cpython/pull/104583
|
||||
# (see the linked issue for merged backports)
|
||||
# Next-to-last patch fixes determination of symlink targets, which were treated
|
||||
# as relative to the root of the archive,
|
||||
# rather than the directory containing the symlink.
|
||||
# Not yet upstream as of this writing.
|
||||
# The last patch is Red Hat configuration, see KB for documentation:
|
||||
# - https://access.redhat.com/articles/7004769
|
||||
Patch397: 00397-tarfile-filter.patch
|
||||
|
||||
# 00399 #
|
||||
# CVE-2023-24329
|
||||
#
|
||||
# gh-102153: Start stripping C0 control and space chars in `urlsplit` (GH-102508)
|
||||
#
|
||||
# `urllib.parse.urlsplit` has already been respecting the WHATWG spec a bit GH-25595.
|
||||
#
|
||||
# This adds more sanitizing to respect the "Remove any leading C0 control or space from input" [rule](https://url.spec.whatwg.org/GH-url-parsing:~:text=Remove%%20any%%20leading%%20and%%20trailing%%20C0%%20control%%20or%%20space%%20from%%20input.) in response to [CVE-2023-24329](https://nvd.nist.gov/vuln/detail/CVE-2023-24329).
|
||||
#
|
||||
# Backported from Python 3.12
|
||||
Patch399: 00399-cve-2023-24329.patch
|
||||
|
||||
# 00404 #
|
||||
# CVE-2023-40217
|
||||
#
|
||||
# Security fix for CVE-2023-40217: Bypass TLS handshake on closed sockets
|
||||
# Resolved upstream: https://github.com/python/cpython/issues/108310
|
||||
# Fixups added on top:
|
||||
# https://github.com/python/cpython/pull/108352
|
||||
# https://github.com/python/cpython/pull/108408
|
||||
#
|
||||
# Backported from Python 3.8
|
||||
Patch404: 00404-cve-2023-40217.patch
|
||||
|
||||
# 00408 #
|
||||
# CVE-2022-48560
|
||||
#
|
||||
# Security fix for CVE-2022-48560: python3: use after free in heappushpop()
|
||||
# of heapq module
|
||||
# Resolved upstream: https://github.com/python/cpython/issues/83602
|
||||
Patch408: 00408-CVE-2022-48560.patch
|
||||
|
||||
# 00413 #
|
||||
# CVE-2022-48564
|
||||
#
|
||||
# DoS when processing malformed Apple Property List files in binary format
|
||||
# Resolved upstream: https://github.com/python/cpython/commit/a63234c49b2fbfb6f0aca32525e525ce3d43b2b4
|
||||
Patch413: 00413-CVE-2022-48564.patch
|
||||
|
||||
# 00414 #
|
||||
#
|
||||
# Skip test_pair() and test_speech128() of test_zlib on s390x since
|
||||
# they fail if zlib uses the s390x hardware accelerator.
|
||||
Patch414: 00414-skip_test_zlib_s390x.patch
|
||||
|
||||
# 00415 #
|
||||
# [CVE-2023-27043] gh-102988: Reject malformed addresses in email.parseaddr() (#111116)
|
||||
#
|
||||
# Detect email address parsing errors and return empty tuple to
|
||||
# indicate the parsing error (old API). Add an optional 'strict'
|
||||
# parameter to getaddresses() and parseaddr() functions. Patch by
|
||||
# Thomas Dwyer.
|
||||
#
|
||||
# Upstream PR: https://github.com/python/cpython/pull/111116
|
||||
#
|
||||
# Second patch implmenets the possibility to restore the old behavior via
|
||||
# config file or environment variable.
|
||||
Patch415: 00415-cve-2023-27043-gh-102988-reject-malformed-addresses-in-email-parseaddr-111116.patch
|
||||
|
||||
# 00422 #
|
||||
# gh-115133: Fix tests for XMLPullParser with Expat 2.6.0
|
||||
#
|
||||
# Feeding the parser by too small chunks defers parsing to prevent
|
||||
# CVE-2023-52425. Future versions of Expat may be more reactive.
|
||||
#
|
||||
# Patch rebased because the CVE fix is backported to older expat in RHEL.
|
||||
Patch422: 00422-gh-115133-fix-tests-for-xmlpullparser-with-expat-2-6-0.patch
|
||||
|
||||
# 426 #
|
||||
# CVE-2023-6597
|
||||
#
|
||||
# Path traversal on tempfile.TemporaryDirectory
|
||||
#
|
||||
# Upstream: https://github.com/python/cpython/issues/91133
|
||||
# Tracking bug: https://bugzilla.redhat.com/show_bug.cgi?id=CVE-2023-6597
|
||||
#
|
||||
# To backport the fix cleanly the patch contains also this rebased commit:
|
||||
# Fix permission errors in TemporaryDirectory cleanup
|
||||
# https://github.com/python/cpython/commit/e9b51c0ad81da1da11ae65840ac8b50a8521373c
|
||||
Patch426: 00426-CVE-2023-6597.patch
|
||||
|
||||
# 427 #
|
||||
# CVE-2024-0450
|
||||
#
|
||||
# The zipfile module is vulnerable to zip-bombs leading to denial of service.
|
||||
#
|
||||
# Upstream: https://github.com/python/cpython/issues/109858
|
||||
# Tracking bug: https://bugzilla.redhat.com/show_bug.cgi?id=CVE-2024-0450
|
||||
#
|
||||
# To backport the fix cleanly also this change is backported:
|
||||
# Add seek and tell functionality to ZipExtFile
|
||||
# https://github.com/python/cpython/commit/066df4fd454d6ff9be66e80b2a65995b10af174f
|
||||
#
|
||||
# Patch rebased from 3.8.
|
||||
Patch427: 00427-CVE-2024-0450.patch
|
||||
|
||||
# (New patches go here ^^^)
|
||||
#
|
||||
# When adding new patches to "python" and "python3" in Fedora, EL, etc.,
|
||||
|
@ -565,10 +920,10 @@ Requires: python3-setuptools-wheel
|
|||
Requires: python3-pip-wheel
|
||||
%endif
|
||||
|
||||
# Runtime require alternatives
|
||||
Requires: %{_sbindir}/alternatives
|
||||
Requires(post): %{_sbindir}/alternatives
|
||||
Requires(postun): %{_sbindir}/alternatives
|
||||
# Require alternatives version that implements the --keep-foreign flag
|
||||
Requires: alternatives >= 1.19.1-1
|
||||
Requires(post): alternatives >= 1.19.1-1
|
||||
Requires(postun): alternatives >= 1.19.1-1
|
||||
|
||||
# This prevents ALL subpackages built from this spec to require
|
||||
# /usr/bin/python3*. Granularity per subpackage is impossible.
|
||||
|
@ -709,6 +1064,9 @@ Provides: %{name}-tools = %{version}-%{release}
|
|||
Provides: %{name}-tools%{?_isa} = %{version}-%{release}
|
||||
Obsoletes: %{name}-tools < %{version}-%{release}
|
||||
|
||||
|
||||
# Require alternatives version that implements the --keep-foreign flag
|
||||
Requires(postun): alternatives >= 1.19.1-1
|
||||
# python36 installs the alternatives master symlink to which we attach a slave
|
||||
Requires: python36
|
||||
Requires(post): python36
|
||||
|
@ -730,6 +1088,7 @@ configuration, browsers, and other dialogs.
|
|||
%package tkinter
|
||||
Summary: A GUI toolkit for Python
|
||||
Requires: platform-python = %{version}-%{release}
|
||||
Requires: %{name}-libs%{?_isa} = %{version}-%{release}
|
||||
|
||||
%description tkinter
|
||||
The Tkinter (Tk interface) library is a graphical user interface toolkit for
|
||||
|
@ -816,8 +1175,8 @@ rm Lib/ensurepip/_bundled/*.whl
|
|||
%endif
|
||||
|
||||
%patch251 -p1
|
||||
%patch257 -p1
|
||||
%patch262 -p1
|
||||
%patch274 -p1
|
||||
%patch294 -p1
|
||||
%patch316 -p1
|
||||
%patch317 -p1
|
||||
|
@ -841,11 +1200,47 @@ rm Lib/ensurepip/_bundled/*.whl
|
|||
git apply %{PATCH351}
|
||||
|
||||
%patch352 -p1
|
||||
%patch353 -p1
|
||||
%patch354 -p1
|
||||
%patch355 -p1
|
||||
%patch356 -p1
|
||||
%patch357 -p1
|
||||
%patch359 -p1
|
||||
%patch360 -p1
|
||||
%patch362 -p1
|
||||
%patch364 -p1
|
||||
%patch366 -p1
|
||||
%patch368 -p1
|
||||
%patch369 -p1
|
||||
%patch370 -p1
|
||||
%patch372 -p1
|
||||
%patch377 -p1
|
||||
%patch378 -p1
|
||||
%patch382 -p1
|
||||
%patch386 -p1
|
||||
%patch387 -p1
|
||||
%patch394 -p1
|
||||
%patch397 -p1
|
||||
%patch399 -p1
|
||||
%patch404 -p1
|
||||
%patch408 -p1
|
||||
%patch413 -p1
|
||||
%patch414 -p1
|
||||
%patch415 -p1
|
||||
%patch422 -p1
|
||||
%patch426 -p1
|
||||
%patch427 -p1
|
||||
|
||||
# Remove files that should be generated by the build
|
||||
# (This is after patching, so that we can use patches directly from upstream)
|
||||
rm configure pyconfig.h.in
|
||||
|
||||
# When we use the legacy arch names, we need to change them in configure.ac
|
||||
%if %{with legacy_archnames}
|
||||
sed -i configure.ac \
|
||||
-e 's/\b%{platform_triplet_upstream}\b/%{platform_triplet_legacy}/'
|
||||
%endif
|
||||
|
||||
|
||||
# ======================================================
|
||||
# Configuring and building the code:
|
||||
|
@ -926,6 +1321,9 @@ BuildPython() {
|
|||
$ExtraConfigArgs \
|
||||
%{nil}
|
||||
|
||||
# Regenerate generated importlib frozen modules (see patch 353)
|
||||
%make_build CFLAGS_NODIST="$CFLAGS_NODIST $MoreCFlags" regen-importlib
|
||||
|
||||
# Invoke the build
|
||||
%make_build CFLAGS_NODIST="$CFLAGS_NODIST $MoreCFlags"
|
||||
|
||||
|
@ -989,6 +1387,7 @@ mkdir -p %{buildroot}$DirHoldingGdbPy
|
|||
%global _pyconfig64_h pyconfig-64.h
|
||||
%global _pyconfig_h pyconfig-%{wordsize}.h
|
||||
|
||||
|
||||
# Use a common function to do an install for all our configurations:
|
||||
InstallPython() {
|
||||
|
||||
|
@ -1125,7 +1524,7 @@ do
|
|||
LD_LIBRARY_PATH=./build/optimized ./build/optimized/python \
|
||||
Tools/scripts/pathfix.py \
|
||||
-i "%{_libexecdir}/platform-python${LDVersion}" -pn \
|
||||
%{buildroot}%{pylibdir}/config-${LDVersion}-%{_arch}-linux%{_gnu}/python-config.py
|
||||
%{buildroot}%{pylibdir}/config-${LDVersion}-%{platform_triplet}/python-config.py
|
||||
done
|
||||
|
||||
# Remove tests for python3-tools which was removed in
|
||||
|
@ -1206,6 +1605,11 @@ touch %{buildroot}%{_bindir}/unversioned-python
|
|||
touch %{buildroot}%{_bindir}/idle3
|
||||
touch %{buildroot}%{_mandir}/man1/python.1.gz
|
||||
|
||||
# Strip the LTO bytecode from python.o
|
||||
# Based on the fedora brp-strip-lto scriptlet
|
||||
# https://src.fedoraproject.org/rpms/redhat-rpm-config/blob/9dd5528cf9805ebfe31cff04fe7828ad06a6023f/f/brp-strip-lto
|
||||
find %{buildroot} -type f -name 'python.o' -print0 | xargs -0 \
|
||||
bash -c "strip -p -R .gnu.lto_* -R .gnu.debuglto_* -N __gnu_lto_v1 \"\$@\"" ARG0
|
||||
|
||||
# ======================================================
|
||||
# Checks for packaging issues
|
||||
|
@ -1299,7 +1703,7 @@ alternatives --install %{_bindir}/unversioned-python \
|
|||
%postun -n platform-python
|
||||
# Do this only during uninstall process (not during update)
|
||||
if [ $1 -eq 0 ]; then
|
||||
alternatives --remove python \
|
||||
alternatives --keep-foreign --remove python \
|
||||
%{_libexecdir}/no-python
|
||||
|
||||
fi
|
||||
|
@ -1314,7 +1718,7 @@ alternatives --add-slave python3 %{_bindir}/python3.6 \
|
|||
%postun -n python3-idle
|
||||
# Do this only during uninstall process (not during update)
|
||||
if [ $1 -eq 0 ]; then
|
||||
alternatives --remove-slave python3 %{_bindir}/python3.6 \
|
||||
alternatives --keep-foreign --remove-slave python3 %{_bindir}/python3.6 \
|
||||
idle3
|
||||
fi
|
||||
|
||||
|
@ -1545,8 +1949,8 @@ fi
|
|||
# "Makefile" and the config-32/64.h file are needed by
|
||||
# distutils/sysconfig.py:_init_posix(), so we include them in the core
|
||||
# package, along with their parent directories (bug 531901):
|
||||
%dir %{pylibdir}/config-%{LDVERSION_optimized}-%{_arch}-linux%{_gnu}/
|
||||
%{pylibdir}/config-%{LDVERSION_optimized}-%{_arch}-linux%{_gnu}/Makefile
|
||||
%dir %{pylibdir}/config-%{LDVERSION_optimized}-%{platform_triplet}/
|
||||
%{pylibdir}/config-%{LDVERSION_optimized}-%{platform_triplet}/Makefile
|
||||
%dir %{_includedir}/python%{LDVERSION_optimized}/
|
||||
%{_includedir}/python%{LDVERSION_optimized}/%{_pyconfig_h}
|
||||
|
||||
|
@ -1560,8 +1964,8 @@ fi
|
|||
%{_bindir}/2to3
|
||||
# TODO: Remove 2to3-3.7 once rebased to 3.7
|
||||
%{_bindir}/2to3-%{pybasever}
|
||||
%{pylibdir}/config-%{LDVERSION_optimized}-%{_arch}-linux%{_gnu}/*
|
||||
%exclude %{pylibdir}/config-%{LDVERSION_optimized}-%{_arch}-linux%{_gnu}/Makefile
|
||||
%{pylibdir}/config-%{LDVERSION_optimized}-%{platform_triplet}/*
|
||||
%exclude %{pylibdir}/config-%{LDVERSION_optimized}-%{platform_triplet}/Makefile
|
||||
%exclude %{pylibdir}/distutils/command/wininst-*.exe
|
||||
%{_includedir}/python%{LDVERSION_optimized}/*.h
|
||||
%exclude %{_includedir}/python%{LDVERSION_optimized}/%{_pyconfig_h}
|
||||
|
@ -1709,7 +2113,7 @@ fi
|
|||
%{_libdir}/%{py_INSTSONAME_debug}
|
||||
|
||||
# Analog of the -devel subpackage's files:
|
||||
%{pylibdir}/config-%{LDVERSION_debug}-%{_arch}-linux%{_gnu}
|
||||
%{pylibdir}/config-%{LDVERSION_debug}-%{platform_triplet}
|
||||
%{_includedir}/python%{LDVERSION_debug}
|
||||
|
||||
%exclude %{_bindir}/python%{LDVERSION_debug}-config
|
||||
|
@ -1757,6 +2161,137 @@ fi
|
|||
# ======================================================
|
||||
|
||||
%changelog
|
||||
* Wed Apr 24 2024 Lumír Balhar <lbalhar@redhat.com> - 3.6.8-62
|
||||
- Security fix for CVE-2024-0450
|
||||
Resolves: RHEL-33683
|
||||
|
||||
* Wed Apr 24 2024 Lumír Balhar <lbalhar@redhat.com> - 3.6.8-61
|
||||
- Security fix for CVE-2023-6597
|
||||
Resolves: RHEL-33671
|
||||
|
||||
* Wed Apr 24 2024 Lumír Balhar <lbalhar@redhat.com> - 3.6.8-60
|
||||
- Fix build with expat with fixed CVE-2023-52425
|
||||
Related: RHEL-33671
|
||||
|
||||
* Thu Jan 04 2024 Lumír Balhar <lbalhar@redhat.com> - 3.6.8-59
|
||||
- Security fix for CVE-2023-27043
|
||||
Resolves: RHEL-20610
|
||||
|
||||
* Tue Dec 12 2023 Lumír Balhar <lbalhar@redhat.com> - 3.6.8-58
|
||||
- Security fix for CVE-2022-48564
|
||||
Resolves: RHEL-16674
|
||||
- Skip tests failing on s390x
|
||||
Resolves: RHEL-19252
|
||||
|
||||
* Thu Nov 23 2023 Lumír Balhar <lbalhar@redhat.com> - 3.6.8-57
|
||||
- Security fix for CVE-2022-48560
|
||||
Resolves: RHEL-16707
|
||||
|
||||
* Thu Sep 07 2023 Charalampos Stratakis <cstratak@redhat.com> - 3.6.8-56
|
||||
- Security fix for CVE-2023-40217
|
||||
Resolves: RHEL-3041
|
||||
|
||||
* Wed Aug 09 2023 Petr Viktorin <pviktori@redhat.com> - 3.6.8-55
|
||||
- Fix symlink handling in the fix for CVE-2007-4559
|
||||
Resolves: rhbz#263261
|
||||
|
||||
* Fri Jul 07 2023 Charalampos Stratakis <cstratak@redhat.com> - 3.6.8-54
|
||||
- Bump release for rebuild
|
||||
Resolves: rhbz#2173917
|
||||
|
||||
* Fri Jun 30 2023 Charalampos Stratakis <cstratak@redhat.com> - 3.6.8-53
|
||||
- Security fix for CVE-2023-24329
|
||||
Resolves: rhbz#2173917
|
||||
|
||||
* Tue Jun 06 2023 Petr Viktorin <pviktori@redhat.com> - 3.6.8-52
|
||||
- Add filters for tarfile extraction (CVE-2007-4559, PEP-706)
|
||||
Resolves: rhbz#263261
|
||||
|
||||
* Tue Jan 24 2023 Charalampos Stratakis <cstratak@redhat.com> - 3.6.8-51
|
||||
- Properly strip the LTO bytecode from python.o
|
||||
Resolves: rhbz#2137707
|
||||
|
||||
* Wed Dec 21 2022 Charalampos Stratakis <cstratak@redhat.com> - 3.6.8-50
|
||||
- Security fix for CVE-2022-45061
|
||||
- Strip the LTO bytecode from python.o
|
||||
Resolves: rhbz#2144072, rhbz#2137707
|
||||
|
||||
* Tue Oct 25 2022 Charalampos Stratakis <cstratak@redhat.com> - 3.6.8-49
|
||||
- Security fixes for CVE-2020-10735 and CVE-2021-28861
|
||||
Resolves: rhbz#1834423, rhbz#2120642
|
||||
|
||||
* Thu Oct 20 2022 Charalampos Stratakis <cstratak@redhat.com> - 3.6.8-48
|
||||
- Release bump
|
||||
Resolves: rhbz#2136435
|
||||
|
||||
* Tue Jun 14 2022 Charalampos Stratakis <cstratak@redhat.com> - 3.6.8-47
|
||||
- Security fix for CVE-2015-20107
|
||||
Resolves: rhbz#2075390
|
||||
|
||||
* Wed Mar 09 2022 Charalampos Stratakis <cstratak@redhat.com> - 3.6.8-46
|
||||
- Security fix for CVE-2022-0391: urlparse does not sanitize URLs containing ASCII newline and tabs
|
||||
- Fix the test suite support for Expat >= 2.4.5
|
||||
Resolves: rhbz#2047376, rhbz#2060435
|
||||
|
||||
* Fri Jan 07 2022 Charalampos Stratakis <cstratak@redhat.com> - 3.6.8-45
|
||||
- Security fix for CVE-2021-4189: ftplib should not use the host from the PASV response
|
||||
Resolves: rhbz#2036020
|
||||
|
||||
* Tue Oct 12 2021 Charalampos Stratakis <cstratak@redhat.com> - 3.6.8-44
|
||||
- Use the monotonic clock for theading.Condition
|
||||
- Use the monotonic clock for the global interpreter lock
|
||||
Resolves: rhbz#2003758
|
||||
|
||||
* Mon Oct 11 2021 Charalampos Stratakis <cstratak@redhat.com> - 3.6.8-43
|
||||
- Change shouldRollover() methods of logging.handlers to only rollover regular files
|
||||
Resolves: rhbz#2009200
|
||||
|
||||
* Fri Sep 17 2021 Lumír Balhar <lbalhar@redhat.com> - 3.6.8-42
|
||||
- Security fix for CVE-2021-3737
|
||||
Resolves: rhbz#1995162
|
||||
|
||||
* Thu Sep 09 2021 Lumír Balhar <lbalhar@redhat.com> - 3.6.8-41
|
||||
- Security fix for CVE-2021-3733: Denial of service when identifying crafted invalid RFCs
|
||||
Resolves: rhbz#1995234
|
||||
|
||||
* Thu Jul 29 2021 Tomas Orsava <torsava@redhat.com> - 3.6.8-40
|
||||
- Adjusted the postun scriptlets to enable upgrading to RHEL 9
|
||||
- Resolves: rhbz#1933055
|
||||
|
||||
* Fri Jul 09 2021 Victor Stinner <vstinner@redhat.com> - 3.6.8-39
|
||||
- Fix reentrant call to threading.enumerate() (rhbz#1959459)
|
||||
- Don't exit Python with abort() when a thread exit and there is no available
|
||||
file descriptor to load dynamically the libgcc_s.so.1 library (rhbz#1972293)
|
||||
|
||||
* Fri Apr 30 2021 Charalampos Stratakis <cstratak@redhat.com> - 3.6.8-38
|
||||
- Security fix for CVE-2021-3426: information disclosure via pydoc
|
||||
Resolves: rhbz#1935913
|
||||
|
||||
* Thu Mar 04 2021 Petr Viktorin <pviktori@redhat.com> - 3.6.8-37
|
||||
- Fix for CVE-2021-23336
|
||||
Resolves: rhbz#1928904
|
||||
|
||||
* Fri Jan 22 2021 Lumír Balhar <lbalhar@redhat.com> - 3.6.8-36
|
||||
- Fix for CVE-2021-3177
|
||||
Resolves: rhbz#1918168
|
||||
|
||||
* Mon Jan 18 2021 Lumír Balhar <lbalhar@redhat.com> - 3.6.8-35
|
||||
- New options -a and -k for pathfix.py script backported from upstream
|
||||
Resolves: rhbz#1917691
|
||||
|
||||
* Fri Dec 04 2020 Charalampos Stratakis <cstratak@redhat.com> - 3.6.8-34
|
||||
- Security fix for CVE-2020-27619: eval() call on content received via HTTP in the CJK codec tests
|
||||
Resolves: rhbz#1890237
|
||||
|
||||
* Tue Nov 24 2020 Lumír Balhar <lbalhar@redhat.com> - 3.6.8-33
|
||||
- Add support for upstream architecture names
|
||||
https://fedoraproject.org/wiki/Changes/Python_Upstream_Architecture_Names
|
||||
Resolves: rhbz#1868003
|
||||
|
||||
* Mon Nov 09 2020 Charalampos Stratakis <cstratak@redhat.com> - 3.6.8-32
|
||||
- Security fix for CVE-2020-26116: Reject control chars in HTTP method in http.client
|
||||
Resolves: rhbz#1883257
|
||||
|
||||
* Mon Aug 17 2020 Tomas Orsava <torsava@redhat.com> - 3.6.8-31
|
||||
- Avoid infinite loop when reading specially crafted TAR files (CVE-2019-20907)
|
||||
Resolves: rhbz#1856481
|
||||
|
|
Loading…
Reference in New Issue