From 7c0aac331f036d3b0c81e7a82c8f8755e3100400 Mon Sep 17 00:00:00 2001 From: CentOS Sources Date: Tue, 18 May 2021 02:36:27 -0400 Subject: [PATCH] import python3-3.6.8-37.el8 --- SOURCES/00274-fix-arch-names.patch | 58 -- ...chitecture-names-upstream-downstream.patch | 97 +++ ...est-method-crlf-injection-in-httplib.patch | 73 ++ SOURCES/00355-CVE-2020-27619.patch | 42 ++ .../00356-k_and_a_options_for_pathfix.patch | 269 +++++++ SOURCES/00357-CVE-2021-3177.patch | 184 +++++ SOURCES/00359-CVE-2021-23336.patch | 684 ++++++++++++++++++ SPECS/python3.spec | 139 +++- 8 files changed, 1474 insertions(+), 72 deletions(-) delete mode 100644 SOURCES/00274-fix-arch-names.patch create mode 100644 SOURCES/00353-architecture-names-upstream-downstream.patch create mode 100644 SOURCES/00354-cve-2020-26116-http-request-method-crlf-injection-in-httplib.patch create mode 100644 SOURCES/00355-CVE-2020-27619.patch create mode 100644 SOURCES/00356-k_and_a_options_for_pathfix.patch create mode 100644 SOURCES/00357-CVE-2021-3177.patch create mode 100644 SOURCES/00359-CVE-2021-23336.patch diff --git a/SOURCES/00274-fix-arch-names.patch b/SOURCES/00274-fix-arch-names.patch deleted file mode 100644 index 9d69223..0000000 --- a/SOURCES/00274-fix-arch-names.patch +++ /dev/null @@ -1,58 +0,0 @@ -diff -up Python-3.5.0/configure.ac.than Python-3.5.0/configure.ac ---- Python-3.5.0/configure.ac.than 2015-11-13 11:51:32.039560172 -0500 -+++ Python-3.5.0/configure.ac 2015-11-13 11:52:11.670168157 -0500 -@@ -788,9 +788,9 @@ cat >> conftest.c <> conftest.c <> conftest.c <> conftest.c < +Date: Tue, 4 Aug 2020 12:04:03 +0200 +Subject: [PATCH] 00353: Original names for architectures with different names + downstream +MIME-Version: 1.0 +Content-Type: text/plain; charset=UTF-8 +Content-Transfer-Encoding: 8bit + +https://fedoraproject.org/wiki/Changes/Python_Upstream_Architecture_Names + +Pythons in RHEL/Fedora used different names for some architectures +than upstream and other distros (for example ppc64 vs. powerpc64). +This was patched in patch 274, now it is sedded if %with legacy_archnames. + +That meant that an extension built with the default upstream settings +(on other distro or as an manylinux wheel) could not been found by Python +on RHEL/Fedora because it had a different suffix. +This patch adds the legacy names to importlib so Python is able +to import extensions with a legacy architecture name in its +file name. +It work both ways, so it support both %with and %without legacy_archnames. + +WARNING: This patch has no effect on Python built with bootstrap +enabled because Python/importlib_external.h is not regenerated +and therefore Python during bootstrap contains importlib from +upstream without this feature. It's possible to include +Python/importlib_external.h to this patch but it'd make rebasing +a nightmare because it's basically a binary file. + +Co-authored-by: Miro Hrončok +--- + Lib/importlib/_bootstrap_external.py | 40 ++++++++++++++++++++++++++-- + 1 file changed, 38 insertions(+), 2 deletions(-) + +diff --git a/Lib/importlib/_bootstrap_external.py b/Lib/importlib/_bootstrap_external.py +index 9feec50842..5bb2454a5c 100644 +--- a/Lib/importlib/_bootstrap_external.py ++++ b/Lib/importlib/_bootstrap_external.py +@@ -1361,7 +1361,7 @@ def _get_supported_file_loaders(): + + Each item is a tuple (loader, suffixes). + """ +- extensions = ExtensionFileLoader, _imp.extension_suffixes() ++ extensions = ExtensionFileLoader, _alternative_architectures(_imp.extension_suffixes()) + source = SourceFileLoader, SOURCE_SUFFIXES + bytecode = SourcelessFileLoader, BYTECODE_SUFFIXES + return [extensions, source, bytecode] +@@ -1428,7 +1428,7 @@ def _setup(_bootstrap_module): + + # Constants + setattr(self_module, '_relax_case', _make_relax_case()) +- EXTENSION_SUFFIXES.extend(_imp.extension_suffixes()) ++ EXTENSION_SUFFIXES.extend(_alternative_architectures(_imp.extension_suffixes())) + if builtin_os == 'nt': + SOURCE_SUFFIXES.append('.pyw') + if '_d.pyd' in EXTENSION_SUFFIXES: +@@ -1441,3 +1441,39 @@ def _install(_bootstrap_module): + supported_loaders = _get_supported_file_loaders() + sys.path_hooks.extend([FileFinder.path_hook(*supported_loaders)]) + sys.meta_path.append(PathFinder) ++ ++ ++_ARCH_MAP = { ++ "-arm-linux-gnueabi.": "-arm-linux-gnueabihf.", ++ "-armeb-linux-gnueabi.": "-armeb-linux-gnueabihf.", ++ "-mips64-linux-gnu.": "-mips64-linux-gnuabi64.", ++ "-mips64el-linux-gnu.": "-mips64el-linux-gnuabi64.", ++ "-ppc-linux-gnu.": "-powerpc-linux-gnu.", ++ "-ppc-linux-gnuspe.": "-powerpc-linux-gnuspe.", ++ "-ppc64-linux-gnu.": "-powerpc64-linux-gnu.", ++ "-ppc64le-linux-gnu.": "-powerpc64le-linux-gnu.", ++ # The above, but the other way around: ++ "-arm-linux-gnueabihf.": "-arm-linux-gnueabi.", ++ "-armeb-linux-gnueabihf.": "-armeb-linux-gnueabi.", ++ "-mips64-linux-gnuabi64.": "-mips64-linux-gnu.", ++ "-mips64el-linux-gnuabi64.": "-mips64el-linux-gnu.", ++ "-powerpc-linux-gnu.": "-ppc-linux-gnu.", ++ "-powerpc-linux-gnuspe.": "-ppc-linux-gnuspe.", ++ "-powerpc64-linux-gnu.": "-ppc64-linux-gnu.", ++ "-powerpc64le-linux-gnu.": "-ppc64le-linux-gnu.", ++} ++ ++ ++def _alternative_architectures(suffixes): ++ """Add a suffix with an alternative architecture name ++ to the list of suffixes so an extension built with ++ the default (upstream) setting is loadable with our Pythons ++ """ ++ ++ for suffix in suffixes: ++ for original, alternative in _ARCH_MAP.items(): ++ if original in suffix: ++ suffixes.append(suffix.replace(original, alternative)) ++ return suffixes ++ ++ return suffixes diff --git a/SOURCES/00354-cve-2020-26116-http-request-method-crlf-injection-in-httplib.patch b/SOURCES/00354-cve-2020-26116-http-request-method-crlf-injection-in-httplib.patch new file mode 100644 index 0000000..aa0b385 --- /dev/null +++ b/SOURCES/00354-cve-2020-26116-http-request-method-crlf-injection-in-httplib.patch @@ -0,0 +1,73 @@ +diff --git a/Lib/http/client.py b/Lib/http/client.py +index f0d2642..0a044e9 100644 +--- a/Lib/http/client.py ++++ b/Lib/http/client.py +@@ -151,6 +151,10 @@ _contains_disallowed_url_pchar_re = re.compile('[\x00-\x20\x7f]') + # _is_allowed_url_pchars_re = re.compile(r"^[/!$&'()*+,;=:@%a-zA-Z0-9._~-]+$") + # We are more lenient for assumed real world compatibility purposes. + ++# These characters are not allowed within HTTP method names ++# to prevent http header injection. ++_contains_disallowed_method_pchar_re = re.compile('[\x00-\x1f]') ++ + # We always set the Content-Length header for these methods because some + # servers will otherwise respond with a 411 + _METHODS_EXPECTING_BODY = {'PATCH', 'POST', 'PUT'} +@@ -1117,6 +1121,8 @@ class HTTPConnection: + else: + raise CannotSendRequest(self.__state) + ++ self._validate_method(method) ++ + # Save the method we use, we need it later in the response phase + self._method = method + if not url: +@@ -1207,6 +1213,15 @@ class HTTPConnection: + # For HTTP/1.0, the server will assume "not chunked" + pass + ++ def _validate_method(self, method): ++ """Validate a method name for putrequest.""" ++ # prevent http header injection ++ match = _contains_disallowed_method_pchar_re.search(method) ++ if match: ++ raise ValueError( ++ f"method can't contain control characters. {method!r} " ++ f"(found at least {match.group()!r})") ++ + def putheader(self, header, *values): + """Send a request header line to the server. + +diff --git a/Lib/test/test_httplib.py b/Lib/test/test_httplib.py +index 5795b7a..af0350f 100644 +--- a/Lib/test/test_httplib.py ++++ b/Lib/test/test_httplib.py +@@ -359,6 +359,28 @@ class HeaderTests(TestCase): + self.assertEqual(lines[2], "header: Second: val") + + ++class HttpMethodTests(TestCase): ++ def test_invalid_method_names(self): ++ methods = ( ++ 'GET\r', ++ 'POST\n', ++ 'PUT\n\r', ++ 'POST\nValue', ++ 'POST\nHOST:abc', ++ 'GET\nrHost:abc\n', ++ 'POST\rRemainder:\r', ++ 'GET\rHOST:\n', ++ '\nPUT' ++ ) ++ ++ for method in methods: ++ with self.assertRaisesRegex( ++ ValueError, "method can't contain control characters"): ++ conn = client.HTTPConnection('example.com') ++ conn.sock = FakeSocket(None) ++ conn.request(method=method, url="/") ++ ++ + class TransferEncodingTest(TestCase): + expected_body = b"It's just a flesh wound" + diff --git a/SOURCES/00355-CVE-2020-27619.patch b/SOURCES/00355-CVE-2020-27619.patch new file mode 100644 index 0000000..6a4082d --- /dev/null +++ b/SOURCES/00355-CVE-2020-27619.patch @@ -0,0 +1,42 @@ +diff --git a/Lib/test/multibytecodec_support.py b/Lib/test/multibytecodec_support.py +index f9884c6..98feec2 100644 +--- a/Lib/test/multibytecodec_support.py ++++ b/Lib/test/multibytecodec_support.py +@@ -300,29 +300,23 @@ class TestBase_Mapping(unittest.TestCase): + self._test_mapping_file_plain() + + def _test_mapping_file_plain(self): +- unichrs = lambda s: ''.join(map(chr, map(eval, s.split('+')))) ++ def unichrs(s): ++ return ''.join(chr(int(x, 16)) for x in s.split('+')) ++ + urt_wa = {} + + with self.open_mapping_file() as f: + for line in f: + if not line: + break +- data = line.split('#')[0].strip().split() ++ data = line.split('#')[0].split() + if len(data) != 2: + continue + +- csetval = eval(data[0]) +- if csetval <= 0x7F: +- csetch = bytes([csetval & 0xff]) +- elif csetval >= 0x1000000: +- csetch = bytes([(csetval >> 24), ((csetval >> 16) & 0xff), +- ((csetval >> 8) & 0xff), (csetval & 0xff)]) +- elif csetval >= 0x10000: +- csetch = bytes([(csetval >> 16), ((csetval >> 8) & 0xff), +- (csetval & 0xff)]) +- elif csetval >= 0x100: +- csetch = bytes([(csetval >> 8), (csetval & 0xff)]) +- else: ++ if data[0][:2] != '0x': ++ self.fail(f"Invalid line: {line!r}") ++ csetch = bytes.fromhex(data[0][2:]) ++ if len(csetch) == 1 and 0x80 <= csetch[0]: + continue + + unich = unichrs(data[1]) diff --git a/SOURCES/00356-k_and_a_options_for_pathfix.patch b/SOURCES/00356-k_and_a_options_for_pathfix.patch new file mode 100644 index 0000000..3782e6e --- /dev/null +++ b/SOURCES/00356-k_and_a_options_for_pathfix.patch @@ -0,0 +1,269 @@ +From 0cfd9a7f26488567b9a3e5ec192099a8b80ad9df Mon Sep 17 00:00:00 2001 +From: Lumir Balhar +Date: Tue, 19 Jan 2021 07:55:37 +0100 +Subject: [PATCH] [PATCH] bpo-37064: Add -k and -a options to pathfix.py tool + (GH-16387) + +* bpo-37064: Add option -k to Tools/scripts/pathfix.py (GH-15548) + +Add flag -k to pathscript.py script: preserve shebang flags. + +(cherry picked from commit 50254ac4c179cb412e90682098c97db786143929) + +* bpo-37064: Add option -a to pathfix.py tool (GH-15717) + +Add option -a to Tools/Scripts/pathfix.py script: add flags. + +(cherry picked from commit 1dc1acbd73f05f14c974b7ce1041787d7abef31e) +--- + Lib/test/test_tools/test_pathfix.py | 104 ++++++++++++++++++++++++++++ + Tools/scripts/pathfix.py | 64 +++++++++++++++-- + 2 files changed, 163 insertions(+), 5 deletions(-) + create mode 100644 Lib/test/test_tools/test_pathfix.py + +diff --git a/Lib/test/test_tools/test_pathfix.py b/Lib/test/test_tools/test_pathfix.py +new file mode 100644 +index 0000000..1f0585e +--- /dev/null ++++ b/Lib/test/test_tools/test_pathfix.py +@@ -0,0 +1,104 @@ ++import os ++import subprocess ++import sys ++import unittest ++from test import support ++from test.test_tools import import_tool, scriptsdir ++ ++ ++class TestPathfixFunctional(unittest.TestCase): ++ script = os.path.join(scriptsdir, 'pathfix.py') ++ ++ def setUp(self): ++ self.temp_file = support.TESTFN ++ self.addCleanup(support.unlink, support.TESTFN) ++ ++ def pathfix(self, shebang, pathfix_flags, exitcode=0, stdout='', stderr=''): ++ with open(self.temp_file, 'w', encoding='utf8') as f: ++ f.write(f'{shebang}\n' + 'print("Hello world")\n') ++ ++ proc = subprocess.run( ++ [sys.executable, self.script, ++ *pathfix_flags, '-n', self.temp_file], ++ universal_newlines=True, stdout=subprocess.PIPE, ++ stderr=subprocess.PIPE) ++ ++ if stdout == '' and proc.returncode == 0: ++ stdout = f'{self.temp_file}: updating\n' ++ self.assertEqual(proc.returncode, exitcode, proc) ++ self.assertEqual(proc.stdout, stdout, proc) ++ self.assertEqual(proc.stderr, stderr, proc) ++ ++ with open(self.temp_file, 'r', encoding='utf8') as f: ++ output = f.read() ++ ++ lines = output.split('\n') ++ self.assertEqual(lines[1:], ['print("Hello world")', '']) ++ new_shebang = lines[0] ++ ++ if proc.returncode != 0: ++ self.assertEqual(shebang, new_shebang) ++ ++ return new_shebang ++ ++ def test_pathfix(self): ++ self.assertEqual( ++ self.pathfix( ++ '#! /usr/bin/env python', ++ ['-i', '/usr/bin/python3']), ++ '#! /usr/bin/python3') ++ self.assertEqual( ++ self.pathfix( ++ '#! /usr/bin/env python -R', ++ ['-i', '/usr/bin/python3']), ++ '#! /usr/bin/python3') ++ ++ def test_pathfix_keeping_flags(self): ++ self.assertEqual( ++ self.pathfix( ++ '#! /usr/bin/env python -R', ++ ['-i', '/usr/bin/python3', '-k']), ++ '#! /usr/bin/python3 -R') ++ self.assertEqual( ++ self.pathfix( ++ '#! /usr/bin/env python', ++ ['-i', '/usr/bin/python3', '-k']), ++ '#! /usr/bin/python3') ++ ++ def test_pathfix_adding_flag(self): ++ self.assertEqual( ++ self.pathfix( ++ '#! /usr/bin/env python', ++ ['-i', '/usr/bin/python3', '-a', 's']), ++ '#! /usr/bin/python3 -s') ++ self.assertEqual( ++ self.pathfix( ++ '#! /usr/bin/env python -S', ++ ['-i', '/usr/bin/python3', '-a', 's']), ++ '#! /usr/bin/python3 -s') ++ self.assertEqual( ++ self.pathfix( ++ '#! /usr/bin/env python -V', ++ ['-i', '/usr/bin/python3', '-a', 'v', '-k']), ++ '#! /usr/bin/python3 -vV') ++ self.assertEqual( ++ self.pathfix( ++ '#! /usr/bin/env python', ++ ['-i', '/usr/bin/python3', '-a', 'Rs']), ++ '#! /usr/bin/python3 -Rs') ++ self.assertEqual( ++ self.pathfix( ++ '#! /usr/bin/env python -W default', ++ ['-i', '/usr/bin/python3', '-a', 's', '-k']), ++ '#! /usr/bin/python3 -sW default') ++ ++ def test_pathfix_adding_errors(self): ++ self.pathfix( ++ '#! /usr/bin/env python -E', ++ ['-i', '/usr/bin/python3', '-a', 'W default', '-k'], ++ exitcode=2, ++ stderr="-a option doesn't support whitespaces") ++ ++ ++if __name__ == '__main__': ++ unittest.main() +diff --git a/Tools/scripts/pathfix.py b/Tools/scripts/pathfix.py +index c5bf984..2dfa6e8 100755 +--- a/Tools/scripts/pathfix.py ++++ b/Tools/scripts/pathfix.py +@@ -1,6 +1,6 @@ + #!/usr/bin/env python3 + +-# Change the #! line occurring in Python scripts. The new interpreter ++# Change the #! line (shebang) occurring in Python scripts. The new interpreter + # pathname must be given with a -i option. + # + # Command line arguments are files or directories to be processed. +@@ -10,7 +10,13 @@ + # arguments). + # The original file is kept as a back-up (with a "~" attached to its name), + # -n flag can be used to disable this. +-# ++ ++# Sometimes you may find shebangs with flags such as `#! /usr/bin/env python -si`. ++# Normally, pathfix overwrites the entire line, including the flags. ++# To change interpreter and keep flags from the original shebang line, use -k. ++# If you want to keep flags and add to them one single literal flag, use option -a. ++ ++ + # Undoubtedly you can do this using find and sed or perl, but this is + # a nice example of Python code that recurses down a directory tree + # and uses regular expressions. Also note several subtleties like +@@ -33,16 +39,21 @@ rep = sys.stdout.write + new_interpreter = None + preserve_timestamps = False + create_backup = True ++keep_flags = False ++add_flags = b'' + + + def main(): + global new_interpreter + global preserve_timestamps + global create_backup +- usage = ('usage: %s -i /interpreter -p -n file-or-directory ...\n' % ++ global keep_flags ++ global add_flags ++ ++ usage = ('usage: %s -i /interpreter -p -n -k -a file-or-directory ...\n' % + sys.argv[0]) + try: +- opts, args = getopt.getopt(sys.argv[1:], 'i:pn') ++ opts, args = getopt.getopt(sys.argv[1:], 'i:a:kpn') + except getopt.error as msg: + err(str(msg) + '\n') + err(usage) +@@ -54,6 +65,13 @@ def main(): + preserve_timestamps = True + if o == '-n': + create_backup = False ++ if o == '-k': ++ keep_flags = True ++ if o == '-a': ++ add_flags = a.encode() ++ if b' ' in add_flags: ++ err("-a option doesn't support whitespaces") ++ sys.exit(2) + if not new_interpreter or not new_interpreter.startswith(b'/') or \ + not args: + err('-i option or file-or-directory missing\n') +@@ -70,10 +88,14 @@ def main(): + if fix(arg): bad = 1 + sys.exit(bad) + ++ + ispythonprog = re.compile(r'^[a-zA-Z0-9_]+\.py$') ++ ++ + def ispython(name): + return bool(ispythonprog.match(name)) + ++ + def recursedown(dirname): + dbg('recursedown(%r)\n' % (dirname,)) + bad = 0 +@@ -96,6 +118,7 @@ def recursedown(dirname): + if recursedown(fullname): bad = 1 + return bad + ++ + def fix(filename): + ## dbg('fix(%r)\n' % (filename,)) + try: +@@ -166,12 +189,43 @@ def fix(filename): + # Return success + return 0 + ++ ++def parse_shebang(shebangline): ++ shebangline = shebangline.rstrip(b'\n') ++ start = shebangline.find(b' -') ++ if start == -1: ++ return b'' ++ return shebangline[start:] ++ ++ ++def populate_flags(shebangline): ++ old_flags = b'' ++ if keep_flags: ++ old_flags = parse_shebang(shebangline) ++ if old_flags: ++ old_flags = old_flags[2:] ++ if not (old_flags or add_flags): ++ return b'' ++ # On Linux, the entire string following the interpreter name ++ # is passed as a single argument to the interpreter. ++ # e.g. "#! /usr/bin/python3 -W Error -s" runs "/usr/bin/python3 "-W Error -s" ++ # so shebang should have single '-' where flags are given and ++ # flag might need argument for that reasons adding new flags is ++ # between '-' and original flags ++ # e.g. #! /usr/bin/python3 -sW Error ++ return b' -' + add_flags + old_flags ++ ++ + def fixline(line): + if not line.startswith(b'#!'): + return line ++ + if b"python" not in line: + return line +- return b'#! ' + new_interpreter + b'\n' ++ ++ flags = populate_flags(line) ++ return b'#! ' + new_interpreter + flags + b'\n' ++ + + if __name__ == '__main__': + main() +-- +2.29.2 + diff --git a/SOURCES/00357-CVE-2021-3177.patch b/SOURCES/00357-CVE-2021-3177.patch new file mode 100644 index 0000000..339e1b5 --- /dev/null +++ b/SOURCES/00357-CVE-2021-3177.patch @@ -0,0 +1,184 @@ +From e92381a0a6a3e1f000956e1f1e70e543b9c2bcd5 Mon Sep 17 00:00:00 2001 +From: Benjamin Peterson +Date: Mon, 18 Jan 2021 14:47:05 -0600 +Subject: [PATCH] [3.6] closes bpo-42938: Replace snprintf with Python unicode + formatting in ctypes param reprs. (24239). (cherry picked from commit + 916610ef90a0d0761f08747f7b0905541f0977c7) + +Co-authored-by: Benjamin Peterson +--- + Lib/ctypes/test/test_parameters.py | 43 +++++++++++++++ + .../2021-01-18-09-27-31.bpo-42938.4Zn4Mp.rst | 2 + + Modules/_ctypes/callproc.c | 55 +++++++------------ + 3 files changed, 66 insertions(+), 34 deletions(-) + create mode 100644 Misc/NEWS.d/next/Security/2021-01-18-09-27-31.bpo-42938.4Zn4Mp.rst + +diff --git a/Lib/ctypes/test/test_parameters.py b/Lib/ctypes/test/test_parameters.py +index e4c25fd880cef..531894fdec838 100644 +--- a/Lib/ctypes/test/test_parameters.py ++++ b/Lib/ctypes/test/test_parameters.py +@@ -201,6 +201,49 @@ def __dict__(self): + with self.assertRaises(ZeroDivisionError): + WorseStruct().__setstate__({}, b'foo') + ++ def test_parameter_repr(self): ++ from ctypes import ( ++ c_bool, ++ c_char, ++ c_wchar, ++ c_byte, ++ c_ubyte, ++ c_short, ++ c_ushort, ++ c_int, ++ c_uint, ++ c_long, ++ c_ulong, ++ c_longlong, ++ c_ulonglong, ++ c_float, ++ c_double, ++ c_longdouble, ++ c_char_p, ++ c_wchar_p, ++ c_void_p, ++ ) ++ self.assertRegex(repr(c_bool.from_param(True)), r"^$") ++ self.assertEqual(repr(c_char.from_param(97)), "") ++ self.assertRegex(repr(c_wchar.from_param('a')), r"^$") ++ self.assertEqual(repr(c_byte.from_param(98)), "") ++ self.assertEqual(repr(c_ubyte.from_param(98)), "") ++ self.assertEqual(repr(c_short.from_param(511)), "") ++ self.assertEqual(repr(c_ushort.from_param(511)), "") ++ self.assertRegex(repr(c_int.from_param(20000)), r"^$") ++ self.assertRegex(repr(c_uint.from_param(20000)), r"^$") ++ self.assertRegex(repr(c_long.from_param(20000)), r"^$") ++ self.assertRegex(repr(c_ulong.from_param(20000)), r"^$") ++ self.assertRegex(repr(c_longlong.from_param(20000)), r"^$") ++ self.assertRegex(repr(c_ulonglong.from_param(20000)), r"^$") ++ self.assertEqual(repr(c_float.from_param(1.5)), "") ++ self.assertEqual(repr(c_double.from_param(1.5)), "") ++ self.assertEqual(repr(c_double.from_param(1e300)), "") ++ self.assertRegex(repr(c_longdouble.from_param(1.5)), r"^$") ++ self.assertRegex(repr(c_char_p.from_param(b'hihi')), "^$") ++ self.assertRegex(repr(c_wchar_p.from_param('hihi')), "^$") ++ self.assertRegex(repr(c_void_p.from_param(0x12)), r"^$") ++ + ################################################################ + + if __name__ == '__main__': +diff --git a/Misc/NEWS.d/next/Security/2021-01-18-09-27-31.bpo-42938.4Zn4Mp.rst b/Misc/NEWS.d/next/Security/2021-01-18-09-27-31.bpo-42938.4Zn4Mp.rst +new file mode 100644 +index 0000000000000..7df65a156feab +--- /dev/null ++++ b/Misc/NEWS.d/next/Security/2021-01-18-09-27-31.bpo-42938.4Zn4Mp.rst +@@ -0,0 +1,2 @@ ++Avoid static buffers when computing the repr of :class:`ctypes.c_double` and ++:class:`ctypes.c_longdouble` values. +diff --git a/Modules/_ctypes/callproc.c b/Modules/_ctypes/callproc.c +index d1c190f359108..2bb289bce043f 100644 +--- a/Modules/_ctypes/callproc.c ++++ b/Modules/_ctypes/callproc.c +@@ -461,58 +461,47 @@ is_literal_char(unsigned char c) + static PyObject * + PyCArg_repr(PyCArgObject *self) + { +- char buffer[256]; + switch(self->tag) { + case 'b': + case 'B': +- sprintf(buffer, "", ++ return PyUnicode_FromFormat("", + self->tag, self->value.b); +- break; + case 'h': + case 'H': +- sprintf(buffer, "", ++ return PyUnicode_FromFormat("", + self->tag, self->value.h); +- break; + case 'i': + case 'I': +- sprintf(buffer, "", ++ return PyUnicode_FromFormat("", + self->tag, self->value.i); +- break; + case 'l': + case 'L': +- sprintf(buffer, "", ++ return PyUnicode_FromFormat("", + self->tag, self->value.l); +- break; + + case 'q': + case 'Q': +- sprintf(buffer, +-#ifdef MS_WIN32 +- "", +-#else +- "", +-#endif ++ return PyUnicode_FromFormat("", + self->tag, self->value.q); +- break; + case 'd': +- sprintf(buffer, "", +- self->tag, self->value.d); +- break; +- case 'f': +- sprintf(buffer, "", +- self->tag, self->value.f); +- break; +- ++ case 'f': { ++ PyObject *f = PyFloat_FromDouble((self->tag == 'f') ? self->value.f : self->value.d); ++ if (f == NULL) { ++ return NULL; ++ } ++ PyObject *result = PyUnicode_FromFormat("", self->tag, f); ++ Py_DECREF(f); ++ return result; ++ } + case 'c': + if (is_literal_char((unsigned char)self->value.c)) { +- sprintf(buffer, "", ++ return PyUnicode_FromFormat("", + self->tag, self->value.c); + } + else { +- sprintf(buffer, "", ++ return PyUnicode_FromFormat("", + self->tag, (unsigned char)self->value.c); + } +- break; + + /* Hm, are these 'z' and 'Z' codes useful at all? + Shouldn't they be replaced by the functionality of c_string +@@ -521,22 +510,20 @@ PyCArg_repr(PyCArgObject *self) + case 'z': + case 'Z': + case 'P': +- sprintf(buffer, "", ++ return PyUnicode_FromFormat("", + self->tag, self->value.p); + break; + + default: + if (is_literal_char((unsigned char)self->tag)) { +- sprintf(buffer, "", +- (unsigned char)self->tag, self); ++ return PyUnicode_FromFormat("", ++ (unsigned char)self->tag, (void *)self); + } + else { +- sprintf(buffer, "", +- (unsigned char)self->tag, self); ++ return PyUnicode_FromFormat("", ++ (unsigned char)self->tag, (void *)self); + } +- break; + } +- return PyUnicode_FromString(buffer); + } + + static PyMemberDef PyCArgType_members[] = { diff --git a/SOURCES/00359-CVE-2021-23336.patch b/SOURCES/00359-CVE-2021-23336.patch new file mode 100644 index 0000000..4a0c31e --- /dev/null +++ b/SOURCES/00359-CVE-2021-23336.patch @@ -0,0 +1,684 @@ +commit 9e77ec82c40ab59846f9447b7c483e7b8e368b16 +Author: Petr Viktorin +Date: Thu Mar 4 13:59:56 2021 +0100 + + CVE-2021-23336: Add `separator` argument to parse_qs; warn with default + + Partially backports https://bugs.python.org/issue42967 : [security] Address a web cache-poisoning issue reported in urllib.parse.parse_qsl(). + However, this solution is different than the upstream solution in Python 3.6.13. + + An optional argument seperator is added to specify the separator. + It is recommended to set it to '&' or ';' to match the application or proxy in use. + The default can be set with an env variable of a config file. + If neither the argument, env var or config file specifies a separator, "&" is used + but a warning is raised if parse_qs is used on input that contains ';'. + + Co-authors of the upstream change (who do not necessarily agree with this): + Co-authored-by: Adam Goldschmidt + Co-authored-by: Ken Jin <28750310+Fidget-Spinner@users.noreply.github.com> + Co-authored-by: Éric Araujo + +diff --git a/Doc/library/cgi.rst b/Doc/library/cgi.rst +index 41219eeaaba..ddecc0af23a 100644 +--- a/Doc/library/cgi.rst ++++ b/Doc/library/cgi.rst +@@ -277,13 +277,12 @@ These are useful if you want more control, or if you want to employ some of the + algorithms implemented in this module in other circumstances. + + +-.. function:: parse(fp=None, environ=os.environ, keep_blank_values=False, strict_parsing=False) ++.. function:: parse(fp=None, environ=os.environ, keep_blank_values=False, strict_parsing=False, separator=None) + + Parse a query in the environment or from a file (the file defaults to +- ``sys.stdin``). The *keep_blank_values* and *strict_parsing* parameters are ++ ``sys.stdin``). The *keep_blank_values*, *strict_parsing* and *separator* parameters are + passed to :func:`urllib.parse.parse_qs` unchanged. + +- + .. function:: parse_qs(qs, keep_blank_values=False, strict_parsing=False) + + This function is deprecated in this module. Use :func:`urllib.parse.parse_qs` +@@ -308,7 +307,6 @@ algorithms implemented in this module in other circumstances. + Note that this does not parse nested multipart parts --- use + :class:`FieldStorage` for that. + +- + .. function:: parse_header(string) + + Parse a MIME header (such as :mailheader:`Content-Type`) into a main value and a +diff --git a/Doc/library/urllib.parse.rst b/Doc/library/urllib.parse.rst +index 647af613a31..bcab7c142bc 100644 +--- a/Doc/library/urllib.parse.rst ++++ b/Doc/library/urllib.parse.rst +@@ -143,7 +143,7 @@ or on combining URL components into a URL string. + now raise :exc:`ValueError`. + + +-.. function:: parse_qs(qs, keep_blank_values=False, strict_parsing=False, encoding='utf-8', errors='replace', max_num_fields=None) ++.. function:: parse_qs(qs, keep_blank_values=False, strict_parsing=False, encoding='utf-8', errors='replace', max_num_fields=None, separator=None) + + Parse a query string given as a string argument (data of type + :mimetype:`application/x-www-form-urlencoded`). Data are returned as a +@@ -168,6 +168,15 @@ or on combining URL components into a URL string. + read. If set, then throws a :exc:`ValueError` if there are more than + *max_num_fields* fields read. + ++ The optional argument *separator* is the symbol to use for separating the ++ query arguments. It is recommended to set it to ``'&'`` or ``';'``. ++ It defaults to ``'&'``; a warning is raised if this default is used. ++ This default may be changed with the following environment variable settings: ++ ++ - ``PYTHON_URLLIB_QS_SEPARATOR='&'``: use only ``&`` as separator, without warning (as in Python 3.6.13+ or 3.10) ++ - ``PYTHON_URLLIB_QS_SEPARATOR=';'``: use only ``;`` as separator ++ - ``PYTHON_URLLIB_QS_SEPARATOR=legacy``: use both ``&`` and ``;`` (as in previous versions of Python) ++ + Use the :func:`urllib.parse.urlencode` function (with the ``doseq`` + parameter set to ``True``) to convert such dictionaries into query + strings. +@@ -204,6 +213,9 @@ or on combining URL components into a URL string. + read. If set, then throws a :exc:`ValueError` if there are more than + *max_num_fields* fields read. + ++ The optional argument *separator* is the symbol to use for separating the ++ query arguments. It works as in :py:func:`parse_qs`. ++ + Use the :func:`urllib.parse.urlencode` function to convert such lists of pairs into + query strings. + +@@ -213,7 +225,6 @@ or on combining URL components into a URL string. + .. versionchanged:: 3.6.8 + Added *max_num_fields* parameter. + +- + .. function:: urlunparse(parts) + + Construct a URL from a tuple as returned by ``urlparse()``. The *parts* +diff --git a/Lib/cgi.py b/Lib/cgi.py +index 56f243e09f0..5ab2a5d6af6 100755 +--- a/Lib/cgi.py ++++ b/Lib/cgi.py +@@ -117,7 +117,8 @@ log = initlog # The current logging function + # 0 ==> unlimited input + maxlen = 0 + +-def parse(fp=None, environ=os.environ, keep_blank_values=0, strict_parsing=0): ++def parse(fp=None, environ=os.environ, keep_blank_values=0, ++ strict_parsing=0, separator=None): + """Parse a query in the environment or from a file (default stdin) + + Arguments, all optional: +@@ -136,6 +137,8 @@ def parse(fp=None, environ=os.environ, keep_blank_values=0, strict_parsing=0): + strict_parsing: flag indicating what to do with parsing errors. + If false (the default), errors are silently ignored. + If true, errors raise a ValueError exception. ++ ++ separator: str. The symbol to use for separating the query arguments. + """ + if fp is None: + fp = sys.stdin +@@ -156,7 +159,7 @@ def parse(fp=None, environ=os.environ, keep_blank_values=0, strict_parsing=0): + if environ['REQUEST_METHOD'] == 'POST': + ctype, pdict = parse_header(environ['CONTENT_TYPE']) + if ctype == 'multipart/form-data': +- return parse_multipart(fp, pdict) ++ return parse_multipart(fp, pdict, separator=separator) + elif ctype == 'application/x-www-form-urlencoded': + clength = int(environ['CONTENT_LENGTH']) + if maxlen and clength > maxlen: +@@ -182,21 +185,21 @@ def parse(fp=None, environ=os.environ, keep_blank_values=0, strict_parsing=0): + return urllib.parse.parse_qs(qs, keep_blank_values, strict_parsing, + encoding=encoding) + +- + # parse query string function called from urlparse, + # this is done in order to maintain backward compatibility. +- +-def parse_qs(qs, keep_blank_values=0, strict_parsing=0): ++def parse_qs(qs, keep_blank_values=0, strict_parsing=0, separator=None): + """Parse a query given as a string argument.""" + warn("cgi.parse_qs is deprecated, use urllib.parse.parse_qs instead", + DeprecationWarning, 2) +- return urllib.parse.parse_qs(qs, keep_blank_values, strict_parsing) ++ return urllib.parse.parse_qs(qs, keep_blank_values, strict_parsing, ++ separator=separator) + +-def parse_qsl(qs, keep_blank_values=0, strict_parsing=0): ++def parse_qsl(qs, keep_blank_values=0, strict_parsing=0, separator=None): + """Parse a query given as a string argument.""" + warn("cgi.parse_qsl is deprecated, use urllib.parse.parse_qsl instead", + DeprecationWarning, 2) +- return urllib.parse.parse_qsl(qs, keep_blank_values, strict_parsing) ++ return urllib.parse.parse_qsl(qs, keep_blank_values, strict_parsing, ++ separator=separator) + + def parse_multipart(fp, pdict): + """Parse multipart input. +@@ -297,7 +300,6 @@ def parse_multipart(fp, pdict): + + return partdict + +- + def _parseparam(s): + while s[:1] == ';': + s = s[1:] +@@ -405,7 +407,7 @@ class FieldStorage: + def __init__(self, fp=None, headers=None, outerboundary=b'', + environ=os.environ, keep_blank_values=0, strict_parsing=0, + limit=None, encoding='utf-8', errors='replace', +- max_num_fields=None): ++ max_num_fields=None, separator=None): + """Constructor. Read multipart/* until last part. + + Arguments, all optional: +@@ -453,6 +455,7 @@ class FieldStorage: + self.keep_blank_values = keep_blank_values + self.strict_parsing = strict_parsing + self.max_num_fields = max_num_fields ++ self.separator = separator + if 'REQUEST_METHOD' in environ: + method = environ['REQUEST_METHOD'].upper() + self.qs_on_post = None +@@ -678,7 +681,7 @@ class FieldStorage: + query = urllib.parse.parse_qsl( + qs, self.keep_blank_values, self.strict_parsing, + encoding=self.encoding, errors=self.errors, +- max_num_fields=self.max_num_fields) ++ max_num_fields=self.max_num_fields, separator=self.separator) + self.list = [MiniFieldStorage(key, value) for key, value in query] + self.skip_lines() + +@@ -694,7 +697,7 @@ class FieldStorage: + query = urllib.parse.parse_qsl( + self.qs_on_post, self.keep_blank_values, self.strict_parsing, + encoding=self.encoding, errors=self.errors, +- max_num_fields=self.max_num_fields) ++ max_num_fields=self.max_num_fields, separator=self.separator) + self.list.extend(MiniFieldStorage(key, value) for key, value in query) + + klass = self.FieldStorageClass or self.__class__ +@@ -736,7 +739,8 @@ class FieldStorage: + + part = klass(self.fp, headers, ib, environ, keep_blank_values, + strict_parsing,self.limit-self.bytes_read, +- self.encoding, self.errors, max_num_fields) ++ self.encoding, self.errors, max_num_fields, ++ separator=self.separator) + + if max_num_fields is not None: + max_num_fields -= 1 +diff --git a/Lib/test/test_cgi.py b/Lib/test/test_cgi.py +index b3e2d4cce8e..5ae3e085e1e 100644 +--- a/Lib/test/test_cgi.py ++++ b/Lib/test/test_cgi.py +@@ -55,12 +55,9 @@ parse_strict_test_cases = [ + ("", ValueError("bad query field: ''")), + ("&", ValueError("bad query field: ''")), + ("&&", ValueError("bad query field: ''")), +- (";", ValueError("bad query field: ''")), +- (";&;", ValueError("bad query field: ''")), + # Should the next few really be valid? + ("=", {}), + ("=&=", {}), +- ("=;=", {}), + # This rest seem to make sense + ("=a", {'': ['a']}), + ("&=a", ValueError("bad query field: ''")), +@@ -75,8 +72,6 @@ parse_strict_test_cases = [ + ("a=a+b&b=b+c", {'a': ['a b'], 'b': ['b c']}), + ("a=a+b&a=b+a", {'a': ['a b', 'b a']}), + ("x=1&y=2.0&z=2-3.%2b0", {'x': ['1'], 'y': ['2.0'], 'z': ['2-3.+0']}), +- ("x=1;y=2.0&z=2-3.%2b0", {'x': ['1'], 'y': ['2.0'], 'z': ['2-3.+0']}), +- ("x=1;y=2.0;z=2-3.%2b0", {'x': ['1'], 'y': ['2.0'], 'z': ['2-3.+0']}), + ("Hbc5161168c542333633315dee1182227:key_store_seqid=400006&cuyer=r&view=bustomer&order_id=0bb2e248638833d48cb7fed300000f1b&expire=964546263&lobale=en-US&kid=130003.300038&ss=env", + {'Hbc5161168c542333633315dee1182227:key_store_seqid': ['400006'], + 'cuyer': ['r'], +@@ -164,6 +159,35 @@ class CgiTests(unittest.TestCase): + + env = {'QUERY_STRING': orig} + fs = cgi.FieldStorage(environ=env) ++ if isinstance(expect, dict): ++ # test dict interface ++ self.assertEqual(len(expect), len(fs)) ++ self.assertCountEqual(expect.keys(), fs.keys()) ++ self.assertEqual(fs.getvalue("nonexistent field", "default"), "default") ++ # test individual fields ++ for key in expect.keys(): ++ expect_val = expect[key] ++ self.assertIn(key, fs) ++ if len(expect_val) > 1: ++ self.assertEqual(fs.getvalue(key), expect_val) ++ else: ++ self.assertEqual(fs.getvalue(key), expect_val[0]) ++ ++ def test_separator(self): ++ parse_semicolon = [ ++ ("x=1;y=2.0", {'x': ['1'], 'y': ['2.0']}), ++ ("x=1;y=2.0;z=2-3.%2b0", {'x': ['1'], 'y': ['2.0'], 'z': ['2-3.+0']}), ++ (";", ValueError("bad query field: ''")), ++ (";;", ValueError("bad query field: ''")), ++ ("=;a", ValueError("bad query field: 'a'")), ++ (";b=a", ValueError("bad query field: ''")), ++ ("b;=a", ValueError("bad query field: 'b'")), ++ ("a=a+b;b=b+c", {'a': ['a b'], 'b': ['b c']}), ++ ("a=a+b;a=b+a", {'a': ['a b', 'b a']}), ++ ] ++ for orig, expect in parse_semicolon: ++ env = {'QUERY_STRING': orig} ++ fs = cgi.FieldStorage(separator=';', environ=env) + if isinstance(expect, dict): + # test dict interface + self.assertEqual(len(expect), len(fs)) +diff --git a/Lib/test/test_urlparse.py b/Lib/test/test_urlparse.py +index 68f633ca3a7..1ec86ba0fc2 100644 +--- a/Lib/test/test_urlparse.py ++++ b/Lib/test/test_urlparse.py +@@ -2,6 +2,11 @@ import sys + import unicodedata + import unittest + import urllib.parse ++from test.support import EnvironmentVarGuard ++from warnings import catch_warnings ++import tempfile ++import contextlib ++import os.path + + RFC1808_BASE = "http://a/b/c/d;p?q#f" + RFC2396_BASE = "http://a/b/c/d;p?q" +@@ -32,6 +37,9 @@ parse_qsl_test_cases = [ + (b"&a=b", [(b'a', b'b')]), + (b"a=a+b&b=b+c", [(b'a', b'a b'), (b'b', b'b c')]), + (b"a=1&a=2", [(b'a', b'1'), (b'a', b'2')]), ++] ++ ++parse_qsl_test_cases_semicolon = [ + (";", []), + (";;", []), + (";a=b", [('a', 'b')]), +@@ -44,6 +52,21 @@ parse_qsl_test_cases = [ + (b"a=1;a=2", [(b'a', b'1'), (b'a', b'2')]), + ] + ++parse_qsl_test_cases_legacy = [ ++ (b"a=1;a=2&a=3", [(b'a', b'1'), (b'a', b'2'), (b'a', b'3')]), ++ (b"a=1;b=2&c=3", [(b'a', b'1'), (b'b', b'2'), (b'c', b'3')]), ++ (b"a=1&b=2&c=3;", [(b'a', b'1'), (b'b', b'2'), (b'c', b'3')]), ++] ++ ++parse_qsl_test_cases_warn = [ ++ (";a=b", [(';a', 'b')]), ++ ("a=a+b;b=b+c", [('a', 'a b;b=b c')]), ++ (b";a=b", [(b';a', b'b')]), ++ (b"a=a+b;b=b+c", [(b'a', b'a b;b=b c')]), ++ ("a=1;a=2&a=3", [('a', '1;a=2'), ('a', '3')]), ++ (b"a=1;a=2&a=3", [(b'a', b'1;a=2'), (b'a', b'3')]), ++] ++ + # Each parse_qs testcase is a two-tuple that contains + # a string with the query and a dictionary with the expected result. + +@@ -68,6 +91,9 @@ parse_qs_test_cases = [ + (b"&a=b", {b'a': [b'b']}), + (b"a=a+b&b=b+c", {b'a': [b'a b'], b'b': [b'b c']}), + (b"a=1&a=2", {b'a': [b'1', b'2']}), ++] ++ ++parse_qs_test_cases_semicolon = [ + (";", {}), + (";;", {}), + (";a=b", {'a': ['b']}), +@@ -80,6 +106,24 @@ parse_qs_test_cases = [ + (b"a=1;a=2", {b'a': [b'1', b'2']}), + ] + ++parse_qs_test_cases_legacy = [ ++ ("a=1;a=2&a=3", {'a': ['1', '2', '3']}), ++ ("a=1;b=2&c=3", {'a': ['1'], 'b': ['2'], 'c': ['3']}), ++ ("a=1&b=2&c=3;", {'a': ['1'], 'b': ['2'], 'c': ['3']}), ++ (b"a=1;a=2&a=3", {b'a': [b'1', b'2', b'3']}), ++ (b"a=1;b=2&c=3", {b'a': [b'1'], b'b': [b'2'], b'c': [b'3']}), ++ (b"a=1&b=2&c=3;", {b'a': [b'1'], b'b': [b'2'], b'c': [b'3']}), ++] ++ ++parse_qs_test_cases_warn = [ ++ (";a=b", {';a': ['b']}), ++ ("a=a+b;b=b+c", {'a': ['a b;b=b c']}), ++ (b";a=b", {b';a': [b'b']}), ++ (b"a=a+b;b=b+c", {b'a':[ b'a b;b=b c']}), ++ ("a=1;a=2&a=3", {'a': ['1;a=2', '3']}), ++ (b"a=1;a=2&a=3", {b'a': [b'1;a=2', b'3']}), ++] ++ + class UrlParseTestCase(unittest.TestCase): + + def checkRoundtrips(self, url, parsed, split): +@@ -152,6 +196,40 @@ class UrlParseTestCase(unittest.TestCase): + self.assertEqual(result, expect_without_blanks, + "Error parsing %r" % orig) + ++ def test_qs_default_warn(self): ++ for orig, expect in parse_qs_test_cases_warn: ++ with self.subTest(orig=orig, expect=expect): ++ with catch_warnings(record=True) as w: ++ result = urllib.parse.parse_qs(orig, keep_blank_values=True) ++ self.assertEqual(result, expect, "Error parsing %r" % orig) ++ self.assertEqual(len(w), 1) ++ self.assertEqual(w[0].category, urllib.parse._QueryStringSeparatorWarning) ++ ++ def test_qsl_default_warn(self): ++ for orig, expect in parse_qsl_test_cases_warn: ++ with self.subTest(orig=orig, expect=expect): ++ with catch_warnings(record=True) as w: ++ result = urllib.parse.parse_qsl(orig, keep_blank_values=True) ++ self.assertEqual(result, expect, "Error parsing %r" % orig) ++ self.assertEqual(len(w), 1) ++ self.assertEqual(w[0].category, urllib.parse._QueryStringSeparatorWarning) ++ ++ def test_default_qs_no_warnings(self): ++ for orig, expect in parse_qs_test_cases: ++ with self.subTest(orig=orig, expect=expect): ++ with catch_warnings(record=True) as w: ++ result = urllib.parse.parse_qs(orig, keep_blank_values=True) ++ self.assertEqual(result, expect, "Error parsing %r" % orig) ++ self.assertEqual(len(w), 0) ++ ++ def test_default_qsl_no_warnings(self): ++ for orig, expect in parse_qsl_test_cases: ++ with self.subTest(orig=orig, expect=expect): ++ with catch_warnings(record=True) as w: ++ result = urllib.parse.parse_qsl(orig, keep_blank_values=True) ++ self.assertEqual(result, expect, "Error parsing %r" % orig) ++ self.assertEqual(len(w), 0) ++ + def test_roundtrips(self): + str_cases = [ + ('file:///tmp/junk.txt', +@@ -885,8 +963,151 @@ class UrlParseTestCase(unittest.TestCase): + with self.assertRaises(ValueError): + urllib.parse.parse_qs('&'.join(['a=a']*11), max_num_fields=10) + with self.assertRaises(ValueError): +- urllib.parse.parse_qs(';'.join(['a=a']*11), max_num_fields=10) ++ urllib.parse.parse_qs(';'.join(['a=a']*11), separator=';', max_num_fields=10) ++ with self.assertRaises(ValueError): ++ urllib.parse.parse_qs('SEP'.join(['a=a']*11), separator='SEP', max_num_fields=10) + urllib.parse.parse_qs('&'.join(['a=a']*10), max_num_fields=10) ++ urllib.parse.parse_qs(';'.join(['a=a']*10), separator=';', max_num_fields=10) ++ urllib.parse.parse_qs('SEP'.join(['a=a']*10), separator='SEP', max_num_fields=10) ++ ++ def test_parse_qs_separator_bytes(self): ++ expected = {b'a': [b'1'], b'b': [b'2']} ++ ++ result = urllib.parse.parse_qs(b'a=1;b=2', separator=b';') ++ self.assertEqual(result, expected) ++ result = urllib.parse.parse_qs(b'a=1;b=2', separator=';') ++ self.assertEqual(result, expected) ++ result = urllib.parse.parse_qs('a=1;b=2', separator=';') ++ self.assertEqual(result, {'a': ['1'], 'b': ['2']}) ++ ++ @contextlib.contextmanager ++ def _qsl_sep_config(self, sep): ++ """Context for the given parse_qsl default separator configured in config file""" ++ old_filename = urllib.parse._QS_SEPARATOR_CONFIG_FILENAME ++ urllib.parse._default_qs_separator = None ++ try: ++ with tempfile.TemporaryDirectory() as tmpdirname: ++ filename = os.path.join(tmpdirname, 'conf.cfg') ++ with open(filename, 'w') as file: ++ file.write(f'[parse_qs]\n') ++ file.write(f'PYTHON_URLLIB_QS_SEPARATOR = {sep}') ++ urllib.parse._QS_SEPARATOR_CONFIG_FILENAME = filename ++ yield ++ finally: ++ urllib.parse._QS_SEPARATOR_CONFIG_FILENAME = old_filename ++ urllib.parse._default_qs_separator = None ++ ++ def test_parse_qs_separator_semicolon(self): ++ for orig, expect in parse_qs_test_cases_semicolon: ++ with self.subTest(orig=orig, expect=expect, method='arg'): ++ result = urllib.parse.parse_qs(orig, separator=';') ++ self.assertEqual(result, expect, "Error parsing %r" % orig) ++ with self.subTest(orig=orig, expect=expect, method='env'): ++ with EnvironmentVarGuard() as environ, catch_warnings(record=True) as w: ++ environ['PYTHON_URLLIB_QS_SEPARATOR'] = ';' ++ result = urllib.parse.parse_qs(orig) ++ self.assertEqual(result, expect, "Error parsing %r" % orig) ++ self.assertEqual(len(w), 0) ++ with self.subTest(orig=orig, expect=expect, method='conf'): ++ with self._qsl_sep_config(';'), catch_warnings(record=True) as w: ++ result = urllib.parse.parse_qs(orig) ++ self.assertEqual(result, expect, "Error parsing %r" % orig) ++ self.assertEqual(len(w), 0) ++ ++ def test_parse_qsl_separator_semicolon(self): ++ for orig, expect in parse_qsl_test_cases_semicolon: ++ with self.subTest(orig=orig, expect=expect, method='arg'): ++ result = urllib.parse.parse_qsl(orig, separator=';') ++ self.assertEqual(result, expect, "Error parsing %r" % orig) ++ with self.subTest(orig=orig, expect=expect, method='env'): ++ with EnvironmentVarGuard() as environ, catch_warnings(record=True) as w: ++ environ['PYTHON_URLLIB_QS_SEPARATOR'] = ';' ++ result = urllib.parse.parse_qsl(orig) ++ self.assertEqual(result, expect, "Error parsing %r" % orig) ++ self.assertEqual(len(w), 0) ++ with self.subTest(orig=orig, expect=expect, method='conf'): ++ with self._qsl_sep_config(';'), catch_warnings(record=True) as w: ++ result = urllib.parse.parse_qsl(orig) ++ self.assertEqual(result, expect, "Error parsing %r" % orig) ++ self.assertEqual(len(w), 0) ++ ++ def test_parse_qs_separator_legacy(self): ++ for orig, expect in parse_qs_test_cases_legacy: ++ with self.subTest(orig=orig, expect=expect, method='env'): ++ with EnvironmentVarGuard() as environ, catch_warnings(record=True) as w: ++ environ['PYTHON_URLLIB_QS_SEPARATOR'] = 'legacy' ++ result = urllib.parse.parse_qs(orig) ++ self.assertEqual(result, expect, "Error parsing %r" % orig) ++ self.assertEqual(len(w), 0) ++ with self.subTest(orig=orig, expect=expect, method='conf'): ++ with self._qsl_sep_config('legacy'), catch_warnings(record=True) as w: ++ result = urllib.parse.parse_qs(orig) ++ self.assertEqual(result, expect, "Error parsing %r" % orig) ++ self.assertEqual(len(w), 0) ++ ++ def test_parse_qsl_separator_legacy(self): ++ for orig, expect in parse_qsl_test_cases_legacy: ++ with self.subTest(orig=orig, expect=expect, method='env'): ++ with EnvironmentVarGuard() as environ, catch_warnings(record=True) as w: ++ environ['PYTHON_URLLIB_QS_SEPARATOR'] = 'legacy' ++ result = urllib.parse.parse_qsl(orig) ++ self.assertEqual(result, expect, "Error parsing %r" % orig) ++ self.assertEqual(len(w), 0) ++ with self.subTest(orig=orig, expect=expect, method='conf'): ++ with self._qsl_sep_config('legacy'), catch_warnings(record=True) as w: ++ result = urllib.parse.parse_qsl(orig) ++ self.assertEqual(result, expect, "Error parsing %r" % orig) ++ self.assertEqual(len(w), 0) ++ ++ def test_parse_qs_separator_bad_value_env_or_config(self): ++ for bad_sep in '', 'abc', 'safe', '&;', 'SEP': ++ with self.subTest(bad_sep, method='env'): ++ with EnvironmentVarGuard() as environ, catch_warnings(record=True) as w: ++ environ['PYTHON_URLLIB_QS_SEPARATOR'] = bad_sep ++ with self.assertRaises(ValueError): ++ urllib.parse.parse_qsl('a=1;b=2') ++ with self.subTest(bad_sep, method='conf'): ++ with self._qsl_sep_config('bad_sep'), catch_warnings(record=True) as w: ++ with self.assertRaises(ValueError): ++ urllib.parse.parse_qsl('a=1;b=2') ++ ++ def test_parse_qs_separator_bad_value_arg(self): ++ for bad_sep in True, {}, '': ++ with self.subTest(bad_sep): ++ with self.assertRaises(ValueError): ++ urllib.parse.parse_qsl('a=1;b=2', separator=bad_sep) ++ ++ def test_parse_qs_separator_num_fields(self): ++ for qs, sep in ( ++ ('a&b&c', '&'), ++ ('a;b;c', ';'), ++ ('a&b;c', 'legacy'), ++ ): ++ with self.subTest(qs=qs, sep=sep): ++ with EnvironmentVarGuard() as environ, catch_warnings(record=True) as w: ++ if sep != 'legacy': ++ with self.assertRaises(ValueError): ++ urllib.parse.parse_qsl(qs, separator=sep, max_num_fields=2) ++ if sep: ++ environ['PYTHON_URLLIB_QS_SEPARATOR'] = sep ++ with self.assertRaises(ValueError): ++ urllib.parse.parse_qsl(qs, max_num_fields=2) ++ ++ def test_parse_qs_separator_priority(self): ++ # env variable trumps config file ++ with self._qsl_sep_config('~'), EnvironmentVarGuard() as environ: ++ environ['PYTHON_URLLIB_QS_SEPARATOR'] = '!' ++ result = urllib.parse.parse_qs('a=1!b=2~c=3') ++ self.assertEqual(result, {'a': ['1'], 'b': ['2~c=3']}) ++ # argument trumps config file ++ with self._qsl_sep_config('~'): ++ result = urllib.parse.parse_qs('a=1$b=2~c=3', separator='$') ++ self.assertEqual(result, {'a': ['1'], 'b': ['2~c=3']}) ++ # argument trumps env variable ++ with EnvironmentVarGuard() as environ: ++ environ['PYTHON_URLLIB_QS_SEPARATOR'] = '~' ++ result = urllib.parse.parse_qs('a=1$b=2~c=3', separator='$') ++ self.assertEqual(result, {'a': ['1'], 'b': ['2~c=3']}) + + def test_urlencode_sequences(self): + # Other tests incidentally urlencode things; test non-covered cases: +diff --git a/Lib/urllib/parse.py b/Lib/urllib/parse.py +index fa8827a9fa7..57b8fcf8bbd 100644 +--- a/Lib/urllib/parse.py ++++ b/Lib/urllib/parse.py +@@ -28,6 +28,7 @@ test_urlparse.py provides a good indicator of parsing behavior. + """ + + import re ++import os + import sys + import collections + +@@ -644,7 +645,8 @@ def unquote(string, encoding='utf-8', errors='replace'): + + + def parse_qs(qs, keep_blank_values=False, strict_parsing=False, +- encoding='utf-8', errors='replace', max_num_fields=None): ++ encoding='utf-8', errors='replace', max_num_fields=None, ++ separator=None): + """Parse a query given as a string argument. + + Arguments: +@@ -673,7 +675,8 @@ def parse_qs(qs, keep_blank_values=False, strict_parsing=False, + parsed_result = {} + pairs = parse_qsl(qs, keep_blank_values, strict_parsing, + encoding=encoding, errors=errors, +- max_num_fields=max_num_fields) ++ max_num_fields=max_num_fields, ++ separator=separator) + for name, value in pairs: + if name in parsed_result: + parsed_result[name].append(value) +@@ -681,9 +684,16 @@ def parse_qs(qs, keep_blank_values=False, strict_parsing=False, + parsed_result[name] = [value] + return parsed_result + ++class _QueryStringSeparatorWarning(RuntimeWarning): ++ """Warning for using default `separator` in parse_qs or parse_qsl""" ++ ++# The default "separator" for parse_qsl can be specified in a config file. ++# It's cached after first read. ++_QS_SEPARATOR_CONFIG_FILENAME = '/etc/python/urllib.cfg' ++_default_qs_separator = None + + def parse_qsl(qs, keep_blank_values=False, strict_parsing=False, +- encoding='utf-8', errors='replace', max_num_fields=None): ++ encoding='utf-8', errors='replace', max_num_fields=None, separator=None): + """Parse a query given as a string argument. + + Arguments: +@@ -710,15 +720,77 @@ def parse_qsl(qs, keep_blank_values=False, strict_parsing=False, + """ + qs, _coerce_result = _coerce_args(qs) + ++ if isinstance(separator, bytes): ++ separator = separator.decode('ascii') ++ ++ if (not separator or (not isinstance(separator, (str, bytes)))) and separator is not None: ++ raise ValueError("Separator must be of type string or bytes.") ++ ++ # Used when both "&" and ";" act as separators. (Need a non-string value.) ++ _legacy = object() ++ ++ if separator is None: ++ global _default_qs_separator ++ separator = _default_qs_separator ++ envvar_name = 'PYTHON_URLLIB_QS_SEPARATOR' ++ if separator is None: ++ # Set default separator from environment variable ++ separator = os.environ.get(envvar_name) ++ config_source = 'environment variable' ++ if separator is None: ++ # Set default separator from the configuration file ++ try: ++ file = open(_QS_SEPARATOR_CONFIG_FILENAME) ++ except FileNotFoundError: ++ pass ++ else: ++ with file: ++ import configparser ++ config = configparser.ConfigParser( ++ interpolation=None, ++ comment_prefixes=('#', ), ++ ) ++ config.read_file(file) ++ separator = config.get('parse_qs', envvar_name, fallback=None) ++ _default_qs_separator = separator ++ config_source = _QS_SEPARATOR_CONFIG_FILENAME ++ if separator is None: ++ # The default is '&', but warn if not specified explicitly ++ if ';' in qs: ++ from warnings import warn ++ warn("The default separator of urllib.parse.parse_qsl and " ++ + "parse_qs was changed to '&' to avoid a web cache " ++ + "poisoning issue (CVE-2021-23336). " ++ + "By default, semicolons no longer act as query field " ++ + "separators. " ++ + "See https://access.redhat.com/articles/5860431 for " ++ + "more details.", ++ _QueryStringSeparatorWarning, stacklevel=2) ++ separator = '&' ++ elif separator == 'legacy': ++ separator = _legacy ++ elif len(separator) != 1: ++ raise ValueError( ++ f'{envvar_name} (from {config_source}) must contain ' ++ + '1 character, or "legacy". See ' ++ + 'https://access.redhat.com/articles/5860431 for more details.' ++ ) ++ + # If max_num_fields is defined then check that the number of fields + # is less than max_num_fields. This prevents a memory exhaustion DOS + # attack via post bodies with many fields. + if max_num_fields is not None: +- num_fields = 1 + qs.count('&') + qs.count(';') ++ if separator is _legacy: ++ num_fields = 1 + qs.count('&') + qs.count(';') ++ else: ++ num_fields = 1 + qs.count(separator) + if max_num_fields < num_fields: + raise ValueError('Max number of fields exceeded') + +- pairs = [s2 for s1 in qs.split('&') for s2 in s1.split(';')] ++ if separator is _legacy: ++ pairs = [s2 for s1 in qs.split('&') for s2 in s1.split(';')] ++ else: ++ pairs = [s1 for s1 in qs.split(separator)] + r = [] + for name_value in pairs: + if not name_value and not strict_parsing: +diff --git a/Misc/NEWS.d/next/Security/2021-02-14-15-59-16.bpo-42967.YApqDS.rst b/Misc/NEWS.d/next/Security/2021-02-14-15-59-16.bpo-42967.YApqDS.rst +new file mode 100644 +index 00000000000..bc82c963067 +--- /dev/null ++++ b/Misc/NEWS.d/next/Security/2021-02-14-15-59-16.bpo-42967.YApqDS.rst +@@ -0,0 +1 @@ ++Make it possible to fix web cache poisoning vulnerability by allowing the user to choose a custom separator query args. diff --git a/SPECS/python3.spec b/SPECS/python3.spec index 5e24c4e..1f4699d 100644 --- a/SPECS/python3.spec +++ b/SPECS/python3.spec @@ -14,7 +14,7 @@ URL: https://www.python.org/ # WARNING When rebasing to a new Python version, # remember to update the python3-docs package as well Version: %{pybasever}.8 -Release: 31%{?dist} +Release: 37%{?dist} License: Python @@ -56,6 +56,15 @@ License: Python %bcond_with valgrind %endif +# https://fedoraproject.org/wiki/Changes/Python_Upstream_Architecture_Names +# For a very long time we have converted "upstream architecture names" to "Fedora names". +# This made sense at the time, see https://github.com/pypa/manylinux/issues/687#issuecomment-666362947 +# However, with manylinux wheels popularity growth, this is now a problem. +# Wheels built on a Linux that doesn't do this were not compatible with ours and vice versa. +# We now have a compatibility layer to workaround a problem, +# but we also no longer use the legacy arch names in Fedora 34+. +# This bcond controls the behavior. The defaults should be good for anybody. +%bcond_without legacy_archnames # ================================== # Notes from bootstraping Python 3.6 @@ -111,8 +120,21 @@ License: Python %global LDVERSION_optimized %{pybasever}%{ABIFLAGS_optimized} %global LDVERSION_debug %{pybasever}%{ABIFLAGS_debug} -%global SOABI_optimized cpython-%{pyshortver}%{ABIFLAGS_optimized}-%{_arch}-linux%{_gnu} -%global SOABI_debug cpython-%{pyshortver}%{ABIFLAGS_debug}-%{_arch}-linux%{_gnu} +# When we use the upstream arch triplets, we convert them from the legacy ones +# This is reversed in prep when %%with legacy_archnames, so we keep both macros +%global platform_triplet_legacy %{_arch}-linux%{_gnu} +%global platform_triplet_upstream %{expand:%(echo %{platform_triplet_legacy} | sed -E \\ + -e 's/^arm(eb)?-linux-gnueabi$/arm\\1-linux-gnueabihf/' \\ + -e 's/^mips64(el)?-linux-gnu$/mips64\\1-linux-gnuabi64/' \\ + -e 's/^ppc(64)?(le)?-linux-gnu$/powerpc\\1\\2-linux-gnu/')} +%if %{with legacy_archnames} +%global platform_triplet %{platform_triplet_legacy} +%else +%global platform_triplet %{platform_triplet_upstream} +%endif + +%global SOABI_optimized cpython-%{pyshortver}%{ABIFLAGS_optimized}-%{platform_triplet} +%global SOABI_debug cpython-%{pyshortver}%{ABIFLAGS_debug}-%{platform_triplet} # All bytecode files are in a __pycache__ subdirectory, with a name # reflecting the version of the bytecode. @@ -329,10 +351,6 @@ Patch251: 00251-change-user-install-location.patch # Original proposal: https://bugzilla.redhat.com/show_bug.cgi?id=1404918 Patch262: 00262-pep538_coerce_legacy_c_locale.patch -# 00274 # -# Upstream uses Debian-style architecture naming. Change to match Fedora. -Patch274: 00274-fix-arch-names.patch - # 00294 # # Define TLS cipher suite on build time depending # on the OpenSSL default cipher suite selection. @@ -519,6 +537,60 @@ Patch351: 00351-avoid-infinite-loop-in-the-tarfile-module.patch # Fixed upstream: https://bugs.python.org/issue41004 Patch352: 00352-resolve-hash-collisions-for-ipv4interface-and-ipv6interface.patch +# 00353 # +# Original names for architectures with different names downstream +# +# https://fedoraproject.org/wiki/Changes/Python_Upstream_Architecture_Names +# +# Pythons in RHEL/Fedora used different names for some architectures +# than upstream and other distros (for example ppc64 vs. powerpc64). +# This was patched in patch 274, now it is sedded if %%with legacy_archnames. +# +# That meant that an extension built with the default upstream settings +# (on other distro or as an manylinux wheel) could not been found by Python +# on RHEL/Fedora because it had a different suffix. +# This patch adds the legacy names to importlib so Python is able +# to import extensions with a legacy architecture name in its +# file name. +# It work both ways, so it support both %%with and %%without legacy_archnames. +# +# WARNING: This patch has no effect on Python built with bootstrap +# enabled because Python/importlib_external.h is not regenerated +# and therefore Python during bootstrap contains importlib from +# upstream without this feature. It's possible to include +# Python/importlib_external.h to this patch but it'd make rebasing +# a nightmare because it's basically a binary file. +Patch353: 00353-architecture-names-upstream-downstream.patch + +# 00354 # +# Reject control chars in HTTP method in http.client to prevent +# HTTP header injection +# Fixed ustream: https://bugs.python.org/issue39603 +Patch354: 00354-cve-2020-26116-http-request-method-crlf-injection-in-httplib.patch + +# 00355 # +# No longer call eval() on content received via HTTP in the CJK codec tests +# Fixed upstream: https://bugs.python.org/issue41944 +Patch355: 00355-CVE-2020-27619.patch + +# 00356 # +# options -a and -k for pathfix.py used in %%py3_shebang_fix +# Upstream: https://github.com/python/cpython/commit/c71c54c62600fd721baed3c96709e3d6e9c33817 +Patch356: 00356-k_and_a_options_for_pathfix.patch + +# 00357 # +# CVE-2021-3177 stack-based buffer overflow in PyCArg_repr in _ctypes/callproc.c +# Upstream: https://bugs.python.org/issue42938 +# Main BZ: https://bugzilla.redhat.com/show_bug.cgi?id=1918168 +Patch357: 00357-CVE-2021-3177.patch + +# 00359 # +# CVE-2021-23336 python: Web Cache Poisoning via urllib.parse.parse_qsl and +# urllib.parse.parse_qs by using a semicolon in query parameters +# Upstream: https://bugs.python.org/issue42967 +# Main BZ: https://bugzilla.redhat.com/show_bug.cgi?id=1928904 +Patch359: 00359-CVE-2021-23336.patch + # (New patches go here ^^^) # # When adding new patches to "python" and "python3" in Fedora, EL, etc., @@ -817,7 +889,6 @@ rm Lib/ensurepip/_bundled/*.whl %patch251 -p1 %patch262 -p1 -%patch274 -p1 %patch294 -p1 %patch316 -p1 %patch317 -p1 @@ -841,11 +912,23 @@ rm Lib/ensurepip/_bundled/*.whl git apply %{PATCH351} %patch352 -p1 +%patch353 -p1 +%patch354 -p1 +%patch355 -p1 +%patch356 -p1 +%patch357 -p1 +%patch359 -p1 # Remove files that should be generated by the build # (This is after patching, so that we can use patches directly from upstream) rm configure pyconfig.h.in +# When we use the legacy arch names, we need to change them in configure.ac +%if %{with legacy_archnames} +sed -i configure.ac \ + -e 's/\b%{platform_triplet_upstream}\b/%{platform_triplet_legacy}/' +%endif + # ====================================================== # Configuring and building the code: @@ -926,6 +1009,9 @@ BuildPython() { $ExtraConfigArgs \ %{nil} + # Regenerate generated importlib frozen modules (see patch 353) + %make_build CFLAGS_NODIST="$CFLAGS_NODIST $MoreCFlags" regen-importlib + # Invoke the build %make_build CFLAGS_NODIST="$CFLAGS_NODIST $MoreCFlags" @@ -1125,7 +1211,7 @@ do LD_LIBRARY_PATH=./build/optimized ./build/optimized/python \ Tools/scripts/pathfix.py \ -i "%{_libexecdir}/platform-python${LDVersion}" -pn \ - %{buildroot}%{pylibdir}/config-${LDVersion}-%{_arch}-linux%{_gnu}/python-config.py + %{buildroot}%{pylibdir}/config-${LDVersion}-%{platform_triplet}/python-config.py done # Remove tests for python3-tools which was removed in @@ -1545,8 +1631,8 @@ fi # "Makefile" and the config-32/64.h file are needed by # distutils/sysconfig.py:_init_posix(), so we include them in the core # package, along with their parent directories (bug 531901): -%dir %{pylibdir}/config-%{LDVERSION_optimized}-%{_arch}-linux%{_gnu}/ -%{pylibdir}/config-%{LDVERSION_optimized}-%{_arch}-linux%{_gnu}/Makefile +%dir %{pylibdir}/config-%{LDVERSION_optimized}-%{platform_triplet}/ +%{pylibdir}/config-%{LDVERSION_optimized}-%{platform_triplet}/Makefile %dir %{_includedir}/python%{LDVERSION_optimized}/ %{_includedir}/python%{LDVERSION_optimized}/%{_pyconfig_h} @@ -1560,8 +1646,8 @@ fi %{_bindir}/2to3 # TODO: Remove 2to3-3.7 once rebased to 3.7 %{_bindir}/2to3-%{pybasever} -%{pylibdir}/config-%{LDVERSION_optimized}-%{_arch}-linux%{_gnu}/* -%exclude %{pylibdir}/config-%{LDVERSION_optimized}-%{_arch}-linux%{_gnu}/Makefile +%{pylibdir}/config-%{LDVERSION_optimized}-%{platform_triplet}/* +%exclude %{pylibdir}/config-%{LDVERSION_optimized}-%{platform_triplet}/Makefile %exclude %{pylibdir}/distutils/command/wininst-*.exe %{_includedir}/python%{LDVERSION_optimized}/*.h %exclude %{_includedir}/python%{LDVERSION_optimized}/%{_pyconfig_h} @@ -1709,7 +1795,7 @@ fi %{_libdir}/%{py_INSTSONAME_debug} # Analog of the -devel subpackage's files: -%{pylibdir}/config-%{LDVERSION_debug}-%{_arch}-linux%{_gnu} +%{pylibdir}/config-%{LDVERSION_debug}-%{platform_triplet} %{_includedir}/python%{LDVERSION_debug} %exclude %{_bindir}/python%{LDVERSION_debug}-config @@ -1757,6 +1843,31 @@ fi # ====================================================== %changelog +* Thu Mar 04 2021 Petr Viktorin - 3.6.8-37 +- Fix for CVE-2021-23336 +Resolves: rhbz#1928904 + +* Fri Jan 22 2021 Lumír Balhar - 3.6.8-36 +- Fix for CVE-2021-3177 +Resolves: rhbz#1918168 + +* Mon Jan 18 2021 Lumír Balhar - 3.6.8-35 +- New options -a and -k for pathfix.py script backported from upstream +Resolves: rhbz#1917691 + +* Fri Dec 04 2020 Charalampos Stratakis - 3.6.8-34 +- Security fix for CVE-2020-27619: eval() call on content received via HTTP in the CJK codec tests +Resolves: rhbz#1890237 + +* Tue Nov 24 2020 Lumír Balhar - 3.6.8-33 +- Add support for upstream architecture names +https://fedoraproject.org/wiki/Changes/Python_Upstream_Architecture_Names +Resolves: rhbz#1868003 + +* Mon Nov 09 2020 Charalampos Stratakis - 3.6.8-32 +- Security fix for CVE-2020-26116: Reject control chars in HTTP method in http.client +Resolves: rhbz#1883257 + * Mon Aug 17 2020 Tomas Orsava - 3.6.8-31 - Avoid infinite loop when reading specially crafted TAR files (CVE-2019-20907) Resolves: rhbz#1856481