import python3-3.6.8-37.el8
This commit is contained in:
parent
87132fa408
commit
7c0aac331f
@ -1,58 +0,0 @@
|
||||
diff -up Python-3.5.0/configure.ac.than Python-3.5.0/configure.ac
|
||||
--- Python-3.5.0/configure.ac.than 2015-11-13 11:51:32.039560172 -0500
|
||||
+++ Python-3.5.0/configure.ac 2015-11-13 11:52:11.670168157 -0500
|
||||
@@ -788,9 +788,9 @@ cat >> conftest.c <<EOF
|
||||
alpha-linux-gnu
|
||||
# elif defined(__ARM_EABI__) && defined(__ARM_PCS_VFP)
|
||||
# if defined(__ARMEL__)
|
||||
- arm-linux-gnueabihf
|
||||
+ arm-linux-gnueabi
|
||||
# else
|
||||
- armeb-linux-gnueabihf
|
||||
+ armeb-linux-gnueabi
|
||||
# endif
|
||||
# elif defined(__ARM_EABI__) && !defined(__ARM_PCS_VFP)
|
||||
# if defined(__ARMEL__)
|
||||
@@ -810,7 +810,7 @@ cat >> conftest.c <<EOF
|
||||
# elif _MIPS_SIM == _ABIN32
|
||||
mips64el-linux-gnuabin32
|
||||
# elif _MIPS_SIM == _ABI64
|
||||
- mips64el-linux-gnuabi64
|
||||
+ mips64el-linux-gnu
|
||||
# else
|
||||
# error unknown platform triplet
|
||||
# endif
|
||||
@@ -820,7 +820,7 @@ cat >> conftest.c <<EOF
|
||||
# elif _MIPS_SIM == _ABIN32
|
||||
mips64-linux-gnuabin32
|
||||
# elif _MIPS_SIM == _ABI64
|
||||
- mips64-linux-gnuabi64
|
||||
+ mips64-linux-gnu
|
||||
# else
|
||||
# error unknown platform triplet
|
||||
# endif
|
||||
@@ -830,9 +830,9 @@ cat >> conftest.c <<EOF
|
||||
powerpc-linux-gnuspe
|
||||
# elif defined(__powerpc64__)
|
||||
# if defined(__LITTLE_ENDIAN__)
|
||||
- powerpc64le-linux-gnu
|
||||
+ ppc64le-linux-gnu
|
||||
# else
|
||||
- powerpc64-linux-gnu
|
||||
+ ppc64-linux-gnu
|
||||
# endif
|
||||
# elif defined(__powerpc__)
|
||||
powerpc-linux-gnu
|
||||
diff --git a/config.sub b/config.sub
|
||||
index 40ea5df..932128b 100755
|
||||
--- a/config.sub
|
||||
+++ b/config.sub
|
||||
@@ -1045,7 +1045,7 @@ case $basic_machine in
|
||||
;;
|
||||
ppc64) basic_machine=powerpc64-unknown
|
||||
;;
|
||||
- ppc64-*) basic_machine=powerpc64-`echo $basic_machine | sed 's/^[^-]*-//'`
|
||||
+ ppc64-* | ppc64p7-*) basic_machine=powerpc64-`echo $basic_machine | sed 's/^[^-]*-//'`
|
||||
;;
|
||||
ppc64le | powerpc64little)
|
||||
basic_machine=powerpc64le-unknown
|
97
SOURCES/00353-architecture-names-upstream-downstream.patch
Normal file
97
SOURCES/00353-architecture-names-upstream-downstream.patch
Normal file
@ -0,0 +1,97 @@
|
||||
From 0000000000000000000000000000000000000000 Mon Sep 17 00:00:00 2001
|
||||
From: Lumir Balhar <lbalhar@redhat.com>
|
||||
Date: Tue, 4 Aug 2020 12:04:03 +0200
|
||||
Subject: [PATCH] 00353: Original names for architectures with different names
|
||||
downstream
|
||||
MIME-Version: 1.0
|
||||
Content-Type: text/plain; charset=UTF-8
|
||||
Content-Transfer-Encoding: 8bit
|
||||
|
||||
https://fedoraproject.org/wiki/Changes/Python_Upstream_Architecture_Names
|
||||
|
||||
Pythons in RHEL/Fedora used different names for some architectures
|
||||
than upstream and other distros (for example ppc64 vs. powerpc64).
|
||||
This was patched in patch 274, now it is sedded if %with legacy_archnames.
|
||||
|
||||
That meant that an extension built with the default upstream settings
|
||||
(on other distro or as an manylinux wheel) could not been found by Python
|
||||
on RHEL/Fedora because it had a different suffix.
|
||||
This patch adds the legacy names to importlib so Python is able
|
||||
to import extensions with a legacy architecture name in its
|
||||
file name.
|
||||
It work both ways, so it support both %with and %without legacy_archnames.
|
||||
|
||||
WARNING: This patch has no effect on Python built with bootstrap
|
||||
enabled because Python/importlib_external.h is not regenerated
|
||||
and therefore Python during bootstrap contains importlib from
|
||||
upstream without this feature. It's possible to include
|
||||
Python/importlib_external.h to this patch but it'd make rebasing
|
||||
a nightmare because it's basically a binary file.
|
||||
|
||||
Co-authored-by: Miro Hrončok <miro@hroncok.cz>
|
||||
---
|
||||
Lib/importlib/_bootstrap_external.py | 40 ++++++++++++++++++++++++++--
|
||||
1 file changed, 38 insertions(+), 2 deletions(-)
|
||||
|
||||
diff --git a/Lib/importlib/_bootstrap_external.py b/Lib/importlib/_bootstrap_external.py
|
||||
index 9feec50842..5bb2454a5c 100644
|
||||
--- a/Lib/importlib/_bootstrap_external.py
|
||||
+++ b/Lib/importlib/_bootstrap_external.py
|
||||
@@ -1361,7 +1361,7 @@ def _get_supported_file_loaders():
|
||||
|
||||
Each item is a tuple (loader, suffixes).
|
||||
"""
|
||||
- extensions = ExtensionFileLoader, _imp.extension_suffixes()
|
||||
+ extensions = ExtensionFileLoader, _alternative_architectures(_imp.extension_suffixes())
|
||||
source = SourceFileLoader, SOURCE_SUFFIXES
|
||||
bytecode = SourcelessFileLoader, BYTECODE_SUFFIXES
|
||||
return [extensions, source, bytecode]
|
||||
@@ -1428,7 +1428,7 @@ def _setup(_bootstrap_module):
|
||||
|
||||
# Constants
|
||||
setattr(self_module, '_relax_case', _make_relax_case())
|
||||
- EXTENSION_SUFFIXES.extend(_imp.extension_suffixes())
|
||||
+ EXTENSION_SUFFIXES.extend(_alternative_architectures(_imp.extension_suffixes()))
|
||||
if builtin_os == 'nt':
|
||||
SOURCE_SUFFIXES.append('.pyw')
|
||||
if '_d.pyd' in EXTENSION_SUFFIXES:
|
||||
@@ -1441,3 +1441,39 @@ def _install(_bootstrap_module):
|
||||
supported_loaders = _get_supported_file_loaders()
|
||||
sys.path_hooks.extend([FileFinder.path_hook(*supported_loaders)])
|
||||
sys.meta_path.append(PathFinder)
|
||||
+
|
||||
+
|
||||
+_ARCH_MAP = {
|
||||
+ "-arm-linux-gnueabi.": "-arm-linux-gnueabihf.",
|
||||
+ "-armeb-linux-gnueabi.": "-armeb-linux-gnueabihf.",
|
||||
+ "-mips64-linux-gnu.": "-mips64-linux-gnuabi64.",
|
||||
+ "-mips64el-linux-gnu.": "-mips64el-linux-gnuabi64.",
|
||||
+ "-ppc-linux-gnu.": "-powerpc-linux-gnu.",
|
||||
+ "-ppc-linux-gnuspe.": "-powerpc-linux-gnuspe.",
|
||||
+ "-ppc64-linux-gnu.": "-powerpc64-linux-gnu.",
|
||||
+ "-ppc64le-linux-gnu.": "-powerpc64le-linux-gnu.",
|
||||
+ # The above, but the other way around:
|
||||
+ "-arm-linux-gnueabihf.": "-arm-linux-gnueabi.",
|
||||
+ "-armeb-linux-gnueabihf.": "-armeb-linux-gnueabi.",
|
||||
+ "-mips64-linux-gnuabi64.": "-mips64-linux-gnu.",
|
||||
+ "-mips64el-linux-gnuabi64.": "-mips64el-linux-gnu.",
|
||||
+ "-powerpc-linux-gnu.": "-ppc-linux-gnu.",
|
||||
+ "-powerpc-linux-gnuspe.": "-ppc-linux-gnuspe.",
|
||||
+ "-powerpc64-linux-gnu.": "-ppc64-linux-gnu.",
|
||||
+ "-powerpc64le-linux-gnu.": "-ppc64le-linux-gnu.",
|
||||
+}
|
||||
+
|
||||
+
|
||||
+def _alternative_architectures(suffixes):
|
||||
+ """Add a suffix with an alternative architecture name
|
||||
+ to the list of suffixes so an extension built with
|
||||
+ the default (upstream) setting is loadable with our Pythons
|
||||
+ """
|
||||
+
|
||||
+ for suffix in suffixes:
|
||||
+ for original, alternative in _ARCH_MAP.items():
|
||||
+ if original in suffix:
|
||||
+ suffixes.append(suffix.replace(original, alternative))
|
||||
+ return suffixes
|
||||
+
|
||||
+ return suffixes
|
@ -0,0 +1,73 @@
|
||||
diff --git a/Lib/http/client.py b/Lib/http/client.py
|
||||
index f0d2642..0a044e9 100644
|
||||
--- a/Lib/http/client.py
|
||||
+++ b/Lib/http/client.py
|
||||
@@ -151,6 +151,10 @@ _contains_disallowed_url_pchar_re = re.compile('[\x00-\x20\x7f]')
|
||||
# _is_allowed_url_pchars_re = re.compile(r"^[/!$&'()*+,;=:@%a-zA-Z0-9._~-]+$")
|
||||
# We are more lenient for assumed real world compatibility purposes.
|
||||
|
||||
+# These characters are not allowed within HTTP method names
|
||||
+# to prevent http header injection.
|
||||
+_contains_disallowed_method_pchar_re = re.compile('[\x00-\x1f]')
|
||||
+
|
||||
# We always set the Content-Length header for these methods because some
|
||||
# servers will otherwise respond with a 411
|
||||
_METHODS_EXPECTING_BODY = {'PATCH', 'POST', 'PUT'}
|
||||
@@ -1117,6 +1121,8 @@ class HTTPConnection:
|
||||
else:
|
||||
raise CannotSendRequest(self.__state)
|
||||
|
||||
+ self._validate_method(method)
|
||||
+
|
||||
# Save the method we use, we need it later in the response phase
|
||||
self._method = method
|
||||
if not url:
|
||||
@@ -1207,6 +1213,15 @@ class HTTPConnection:
|
||||
# For HTTP/1.0, the server will assume "not chunked"
|
||||
pass
|
||||
|
||||
+ def _validate_method(self, method):
|
||||
+ """Validate a method name for putrequest."""
|
||||
+ # prevent http header injection
|
||||
+ match = _contains_disallowed_method_pchar_re.search(method)
|
||||
+ if match:
|
||||
+ raise ValueError(
|
||||
+ f"method can't contain control characters. {method!r} "
|
||||
+ f"(found at least {match.group()!r})")
|
||||
+
|
||||
def putheader(self, header, *values):
|
||||
"""Send a request header line to the server.
|
||||
|
||||
diff --git a/Lib/test/test_httplib.py b/Lib/test/test_httplib.py
|
||||
index 5795b7a..af0350f 100644
|
||||
--- a/Lib/test/test_httplib.py
|
||||
+++ b/Lib/test/test_httplib.py
|
||||
@@ -359,6 +359,28 @@ class HeaderTests(TestCase):
|
||||
self.assertEqual(lines[2], "header: Second: val")
|
||||
|
||||
|
||||
+class HttpMethodTests(TestCase):
|
||||
+ def test_invalid_method_names(self):
|
||||
+ methods = (
|
||||
+ 'GET\r',
|
||||
+ 'POST\n',
|
||||
+ 'PUT\n\r',
|
||||
+ 'POST\nValue',
|
||||
+ 'POST\nHOST:abc',
|
||||
+ 'GET\nrHost:abc\n',
|
||||
+ 'POST\rRemainder:\r',
|
||||
+ 'GET\rHOST:\n',
|
||||
+ '\nPUT'
|
||||
+ )
|
||||
+
|
||||
+ for method in methods:
|
||||
+ with self.assertRaisesRegex(
|
||||
+ ValueError, "method can't contain control characters"):
|
||||
+ conn = client.HTTPConnection('example.com')
|
||||
+ conn.sock = FakeSocket(None)
|
||||
+ conn.request(method=method, url="/")
|
||||
+
|
||||
+
|
||||
class TransferEncodingTest(TestCase):
|
||||
expected_body = b"It's just a flesh wound"
|
||||
|
42
SOURCES/00355-CVE-2020-27619.patch
Normal file
42
SOURCES/00355-CVE-2020-27619.patch
Normal file
@ -0,0 +1,42 @@
|
||||
diff --git a/Lib/test/multibytecodec_support.py b/Lib/test/multibytecodec_support.py
|
||||
index f9884c6..98feec2 100644
|
||||
--- a/Lib/test/multibytecodec_support.py
|
||||
+++ b/Lib/test/multibytecodec_support.py
|
||||
@@ -300,29 +300,23 @@ class TestBase_Mapping(unittest.TestCase):
|
||||
self._test_mapping_file_plain()
|
||||
|
||||
def _test_mapping_file_plain(self):
|
||||
- unichrs = lambda s: ''.join(map(chr, map(eval, s.split('+'))))
|
||||
+ def unichrs(s):
|
||||
+ return ''.join(chr(int(x, 16)) for x in s.split('+'))
|
||||
+
|
||||
urt_wa = {}
|
||||
|
||||
with self.open_mapping_file() as f:
|
||||
for line in f:
|
||||
if not line:
|
||||
break
|
||||
- data = line.split('#')[0].strip().split()
|
||||
+ data = line.split('#')[0].split()
|
||||
if len(data) != 2:
|
||||
continue
|
||||
|
||||
- csetval = eval(data[0])
|
||||
- if csetval <= 0x7F:
|
||||
- csetch = bytes([csetval & 0xff])
|
||||
- elif csetval >= 0x1000000:
|
||||
- csetch = bytes([(csetval >> 24), ((csetval >> 16) & 0xff),
|
||||
- ((csetval >> 8) & 0xff), (csetval & 0xff)])
|
||||
- elif csetval >= 0x10000:
|
||||
- csetch = bytes([(csetval >> 16), ((csetval >> 8) & 0xff),
|
||||
- (csetval & 0xff)])
|
||||
- elif csetval >= 0x100:
|
||||
- csetch = bytes([(csetval >> 8), (csetval & 0xff)])
|
||||
- else:
|
||||
+ if data[0][:2] != '0x':
|
||||
+ self.fail(f"Invalid line: {line!r}")
|
||||
+ csetch = bytes.fromhex(data[0][2:])
|
||||
+ if len(csetch) == 1 and 0x80 <= csetch[0]:
|
||||
continue
|
||||
|
||||
unich = unichrs(data[1])
|
269
SOURCES/00356-k_and_a_options_for_pathfix.patch
Normal file
269
SOURCES/00356-k_and_a_options_for_pathfix.patch
Normal file
@ -0,0 +1,269 @@
|
||||
From 0cfd9a7f26488567b9a3e5ec192099a8b80ad9df Mon Sep 17 00:00:00 2001
|
||||
From: Lumir Balhar <lbalhar@redhat.com>
|
||||
Date: Tue, 19 Jan 2021 07:55:37 +0100
|
||||
Subject: [PATCH] [PATCH] bpo-37064: Add -k and -a options to pathfix.py tool
|
||||
(GH-16387)
|
||||
|
||||
* bpo-37064: Add option -k to Tools/scripts/pathfix.py (GH-15548)
|
||||
|
||||
Add flag -k to pathscript.py script: preserve shebang flags.
|
||||
|
||||
(cherry picked from commit 50254ac4c179cb412e90682098c97db786143929)
|
||||
|
||||
* bpo-37064: Add option -a to pathfix.py tool (GH-15717)
|
||||
|
||||
Add option -a to Tools/Scripts/pathfix.py script: add flags.
|
||||
|
||||
(cherry picked from commit 1dc1acbd73f05f14c974b7ce1041787d7abef31e)
|
||||
---
|
||||
Lib/test/test_tools/test_pathfix.py | 104 ++++++++++++++++++++++++++++
|
||||
Tools/scripts/pathfix.py | 64 +++++++++++++++--
|
||||
2 files changed, 163 insertions(+), 5 deletions(-)
|
||||
create mode 100644 Lib/test/test_tools/test_pathfix.py
|
||||
|
||||
diff --git a/Lib/test/test_tools/test_pathfix.py b/Lib/test/test_tools/test_pathfix.py
|
||||
new file mode 100644
|
||||
index 0000000..1f0585e
|
||||
--- /dev/null
|
||||
+++ b/Lib/test/test_tools/test_pathfix.py
|
||||
@@ -0,0 +1,104 @@
|
||||
+import os
|
||||
+import subprocess
|
||||
+import sys
|
||||
+import unittest
|
||||
+from test import support
|
||||
+from test.test_tools import import_tool, scriptsdir
|
||||
+
|
||||
+
|
||||
+class TestPathfixFunctional(unittest.TestCase):
|
||||
+ script = os.path.join(scriptsdir, 'pathfix.py')
|
||||
+
|
||||
+ def setUp(self):
|
||||
+ self.temp_file = support.TESTFN
|
||||
+ self.addCleanup(support.unlink, support.TESTFN)
|
||||
+
|
||||
+ def pathfix(self, shebang, pathfix_flags, exitcode=0, stdout='', stderr=''):
|
||||
+ with open(self.temp_file, 'w', encoding='utf8') as f:
|
||||
+ f.write(f'{shebang}\n' + 'print("Hello world")\n')
|
||||
+
|
||||
+ proc = subprocess.run(
|
||||
+ [sys.executable, self.script,
|
||||
+ *pathfix_flags, '-n', self.temp_file],
|
||||
+ universal_newlines=True, stdout=subprocess.PIPE,
|
||||
+ stderr=subprocess.PIPE)
|
||||
+
|
||||
+ if stdout == '' and proc.returncode == 0:
|
||||
+ stdout = f'{self.temp_file}: updating\n'
|
||||
+ self.assertEqual(proc.returncode, exitcode, proc)
|
||||
+ self.assertEqual(proc.stdout, stdout, proc)
|
||||
+ self.assertEqual(proc.stderr, stderr, proc)
|
||||
+
|
||||
+ with open(self.temp_file, 'r', encoding='utf8') as f:
|
||||
+ output = f.read()
|
||||
+
|
||||
+ lines = output.split('\n')
|
||||
+ self.assertEqual(lines[1:], ['print("Hello world")', ''])
|
||||
+ new_shebang = lines[0]
|
||||
+
|
||||
+ if proc.returncode != 0:
|
||||
+ self.assertEqual(shebang, new_shebang)
|
||||
+
|
||||
+ return new_shebang
|
||||
+
|
||||
+ def test_pathfix(self):
|
||||
+ self.assertEqual(
|
||||
+ self.pathfix(
|
||||
+ '#! /usr/bin/env python',
|
||||
+ ['-i', '/usr/bin/python3']),
|
||||
+ '#! /usr/bin/python3')
|
||||
+ self.assertEqual(
|
||||
+ self.pathfix(
|
||||
+ '#! /usr/bin/env python -R',
|
||||
+ ['-i', '/usr/bin/python3']),
|
||||
+ '#! /usr/bin/python3')
|
||||
+
|
||||
+ def test_pathfix_keeping_flags(self):
|
||||
+ self.assertEqual(
|
||||
+ self.pathfix(
|
||||
+ '#! /usr/bin/env python -R',
|
||||
+ ['-i', '/usr/bin/python3', '-k']),
|
||||
+ '#! /usr/bin/python3 -R')
|
||||
+ self.assertEqual(
|
||||
+ self.pathfix(
|
||||
+ '#! /usr/bin/env python',
|
||||
+ ['-i', '/usr/bin/python3', '-k']),
|
||||
+ '#! /usr/bin/python3')
|
||||
+
|
||||
+ def test_pathfix_adding_flag(self):
|
||||
+ self.assertEqual(
|
||||
+ self.pathfix(
|
||||
+ '#! /usr/bin/env python',
|
||||
+ ['-i', '/usr/bin/python3', '-a', 's']),
|
||||
+ '#! /usr/bin/python3 -s')
|
||||
+ self.assertEqual(
|
||||
+ self.pathfix(
|
||||
+ '#! /usr/bin/env python -S',
|
||||
+ ['-i', '/usr/bin/python3', '-a', 's']),
|
||||
+ '#! /usr/bin/python3 -s')
|
||||
+ self.assertEqual(
|
||||
+ self.pathfix(
|
||||
+ '#! /usr/bin/env python -V',
|
||||
+ ['-i', '/usr/bin/python3', '-a', 'v', '-k']),
|
||||
+ '#! /usr/bin/python3 -vV')
|
||||
+ self.assertEqual(
|
||||
+ self.pathfix(
|
||||
+ '#! /usr/bin/env python',
|
||||
+ ['-i', '/usr/bin/python3', '-a', 'Rs']),
|
||||
+ '#! /usr/bin/python3 -Rs')
|
||||
+ self.assertEqual(
|
||||
+ self.pathfix(
|
||||
+ '#! /usr/bin/env python -W default',
|
||||
+ ['-i', '/usr/bin/python3', '-a', 's', '-k']),
|
||||
+ '#! /usr/bin/python3 -sW default')
|
||||
+
|
||||
+ def test_pathfix_adding_errors(self):
|
||||
+ self.pathfix(
|
||||
+ '#! /usr/bin/env python -E',
|
||||
+ ['-i', '/usr/bin/python3', '-a', 'W default', '-k'],
|
||||
+ exitcode=2,
|
||||
+ stderr="-a option doesn't support whitespaces")
|
||||
+
|
||||
+
|
||||
+if __name__ == '__main__':
|
||||
+ unittest.main()
|
||||
diff --git a/Tools/scripts/pathfix.py b/Tools/scripts/pathfix.py
|
||||
index c5bf984..2dfa6e8 100755
|
||||
--- a/Tools/scripts/pathfix.py
|
||||
+++ b/Tools/scripts/pathfix.py
|
||||
@@ -1,6 +1,6 @@
|
||||
#!/usr/bin/env python3
|
||||
|
||||
-# Change the #! line occurring in Python scripts. The new interpreter
|
||||
+# Change the #! line (shebang) occurring in Python scripts. The new interpreter
|
||||
# pathname must be given with a -i option.
|
||||
#
|
||||
# Command line arguments are files or directories to be processed.
|
||||
@@ -10,7 +10,13 @@
|
||||
# arguments).
|
||||
# The original file is kept as a back-up (with a "~" attached to its name),
|
||||
# -n flag can be used to disable this.
|
||||
-#
|
||||
+
|
||||
+# Sometimes you may find shebangs with flags such as `#! /usr/bin/env python -si`.
|
||||
+# Normally, pathfix overwrites the entire line, including the flags.
|
||||
+# To change interpreter and keep flags from the original shebang line, use -k.
|
||||
+# If you want to keep flags and add to them one single literal flag, use option -a.
|
||||
+
|
||||
+
|
||||
# Undoubtedly you can do this using find and sed or perl, but this is
|
||||
# a nice example of Python code that recurses down a directory tree
|
||||
# and uses regular expressions. Also note several subtleties like
|
||||
@@ -33,16 +39,21 @@ rep = sys.stdout.write
|
||||
new_interpreter = None
|
||||
preserve_timestamps = False
|
||||
create_backup = True
|
||||
+keep_flags = False
|
||||
+add_flags = b''
|
||||
|
||||
|
||||
def main():
|
||||
global new_interpreter
|
||||
global preserve_timestamps
|
||||
global create_backup
|
||||
- usage = ('usage: %s -i /interpreter -p -n file-or-directory ...\n' %
|
||||
+ global keep_flags
|
||||
+ global add_flags
|
||||
+
|
||||
+ usage = ('usage: %s -i /interpreter -p -n -k -a file-or-directory ...\n' %
|
||||
sys.argv[0])
|
||||
try:
|
||||
- opts, args = getopt.getopt(sys.argv[1:], 'i:pn')
|
||||
+ opts, args = getopt.getopt(sys.argv[1:], 'i:a:kpn')
|
||||
except getopt.error as msg:
|
||||
err(str(msg) + '\n')
|
||||
err(usage)
|
||||
@@ -54,6 +65,13 @@ def main():
|
||||
preserve_timestamps = True
|
||||
if o == '-n':
|
||||
create_backup = False
|
||||
+ if o == '-k':
|
||||
+ keep_flags = True
|
||||
+ if o == '-a':
|
||||
+ add_flags = a.encode()
|
||||
+ if b' ' in add_flags:
|
||||
+ err("-a option doesn't support whitespaces")
|
||||
+ sys.exit(2)
|
||||
if not new_interpreter or not new_interpreter.startswith(b'/') or \
|
||||
not args:
|
||||
err('-i option or file-or-directory missing\n')
|
||||
@@ -70,10 +88,14 @@ def main():
|
||||
if fix(arg): bad = 1
|
||||
sys.exit(bad)
|
||||
|
||||
+
|
||||
ispythonprog = re.compile(r'^[a-zA-Z0-9_]+\.py$')
|
||||
+
|
||||
+
|
||||
def ispython(name):
|
||||
return bool(ispythonprog.match(name))
|
||||
|
||||
+
|
||||
def recursedown(dirname):
|
||||
dbg('recursedown(%r)\n' % (dirname,))
|
||||
bad = 0
|
||||
@@ -96,6 +118,7 @@ def recursedown(dirname):
|
||||
if recursedown(fullname): bad = 1
|
||||
return bad
|
||||
|
||||
+
|
||||
def fix(filename):
|
||||
## dbg('fix(%r)\n' % (filename,))
|
||||
try:
|
||||
@@ -166,12 +189,43 @@ def fix(filename):
|
||||
# Return success
|
||||
return 0
|
||||
|
||||
+
|
||||
+def parse_shebang(shebangline):
|
||||
+ shebangline = shebangline.rstrip(b'\n')
|
||||
+ start = shebangline.find(b' -')
|
||||
+ if start == -1:
|
||||
+ return b''
|
||||
+ return shebangline[start:]
|
||||
+
|
||||
+
|
||||
+def populate_flags(shebangline):
|
||||
+ old_flags = b''
|
||||
+ if keep_flags:
|
||||
+ old_flags = parse_shebang(shebangline)
|
||||
+ if old_flags:
|
||||
+ old_flags = old_flags[2:]
|
||||
+ if not (old_flags or add_flags):
|
||||
+ return b''
|
||||
+ # On Linux, the entire string following the interpreter name
|
||||
+ # is passed as a single argument to the interpreter.
|
||||
+ # e.g. "#! /usr/bin/python3 -W Error -s" runs "/usr/bin/python3 "-W Error -s"
|
||||
+ # so shebang should have single '-' where flags are given and
|
||||
+ # flag might need argument for that reasons adding new flags is
|
||||
+ # between '-' and original flags
|
||||
+ # e.g. #! /usr/bin/python3 -sW Error
|
||||
+ return b' -' + add_flags + old_flags
|
||||
+
|
||||
+
|
||||
def fixline(line):
|
||||
if not line.startswith(b'#!'):
|
||||
return line
|
||||
+
|
||||
if b"python" not in line:
|
||||
return line
|
||||
- return b'#! ' + new_interpreter + b'\n'
|
||||
+
|
||||
+ flags = populate_flags(line)
|
||||
+ return b'#! ' + new_interpreter + flags + b'\n'
|
||||
+
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
--
|
||||
2.29.2
|
||||
|
184
SOURCES/00357-CVE-2021-3177.patch
Normal file
184
SOURCES/00357-CVE-2021-3177.patch
Normal file
@ -0,0 +1,184 @@
|
||||
From e92381a0a6a3e1f000956e1f1e70e543b9c2bcd5 Mon Sep 17 00:00:00 2001
|
||||
From: Benjamin Peterson <benjamin@python.org>
|
||||
Date: Mon, 18 Jan 2021 14:47:05 -0600
|
||||
Subject: [PATCH] [3.6] closes bpo-42938: Replace snprintf with Python unicode
|
||||
formatting in ctypes param reprs. (24239). (cherry picked from commit
|
||||
916610ef90a0d0761f08747f7b0905541f0977c7)
|
||||
|
||||
Co-authored-by: Benjamin Peterson <benjamin@python.org>
|
||||
---
|
||||
Lib/ctypes/test/test_parameters.py | 43 +++++++++++++++
|
||||
.../2021-01-18-09-27-31.bpo-42938.4Zn4Mp.rst | 2 +
|
||||
Modules/_ctypes/callproc.c | 55 +++++++------------
|
||||
3 files changed, 66 insertions(+), 34 deletions(-)
|
||||
create mode 100644 Misc/NEWS.d/next/Security/2021-01-18-09-27-31.bpo-42938.4Zn4Mp.rst
|
||||
|
||||
diff --git a/Lib/ctypes/test/test_parameters.py b/Lib/ctypes/test/test_parameters.py
|
||||
index e4c25fd880cef..531894fdec838 100644
|
||||
--- a/Lib/ctypes/test/test_parameters.py
|
||||
+++ b/Lib/ctypes/test/test_parameters.py
|
||||
@@ -201,6 +201,49 @@ def __dict__(self):
|
||||
with self.assertRaises(ZeroDivisionError):
|
||||
WorseStruct().__setstate__({}, b'foo')
|
||||
|
||||
+ def test_parameter_repr(self):
|
||||
+ from ctypes import (
|
||||
+ c_bool,
|
||||
+ c_char,
|
||||
+ c_wchar,
|
||||
+ c_byte,
|
||||
+ c_ubyte,
|
||||
+ c_short,
|
||||
+ c_ushort,
|
||||
+ c_int,
|
||||
+ c_uint,
|
||||
+ c_long,
|
||||
+ c_ulong,
|
||||
+ c_longlong,
|
||||
+ c_ulonglong,
|
||||
+ c_float,
|
||||
+ c_double,
|
||||
+ c_longdouble,
|
||||
+ c_char_p,
|
||||
+ c_wchar_p,
|
||||
+ c_void_p,
|
||||
+ )
|
||||
+ self.assertRegex(repr(c_bool.from_param(True)), r"^<cparam '\?' at 0x[A-Fa-f0-9]+>$")
|
||||
+ self.assertEqual(repr(c_char.from_param(97)), "<cparam 'c' ('a')>")
|
||||
+ self.assertRegex(repr(c_wchar.from_param('a')), r"^<cparam 'u' at 0x[A-Fa-f0-9]+>$")
|
||||
+ self.assertEqual(repr(c_byte.from_param(98)), "<cparam 'b' (98)>")
|
||||
+ self.assertEqual(repr(c_ubyte.from_param(98)), "<cparam 'B' (98)>")
|
||||
+ self.assertEqual(repr(c_short.from_param(511)), "<cparam 'h' (511)>")
|
||||
+ self.assertEqual(repr(c_ushort.from_param(511)), "<cparam 'H' (511)>")
|
||||
+ self.assertRegex(repr(c_int.from_param(20000)), r"^<cparam '[li]' \(20000\)>$")
|
||||
+ self.assertRegex(repr(c_uint.from_param(20000)), r"^<cparam '[LI]' \(20000\)>$")
|
||||
+ self.assertRegex(repr(c_long.from_param(20000)), r"^<cparam '[li]' \(20000\)>$")
|
||||
+ self.assertRegex(repr(c_ulong.from_param(20000)), r"^<cparam '[LI]' \(20000\)>$")
|
||||
+ self.assertRegex(repr(c_longlong.from_param(20000)), r"^<cparam '[liq]' \(20000\)>$")
|
||||
+ self.assertRegex(repr(c_ulonglong.from_param(20000)), r"^<cparam '[LIQ]' \(20000\)>$")
|
||||
+ self.assertEqual(repr(c_float.from_param(1.5)), "<cparam 'f' (1.5)>")
|
||||
+ self.assertEqual(repr(c_double.from_param(1.5)), "<cparam 'd' (1.5)>")
|
||||
+ self.assertEqual(repr(c_double.from_param(1e300)), "<cparam 'd' (1e+300)>")
|
||||
+ self.assertRegex(repr(c_longdouble.from_param(1.5)), r"^<cparam ('d' \(1.5\)|'g' at 0x[A-Fa-f0-9]+)>$")
|
||||
+ self.assertRegex(repr(c_char_p.from_param(b'hihi')), "^<cparam 'z' \(0x[A-Fa-f0-9]+\)>$")
|
||||
+ self.assertRegex(repr(c_wchar_p.from_param('hihi')), "^<cparam 'Z' \(0x[A-Fa-f0-9]+\)>$")
|
||||
+ self.assertRegex(repr(c_void_p.from_param(0x12)), r"^<cparam 'P' \(0x0*12\)>$")
|
||||
+
|
||||
################################################################
|
||||
|
||||
if __name__ == '__main__':
|
||||
diff --git a/Misc/NEWS.d/next/Security/2021-01-18-09-27-31.bpo-42938.4Zn4Mp.rst b/Misc/NEWS.d/next/Security/2021-01-18-09-27-31.bpo-42938.4Zn4Mp.rst
|
||||
new file mode 100644
|
||||
index 0000000000000..7df65a156feab
|
||||
--- /dev/null
|
||||
+++ b/Misc/NEWS.d/next/Security/2021-01-18-09-27-31.bpo-42938.4Zn4Mp.rst
|
||||
@@ -0,0 +1,2 @@
|
||||
+Avoid static buffers when computing the repr of :class:`ctypes.c_double` and
|
||||
+:class:`ctypes.c_longdouble` values.
|
||||
diff --git a/Modules/_ctypes/callproc.c b/Modules/_ctypes/callproc.c
|
||||
index d1c190f359108..2bb289bce043f 100644
|
||||
--- a/Modules/_ctypes/callproc.c
|
||||
+++ b/Modules/_ctypes/callproc.c
|
||||
@@ -461,58 +461,47 @@ is_literal_char(unsigned char c)
|
||||
static PyObject *
|
||||
PyCArg_repr(PyCArgObject *self)
|
||||
{
|
||||
- char buffer[256];
|
||||
switch(self->tag) {
|
||||
case 'b':
|
||||
case 'B':
|
||||
- sprintf(buffer, "<cparam '%c' (%d)>",
|
||||
+ return PyUnicode_FromFormat("<cparam '%c' (%d)>",
|
||||
self->tag, self->value.b);
|
||||
- break;
|
||||
case 'h':
|
||||
case 'H':
|
||||
- sprintf(buffer, "<cparam '%c' (%d)>",
|
||||
+ return PyUnicode_FromFormat("<cparam '%c' (%d)>",
|
||||
self->tag, self->value.h);
|
||||
- break;
|
||||
case 'i':
|
||||
case 'I':
|
||||
- sprintf(buffer, "<cparam '%c' (%d)>",
|
||||
+ return PyUnicode_FromFormat("<cparam '%c' (%d)>",
|
||||
self->tag, self->value.i);
|
||||
- break;
|
||||
case 'l':
|
||||
case 'L':
|
||||
- sprintf(buffer, "<cparam '%c' (%ld)>",
|
||||
+ return PyUnicode_FromFormat("<cparam '%c' (%ld)>",
|
||||
self->tag, self->value.l);
|
||||
- break;
|
||||
|
||||
case 'q':
|
||||
case 'Q':
|
||||
- sprintf(buffer,
|
||||
-#ifdef MS_WIN32
|
||||
- "<cparam '%c' (%I64d)>",
|
||||
-#else
|
||||
- "<cparam '%c' (%lld)>",
|
||||
-#endif
|
||||
+ return PyUnicode_FromFormat("<cparam '%c' (%lld)>",
|
||||
self->tag, self->value.q);
|
||||
- break;
|
||||
case 'd':
|
||||
- sprintf(buffer, "<cparam '%c' (%f)>",
|
||||
- self->tag, self->value.d);
|
||||
- break;
|
||||
- case 'f':
|
||||
- sprintf(buffer, "<cparam '%c' (%f)>",
|
||||
- self->tag, self->value.f);
|
||||
- break;
|
||||
-
|
||||
+ case 'f': {
|
||||
+ PyObject *f = PyFloat_FromDouble((self->tag == 'f') ? self->value.f : self->value.d);
|
||||
+ if (f == NULL) {
|
||||
+ return NULL;
|
||||
+ }
|
||||
+ PyObject *result = PyUnicode_FromFormat("<cparam '%c' (%R)>", self->tag, f);
|
||||
+ Py_DECREF(f);
|
||||
+ return result;
|
||||
+ }
|
||||
case 'c':
|
||||
if (is_literal_char((unsigned char)self->value.c)) {
|
||||
- sprintf(buffer, "<cparam '%c' ('%c')>",
|
||||
+ return PyUnicode_FromFormat("<cparam '%c' ('%c')>",
|
||||
self->tag, self->value.c);
|
||||
}
|
||||
else {
|
||||
- sprintf(buffer, "<cparam '%c' ('\\x%02x')>",
|
||||
+ return PyUnicode_FromFormat("<cparam '%c' ('\\x%02x')>",
|
||||
self->tag, (unsigned char)self->value.c);
|
||||
}
|
||||
- break;
|
||||
|
||||
/* Hm, are these 'z' and 'Z' codes useful at all?
|
||||
Shouldn't they be replaced by the functionality of c_string
|
||||
@@ -521,22 +510,20 @@ PyCArg_repr(PyCArgObject *self)
|
||||
case 'z':
|
||||
case 'Z':
|
||||
case 'P':
|
||||
- sprintf(buffer, "<cparam '%c' (%p)>",
|
||||
+ return PyUnicode_FromFormat("<cparam '%c' (%p)>",
|
||||
self->tag, self->value.p);
|
||||
break;
|
||||
|
||||
default:
|
||||
if (is_literal_char((unsigned char)self->tag)) {
|
||||
- sprintf(buffer, "<cparam '%c' at %p>",
|
||||
- (unsigned char)self->tag, self);
|
||||
+ return PyUnicode_FromFormat("<cparam '%c' at %p>",
|
||||
+ (unsigned char)self->tag, (void *)self);
|
||||
}
|
||||
else {
|
||||
- sprintf(buffer, "<cparam 0x%02x at %p>",
|
||||
- (unsigned char)self->tag, self);
|
||||
+ return PyUnicode_FromFormat("<cparam 0x%02x at %p>",
|
||||
+ (unsigned char)self->tag, (void *)self);
|
||||
}
|
||||
- break;
|
||||
}
|
||||
- return PyUnicode_FromString(buffer);
|
||||
}
|
||||
|
||||
static PyMemberDef PyCArgType_members[] = {
|
684
SOURCES/00359-CVE-2021-23336.patch
Normal file
684
SOURCES/00359-CVE-2021-23336.patch
Normal file
@ -0,0 +1,684 @@
|
||||
commit 9e77ec82c40ab59846f9447b7c483e7b8e368b16
|
||||
Author: Petr Viktorin <pviktori@redhat.com>
|
||||
Date: Thu Mar 4 13:59:56 2021 +0100
|
||||
|
||||
CVE-2021-23336: Add `separator` argument to parse_qs; warn with default
|
||||
|
||||
Partially backports https://bugs.python.org/issue42967 : [security] Address a web cache-poisoning issue reported in urllib.parse.parse_qsl().
|
||||
However, this solution is different than the upstream solution in Python 3.6.13.
|
||||
|
||||
An optional argument seperator is added to specify the separator.
|
||||
It is recommended to set it to '&' or ';' to match the application or proxy in use.
|
||||
The default can be set with an env variable of a config file.
|
||||
If neither the argument, env var or config file specifies a separator, "&" is used
|
||||
but a warning is raised if parse_qs is used on input that contains ';'.
|
||||
|
||||
Co-authors of the upstream change (who do not necessarily agree with this):
|
||||
Co-authored-by: Adam Goldschmidt <adamgold7@gmail.com>
|
||||
Co-authored-by: Ken Jin <28750310+Fidget-Spinner@users.noreply.github.com>
|
||||
Co-authored-by: Éric Araujo <merwok@netwok.org>
|
||||
|
||||
diff --git a/Doc/library/cgi.rst b/Doc/library/cgi.rst
|
||||
index 41219eeaaba..ddecc0af23a 100644
|
||||
--- a/Doc/library/cgi.rst
|
||||
+++ b/Doc/library/cgi.rst
|
||||
@@ -277,13 +277,12 @@ These are useful if you want more control, or if you want to employ some of the
|
||||
algorithms implemented in this module in other circumstances.
|
||||
|
||||
|
||||
-.. function:: parse(fp=None, environ=os.environ, keep_blank_values=False, strict_parsing=False)
|
||||
+.. function:: parse(fp=None, environ=os.environ, keep_blank_values=False, strict_parsing=False, separator=None)
|
||||
|
||||
Parse a query in the environment or from a file (the file defaults to
|
||||
- ``sys.stdin``). The *keep_blank_values* and *strict_parsing* parameters are
|
||||
+ ``sys.stdin``). The *keep_blank_values*, *strict_parsing* and *separator* parameters are
|
||||
passed to :func:`urllib.parse.parse_qs` unchanged.
|
||||
|
||||
-
|
||||
.. function:: parse_qs(qs, keep_blank_values=False, strict_parsing=False)
|
||||
|
||||
This function is deprecated in this module. Use :func:`urllib.parse.parse_qs`
|
||||
@@ -308,7 +307,6 @@ algorithms implemented in this module in other circumstances.
|
||||
Note that this does not parse nested multipart parts --- use
|
||||
:class:`FieldStorage` for that.
|
||||
|
||||
-
|
||||
.. function:: parse_header(string)
|
||||
|
||||
Parse a MIME header (such as :mailheader:`Content-Type`) into a main value and a
|
||||
diff --git a/Doc/library/urllib.parse.rst b/Doc/library/urllib.parse.rst
|
||||
index 647af613a31..bcab7c142bc 100644
|
||||
--- a/Doc/library/urllib.parse.rst
|
||||
+++ b/Doc/library/urllib.parse.rst
|
||||
@@ -143,7 +143,7 @@ or on combining URL components into a URL string.
|
||||
now raise :exc:`ValueError`.
|
||||
|
||||
|
||||
-.. function:: parse_qs(qs, keep_blank_values=False, strict_parsing=False, encoding='utf-8', errors='replace', max_num_fields=None)
|
||||
+.. function:: parse_qs(qs, keep_blank_values=False, strict_parsing=False, encoding='utf-8', errors='replace', max_num_fields=None, separator=None)
|
||||
|
||||
Parse a query string given as a string argument (data of type
|
||||
:mimetype:`application/x-www-form-urlencoded`). Data are returned as a
|
||||
@@ -168,6 +168,15 @@ or on combining URL components into a URL string.
|
||||
read. If set, then throws a :exc:`ValueError` if there are more than
|
||||
*max_num_fields* fields read.
|
||||
|
||||
+ The optional argument *separator* is the symbol to use for separating the
|
||||
+ query arguments. It is recommended to set it to ``'&'`` or ``';'``.
|
||||
+ It defaults to ``'&'``; a warning is raised if this default is used.
|
||||
+ This default may be changed with the following environment variable settings:
|
||||
+
|
||||
+ - ``PYTHON_URLLIB_QS_SEPARATOR='&'``: use only ``&`` as separator, without warning (as in Python 3.6.13+ or 3.10)
|
||||
+ - ``PYTHON_URLLIB_QS_SEPARATOR=';'``: use only ``;`` as separator
|
||||
+ - ``PYTHON_URLLIB_QS_SEPARATOR=legacy``: use both ``&`` and ``;`` (as in previous versions of Python)
|
||||
+
|
||||
Use the :func:`urllib.parse.urlencode` function (with the ``doseq``
|
||||
parameter set to ``True``) to convert such dictionaries into query
|
||||
strings.
|
||||
@@ -204,6 +213,9 @@ or on combining URL components into a URL string.
|
||||
read. If set, then throws a :exc:`ValueError` if there are more than
|
||||
*max_num_fields* fields read.
|
||||
|
||||
+ The optional argument *separator* is the symbol to use for separating the
|
||||
+ query arguments. It works as in :py:func:`parse_qs`.
|
||||
+
|
||||
Use the :func:`urllib.parse.urlencode` function to convert such lists of pairs into
|
||||
query strings.
|
||||
|
||||
@@ -213,7 +225,6 @@ or on combining URL components into a URL string.
|
||||
.. versionchanged:: 3.6.8
|
||||
Added *max_num_fields* parameter.
|
||||
|
||||
-
|
||||
.. function:: urlunparse(parts)
|
||||
|
||||
Construct a URL from a tuple as returned by ``urlparse()``. The *parts*
|
||||
diff --git a/Lib/cgi.py b/Lib/cgi.py
|
||||
index 56f243e09f0..5ab2a5d6af6 100755
|
||||
--- a/Lib/cgi.py
|
||||
+++ b/Lib/cgi.py
|
||||
@@ -117,7 +117,8 @@ log = initlog # The current logging function
|
||||
# 0 ==> unlimited input
|
||||
maxlen = 0
|
||||
|
||||
-def parse(fp=None, environ=os.environ, keep_blank_values=0, strict_parsing=0):
|
||||
+def parse(fp=None, environ=os.environ, keep_blank_values=0,
|
||||
+ strict_parsing=0, separator=None):
|
||||
"""Parse a query in the environment or from a file (default stdin)
|
||||
|
||||
Arguments, all optional:
|
||||
@@ -136,6 +137,8 @@ def parse(fp=None, environ=os.environ, keep_blank_values=0, strict_parsing=0):
|
||||
strict_parsing: flag indicating what to do with parsing errors.
|
||||
If false (the default), errors are silently ignored.
|
||||
If true, errors raise a ValueError exception.
|
||||
+
|
||||
+ separator: str. The symbol to use for separating the query arguments.
|
||||
"""
|
||||
if fp is None:
|
||||
fp = sys.stdin
|
||||
@@ -156,7 +159,7 @@ def parse(fp=None, environ=os.environ, keep_blank_values=0, strict_parsing=0):
|
||||
if environ['REQUEST_METHOD'] == 'POST':
|
||||
ctype, pdict = parse_header(environ['CONTENT_TYPE'])
|
||||
if ctype == 'multipart/form-data':
|
||||
- return parse_multipart(fp, pdict)
|
||||
+ return parse_multipart(fp, pdict, separator=separator)
|
||||
elif ctype == 'application/x-www-form-urlencoded':
|
||||
clength = int(environ['CONTENT_LENGTH'])
|
||||
if maxlen and clength > maxlen:
|
||||
@@ -182,21 +185,21 @@ def parse(fp=None, environ=os.environ, keep_blank_values=0, strict_parsing=0):
|
||||
return urllib.parse.parse_qs(qs, keep_blank_values, strict_parsing,
|
||||
encoding=encoding)
|
||||
|
||||
-
|
||||
# parse query string function called from urlparse,
|
||||
# this is done in order to maintain backward compatibility.
|
||||
-
|
||||
-def parse_qs(qs, keep_blank_values=0, strict_parsing=0):
|
||||
+def parse_qs(qs, keep_blank_values=0, strict_parsing=0, separator=None):
|
||||
"""Parse a query given as a string argument."""
|
||||
warn("cgi.parse_qs is deprecated, use urllib.parse.parse_qs instead",
|
||||
DeprecationWarning, 2)
|
||||
- return urllib.parse.parse_qs(qs, keep_blank_values, strict_parsing)
|
||||
+ return urllib.parse.parse_qs(qs, keep_blank_values, strict_parsing,
|
||||
+ separator=separator)
|
||||
|
||||
-def parse_qsl(qs, keep_blank_values=0, strict_parsing=0):
|
||||
+def parse_qsl(qs, keep_blank_values=0, strict_parsing=0, separator=None):
|
||||
"""Parse a query given as a string argument."""
|
||||
warn("cgi.parse_qsl is deprecated, use urllib.parse.parse_qsl instead",
|
||||
DeprecationWarning, 2)
|
||||
- return urllib.parse.parse_qsl(qs, keep_blank_values, strict_parsing)
|
||||
+ return urllib.parse.parse_qsl(qs, keep_blank_values, strict_parsing,
|
||||
+ separator=separator)
|
||||
|
||||
def parse_multipart(fp, pdict):
|
||||
"""Parse multipart input.
|
||||
@@ -297,7 +300,6 @@ def parse_multipart(fp, pdict):
|
||||
|
||||
return partdict
|
||||
|
||||
-
|
||||
def _parseparam(s):
|
||||
while s[:1] == ';':
|
||||
s = s[1:]
|
||||
@@ -405,7 +407,7 @@ class FieldStorage:
|
||||
def __init__(self, fp=None, headers=None, outerboundary=b'',
|
||||
environ=os.environ, keep_blank_values=0, strict_parsing=0,
|
||||
limit=None, encoding='utf-8', errors='replace',
|
||||
- max_num_fields=None):
|
||||
+ max_num_fields=None, separator=None):
|
||||
"""Constructor. Read multipart/* until last part.
|
||||
|
||||
Arguments, all optional:
|
||||
@@ -453,6 +455,7 @@ class FieldStorage:
|
||||
self.keep_blank_values = keep_blank_values
|
||||
self.strict_parsing = strict_parsing
|
||||
self.max_num_fields = max_num_fields
|
||||
+ self.separator = separator
|
||||
if 'REQUEST_METHOD' in environ:
|
||||
method = environ['REQUEST_METHOD'].upper()
|
||||
self.qs_on_post = None
|
||||
@@ -678,7 +681,7 @@ class FieldStorage:
|
||||
query = urllib.parse.parse_qsl(
|
||||
qs, self.keep_blank_values, self.strict_parsing,
|
||||
encoding=self.encoding, errors=self.errors,
|
||||
- max_num_fields=self.max_num_fields)
|
||||
+ max_num_fields=self.max_num_fields, separator=self.separator)
|
||||
self.list = [MiniFieldStorage(key, value) for key, value in query]
|
||||
self.skip_lines()
|
||||
|
||||
@@ -694,7 +697,7 @@ class FieldStorage:
|
||||
query = urllib.parse.parse_qsl(
|
||||
self.qs_on_post, self.keep_blank_values, self.strict_parsing,
|
||||
encoding=self.encoding, errors=self.errors,
|
||||
- max_num_fields=self.max_num_fields)
|
||||
+ max_num_fields=self.max_num_fields, separator=self.separator)
|
||||
self.list.extend(MiniFieldStorage(key, value) for key, value in query)
|
||||
|
||||
klass = self.FieldStorageClass or self.__class__
|
||||
@@ -736,7 +739,8 @@ class FieldStorage:
|
||||
|
||||
part = klass(self.fp, headers, ib, environ, keep_blank_values,
|
||||
strict_parsing,self.limit-self.bytes_read,
|
||||
- self.encoding, self.errors, max_num_fields)
|
||||
+ self.encoding, self.errors, max_num_fields,
|
||||
+ separator=self.separator)
|
||||
|
||||
if max_num_fields is not None:
|
||||
max_num_fields -= 1
|
||||
diff --git a/Lib/test/test_cgi.py b/Lib/test/test_cgi.py
|
||||
index b3e2d4cce8e..5ae3e085e1e 100644
|
||||
--- a/Lib/test/test_cgi.py
|
||||
+++ b/Lib/test/test_cgi.py
|
||||
@@ -55,12 +55,9 @@ parse_strict_test_cases = [
|
||||
("", ValueError("bad query field: ''")),
|
||||
("&", ValueError("bad query field: ''")),
|
||||
("&&", ValueError("bad query field: ''")),
|
||||
- (";", ValueError("bad query field: ''")),
|
||||
- (";&;", ValueError("bad query field: ''")),
|
||||
# Should the next few really be valid?
|
||||
("=", {}),
|
||||
("=&=", {}),
|
||||
- ("=;=", {}),
|
||||
# This rest seem to make sense
|
||||
("=a", {'': ['a']}),
|
||||
("&=a", ValueError("bad query field: ''")),
|
||||
@@ -75,8 +72,6 @@ parse_strict_test_cases = [
|
||||
("a=a+b&b=b+c", {'a': ['a b'], 'b': ['b c']}),
|
||||
("a=a+b&a=b+a", {'a': ['a b', 'b a']}),
|
||||
("x=1&y=2.0&z=2-3.%2b0", {'x': ['1'], 'y': ['2.0'], 'z': ['2-3.+0']}),
|
||||
- ("x=1;y=2.0&z=2-3.%2b0", {'x': ['1'], 'y': ['2.0'], 'z': ['2-3.+0']}),
|
||||
- ("x=1;y=2.0;z=2-3.%2b0", {'x': ['1'], 'y': ['2.0'], 'z': ['2-3.+0']}),
|
||||
("Hbc5161168c542333633315dee1182227:key_store_seqid=400006&cuyer=r&view=bustomer&order_id=0bb2e248638833d48cb7fed300000f1b&expire=964546263&lobale=en-US&kid=130003.300038&ss=env",
|
||||
{'Hbc5161168c542333633315dee1182227:key_store_seqid': ['400006'],
|
||||
'cuyer': ['r'],
|
||||
@@ -164,6 +159,35 @@ class CgiTests(unittest.TestCase):
|
||||
|
||||
env = {'QUERY_STRING': orig}
|
||||
fs = cgi.FieldStorage(environ=env)
|
||||
+ if isinstance(expect, dict):
|
||||
+ # test dict interface
|
||||
+ self.assertEqual(len(expect), len(fs))
|
||||
+ self.assertCountEqual(expect.keys(), fs.keys())
|
||||
+ self.assertEqual(fs.getvalue("nonexistent field", "default"), "default")
|
||||
+ # test individual fields
|
||||
+ for key in expect.keys():
|
||||
+ expect_val = expect[key]
|
||||
+ self.assertIn(key, fs)
|
||||
+ if len(expect_val) > 1:
|
||||
+ self.assertEqual(fs.getvalue(key), expect_val)
|
||||
+ else:
|
||||
+ self.assertEqual(fs.getvalue(key), expect_val[0])
|
||||
+
|
||||
+ def test_separator(self):
|
||||
+ parse_semicolon = [
|
||||
+ ("x=1;y=2.0", {'x': ['1'], 'y': ['2.0']}),
|
||||
+ ("x=1;y=2.0;z=2-3.%2b0", {'x': ['1'], 'y': ['2.0'], 'z': ['2-3.+0']}),
|
||||
+ (";", ValueError("bad query field: ''")),
|
||||
+ (";;", ValueError("bad query field: ''")),
|
||||
+ ("=;a", ValueError("bad query field: 'a'")),
|
||||
+ (";b=a", ValueError("bad query field: ''")),
|
||||
+ ("b;=a", ValueError("bad query field: 'b'")),
|
||||
+ ("a=a+b;b=b+c", {'a': ['a b'], 'b': ['b c']}),
|
||||
+ ("a=a+b;a=b+a", {'a': ['a b', 'b a']}),
|
||||
+ ]
|
||||
+ for orig, expect in parse_semicolon:
|
||||
+ env = {'QUERY_STRING': orig}
|
||||
+ fs = cgi.FieldStorage(separator=';', environ=env)
|
||||
if isinstance(expect, dict):
|
||||
# test dict interface
|
||||
self.assertEqual(len(expect), len(fs))
|
||||
diff --git a/Lib/test/test_urlparse.py b/Lib/test/test_urlparse.py
|
||||
index 68f633ca3a7..1ec86ba0fc2 100644
|
||||
--- a/Lib/test/test_urlparse.py
|
||||
+++ b/Lib/test/test_urlparse.py
|
||||
@@ -2,6 +2,11 @@ import sys
|
||||
import unicodedata
|
||||
import unittest
|
||||
import urllib.parse
|
||||
+from test.support import EnvironmentVarGuard
|
||||
+from warnings import catch_warnings
|
||||
+import tempfile
|
||||
+import contextlib
|
||||
+import os.path
|
||||
|
||||
RFC1808_BASE = "http://a/b/c/d;p?q#f"
|
||||
RFC2396_BASE = "http://a/b/c/d;p?q"
|
||||
@@ -32,6 +37,9 @@ parse_qsl_test_cases = [
|
||||
(b"&a=b", [(b'a', b'b')]),
|
||||
(b"a=a+b&b=b+c", [(b'a', b'a b'), (b'b', b'b c')]),
|
||||
(b"a=1&a=2", [(b'a', b'1'), (b'a', b'2')]),
|
||||
+]
|
||||
+
|
||||
+parse_qsl_test_cases_semicolon = [
|
||||
(";", []),
|
||||
(";;", []),
|
||||
(";a=b", [('a', 'b')]),
|
||||
@@ -44,6 +52,21 @@ parse_qsl_test_cases = [
|
||||
(b"a=1;a=2", [(b'a', b'1'), (b'a', b'2')]),
|
||||
]
|
||||
|
||||
+parse_qsl_test_cases_legacy = [
|
||||
+ (b"a=1;a=2&a=3", [(b'a', b'1'), (b'a', b'2'), (b'a', b'3')]),
|
||||
+ (b"a=1;b=2&c=3", [(b'a', b'1'), (b'b', b'2'), (b'c', b'3')]),
|
||||
+ (b"a=1&b=2&c=3;", [(b'a', b'1'), (b'b', b'2'), (b'c', b'3')]),
|
||||
+]
|
||||
+
|
||||
+parse_qsl_test_cases_warn = [
|
||||
+ (";a=b", [(';a', 'b')]),
|
||||
+ ("a=a+b;b=b+c", [('a', 'a b;b=b c')]),
|
||||
+ (b";a=b", [(b';a', b'b')]),
|
||||
+ (b"a=a+b;b=b+c", [(b'a', b'a b;b=b c')]),
|
||||
+ ("a=1;a=2&a=3", [('a', '1;a=2'), ('a', '3')]),
|
||||
+ (b"a=1;a=2&a=3", [(b'a', b'1;a=2'), (b'a', b'3')]),
|
||||
+]
|
||||
+
|
||||
# Each parse_qs testcase is a two-tuple that contains
|
||||
# a string with the query and a dictionary with the expected result.
|
||||
|
||||
@@ -68,6 +91,9 @@ parse_qs_test_cases = [
|
||||
(b"&a=b", {b'a': [b'b']}),
|
||||
(b"a=a+b&b=b+c", {b'a': [b'a b'], b'b': [b'b c']}),
|
||||
(b"a=1&a=2", {b'a': [b'1', b'2']}),
|
||||
+]
|
||||
+
|
||||
+parse_qs_test_cases_semicolon = [
|
||||
(";", {}),
|
||||
(";;", {}),
|
||||
(";a=b", {'a': ['b']}),
|
||||
@@ -80,6 +106,24 @@ parse_qs_test_cases = [
|
||||
(b"a=1;a=2", {b'a': [b'1', b'2']}),
|
||||
]
|
||||
|
||||
+parse_qs_test_cases_legacy = [
|
||||
+ ("a=1;a=2&a=3", {'a': ['1', '2', '3']}),
|
||||
+ ("a=1;b=2&c=3", {'a': ['1'], 'b': ['2'], 'c': ['3']}),
|
||||
+ ("a=1&b=2&c=3;", {'a': ['1'], 'b': ['2'], 'c': ['3']}),
|
||||
+ (b"a=1;a=2&a=3", {b'a': [b'1', b'2', b'3']}),
|
||||
+ (b"a=1;b=2&c=3", {b'a': [b'1'], b'b': [b'2'], b'c': [b'3']}),
|
||||
+ (b"a=1&b=2&c=3;", {b'a': [b'1'], b'b': [b'2'], b'c': [b'3']}),
|
||||
+]
|
||||
+
|
||||
+parse_qs_test_cases_warn = [
|
||||
+ (";a=b", {';a': ['b']}),
|
||||
+ ("a=a+b;b=b+c", {'a': ['a b;b=b c']}),
|
||||
+ (b";a=b", {b';a': [b'b']}),
|
||||
+ (b"a=a+b;b=b+c", {b'a':[ b'a b;b=b c']}),
|
||||
+ ("a=1;a=2&a=3", {'a': ['1;a=2', '3']}),
|
||||
+ (b"a=1;a=2&a=3", {b'a': [b'1;a=2', b'3']}),
|
||||
+]
|
||||
+
|
||||
class UrlParseTestCase(unittest.TestCase):
|
||||
|
||||
def checkRoundtrips(self, url, parsed, split):
|
||||
@@ -152,6 +196,40 @@ class UrlParseTestCase(unittest.TestCase):
|
||||
self.assertEqual(result, expect_without_blanks,
|
||||
"Error parsing %r" % orig)
|
||||
|
||||
+ def test_qs_default_warn(self):
|
||||
+ for orig, expect in parse_qs_test_cases_warn:
|
||||
+ with self.subTest(orig=orig, expect=expect):
|
||||
+ with catch_warnings(record=True) as w:
|
||||
+ result = urllib.parse.parse_qs(orig, keep_blank_values=True)
|
||||
+ self.assertEqual(result, expect, "Error parsing %r" % orig)
|
||||
+ self.assertEqual(len(w), 1)
|
||||
+ self.assertEqual(w[0].category, urllib.parse._QueryStringSeparatorWarning)
|
||||
+
|
||||
+ def test_qsl_default_warn(self):
|
||||
+ for orig, expect in parse_qsl_test_cases_warn:
|
||||
+ with self.subTest(orig=orig, expect=expect):
|
||||
+ with catch_warnings(record=True) as w:
|
||||
+ result = urllib.parse.parse_qsl(orig, keep_blank_values=True)
|
||||
+ self.assertEqual(result, expect, "Error parsing %r" % orig)
|
||||
+ self.assertEqual(len(w), 1)
|
||||
+ self.assertEqual(w[0].category, urllib.parse._QueryStringSeparatorWarning)
|
||||
+
|
||||
+ def test_default_qs_no_warnings(self):
|
||||
+ for orig, expect in parse_qs_test_cases:
|
||||
+ with self.subTest(orig=orig, expect=expect):
|
||||
+ with catch_warnings(record=True) as w:
|
||||
+ result = urllib.parse.parse_qs(orig, keep_blank_values=True)
|
||||
+ self.assertEqual(result, expect, "Error parsing %r" % orig)
|
||||
+ self.assertEqual(len(w), 0)
|
||||
+
|
||||
+ def test_default_qsl_no_warnings(self):
|
||||
+ for orig, expect in parse_qsl_test_cases:
|
||||
+ with self.subTest(orig=orig, expect=expect):
|
||||
+ with catch_warnings(record=True) as w:
|
||||
+ result = urllib.parse.parse_qsl(orig, keep_blank_values=True)
|
||||
+ self.assertEqual(result, expect, "Error parsing %r" % orig)
|
||||
+ self.assertEqual(len(w), 0)
|
||||
+
|
||||
def test_roundtrips(self):
|
||||
str_cases = [
|
||||
('file:///tmp/junk.txt',
|
||||
@@ -885,8 +963,151 @@ class UrlParseTestCase(unittest.TestCase):
|
||||
with self.assertRaises(ValueError):
|
||||
urllib.parse.parse_qs('&'.join(['a=a']*11), max_num_fields=10)
|
||||
with self.assertRaises(ValueError):
|
||||
- urllib.parse.parse_qs(';'.join(['a=a']*11), max_num_fields=10)
|
||||
+ urllib.parse.parse_qs(';'.join(['a=a']*11), separator=';', max_num_fields=10)
|
||||
+ with self.assertRaises(ValueError):
|
||||
+ urllib.parse.parse_qs('SEP'.join(['a=a']*11), separator='SEP', max_num_fields=10)
|
||||
urllib.parse.parse_qs('&'.join(['a=a']*10), max_num_fields=10)
|
||||
+ urllib.parse.parse_qs(';'.join(['a=a']*10), separator=';', max_num_fields=10)
|
||||
+ urllib.parse.parse_qs('SEP'.join(['a=a']*10), separator='SEP', max_num_fields=10)
|
||||
+
|
||||
+ def test_parse_qs_separator_bytes(self):
|
||||
+ expected = {b'a': [b'1'], b'b': [b'2']}
|
||||
+
|
||||
+ result = urllib.parse.parse_qs(b'a=1;b=2', separator=b';')
|
||||
+ self.assertEqual(result, expected)
|
||||
+ result = urllib.parse.parse_qs(b'a=1;b=2', separator=';')
|
||||
+ self.assertEqual(result, expected)
|
||||
+ result = urllib.parse.parse_qs('a=1;b=2', separator=';')
|
||||
+ self.assertEqual(result, {'a': ['1'], 'b': ['2']})
|
||||
+
|
||||
+ @contextlib.contextmanager
|
||||
+ def _qsl_sep_config(self, sep):
|
||||
+ """Context for the given parse_qsl default separator configured in config file"""
|
||||
+ old_filename = urllib.parse._QS_SEPARATOR_CONFIG_FILENAME
|
||||
+ urllib.parse._default_qs_separator = None
|
||||
+ try:
|
||||
+ with tempfile.TemporaryDirectory() as tmpdirname:
|
||||
+ filename = os.path.join(tmpdirname, 'conf.cfg')
|
||||
+ with open(filename, 'w') as file:
|
||||
+ file.write(f'[parse_qs]\n')
|
||||
+ file.write(f'PYTHON_URLLIB_QS_SEPARATOR = {sep}')
|
||||
+ urllib.parse._QS_SEPARATOR_CONFIG_FILENAME = filename
|
||||
+ yield
|
||||
+ finally:
|
||||
+ urllib.parse._QS_SEPARATOR_CONFIG_FILENAME = old_filename
|
||||
+ urllib.parse._default_qs_separator = None
|
||||
+
|
||||
+ def test_parse_qs_separator_semicolon(self):
|
||||
+ for orig, expect in parse_qs_test_cases_semicolon:
|
||||
+ with self.subTest(orig=orig, expect=expect, method='arg'):
|
||||
+ result = urllib.parse.parse_qs(orig, separator=';')
|
||||
+ self.assertEqual(result, expect, "Error parsing %r" % orig)
|
||||
+ with self.subTest(orig=orig, expect=expect, method='env'):
|
||||
+ with EnvironmentVarGuard() as environ, catch_warnings(record=True) as w:
|
||||
+ environ['PYTHON_URLLIB_QS_SEPARATOR'] = ';'
|
||||
+ result = urllib.parse.parse_qs(orig)
|
||||
+ self.assertEqual(result, expect, "Error parsing %r" % orig)
|
||||
+ self.assertEqual(len(w), 0)
|
||||
+ with self.subTest(orig=orig, expect=expect, method='conf'):
|
||||
+ with self._qsl_sep_config(';'), catch_warnings(record=True) as w:
|
||||
+ result = urllib.parse.parse_qs(orig)
|
||||
+ self.assertEqual(result, expect, "Error parsing %r" % orig)
|
||||
+ self.assertEqual(len(w), 0)
|
||||
+
|
||||
+ def test_parse_qsl_separator_semicolon(self):
|
||||
+ for orig, expect in parse_qsl_test_cases_semicolon:
|
||||
+ with self.subTest(orig=orig, expect=expect, method='arg'):
|
||||
+ result = urllib.parse.parse_qsl(orig, separator=';')
|
||||
+ self.assertEqual(result, expect, "Error parsing %r" % orig)
|
||||
+ with self.subTest(orig=orig, expect=expect, method='env'):
|
||||
+ with EnvironmentVarGuard() as environ, catch_warnings(record=True) as w:
|
||||
+ environ['PYTHON_URLLIB_QS_SEPARATOR'] = ';'
|
||||
+ result = urllib.parse.parse_qsl(orig)
|
||||
+ self.assertEqual(result, expect, "Error parsing %r" % orig)
|
||||
+ self.assertEqual(len(w), 0)
|
||||
+ with self.subTest(orig=orig, expect=expect, method='conf'):
|
||||
+ with self._qsl_sep_config(';'), catch_warnings(record=True) as w:
|
||||
+ result = urllib.parse.parse_qsl(orig)
|
||||
+ self.assertEqual(result, expect, "Error parsing %r" % orig)
|
||||
+ self.assertEqual(len(w), 0)
|
||||
+
|
||||
+ def test_parse_qs_separator_legacy(self):
|
||||
+ for orig, expect in parse_qs_test_cases_legacy:
|
||||
+ with self.subTest(orig=orig, expect=expect, method='env'):
|
||||
+ with EnvironmentVarGuard() as environ, catch_warnings(record=True) as w:
|
||||
+ environ['PYTHON_URLLIB_QS_SEPARATOR'] = 'legacy'
|
||||
+ result = urllib.parse.parse_qs(orig)
|
||||
+ self.assertEqual(result, expect, "Error parsing %r" % orig)
|
||||
+ self.assertEqual(len(w), 0)
|
||||
+ with self.subTest(orig=orig, expect=expect, method='conf'):
|
||||
+ with self._qsl_sep_config('legacy'), catch_warnings(record=True) as w:
|
||||
+ result = urllib.parse.parse_qs(orig)
|
||||
+ self.assertEqual(result, expect, "Error parsing %r" % orig)
|
||||
+ self.assertEqual(len(w), 0)
|
||||
+
|
||||
+ def test_parse_qsl_separator_legacy(self):
|
||||
+ for orig, expect in parse_qsl_test_cases_legacy:
|
||||
+ with self.subTest(orig=orig, expect=expect, method='env'):
|
||||
+ with EnvironmentVarGuard() as environ, catch_warnings(record=True) as w:
|
||||
+ environ['PYTHON_URLLIB_QS_SEPARATOR'] = 'legacy'
|
||||
+ result = urllib.parse.parse_qsl(orig)
|
||||
+ self.assertEqual(result, expect, "Error parsing %r" % orig)
|
||||
+ self.assertEqual(len(w), 0)
|
||||
+ with self.subTest(orig=orig, expect=expect, method='conf'):
|
||||
+ with self._qsl_sep_config('legacy'), catch_warnings(record=True) as w:
|
||||
+ result = urllib.parse.parse_qsl(orig)
|
||||
+ self.assertEqual(result, expect, "Error parsing %r" % orig)
|
||||
+ self.assertEqual(len(w), 0)
|
||||
+
|
||||
+ def test_parse_qs_separator_bad_value_env_or_config(self):
|
||||
+ for bad_sep in '', 'abc', 'safe', '&;', 'SEP':
|
||||
+ with self.subTest(bad_sep, method='env'):
|
||||
+ with EnvironmentVarGuard() as environ, catch_warnings(record=True) as w:
|
||||
+ environ['PYTHON_URLLIB_QS_SEPARATOR'] = bad_sep
|
||||
+ with self.assertRaises(ValueError):
|
||||
+ urllib.parse.parse_qsl('a=1;b=2')
|
||||
+ with self.subTest(bad_sep, method='conf'):
|
||||
+ with self._qsl_sep_config('bad_sep'), catch_warnings(record=True) as w:
|
||||
+ with self.assertRaises(ValueError):
|
||||
+ urllib.parse.parse_qsl('a=1;b=2')
|
||||
+
|
||||
+ def test_parse_qs_separator_bad_value_arg(self):
|
||||
+ for bad_sep in True, {}, '':
|
||||
+ with self.subTest(bad_sep):
|
||||
+ with self.assertRaises(ValueError):
|
||||
+ urllib.parse.parse_qsl('a=1;b=2', separator=bad_sep)
|
||||
+
|
||||
+ def test_parse_qs_separator_num_fields(self):
|
||||
+ for qs, sep in (
|
||||
+ ('a&b&c', '&'),
|
||||
+ ('a;b;c', ';'),
|
||||
+ ('a&b;c', 'legacy'),
|
||||
+ ):
|
||||
+ with self.subTest(qs=qs, sep=sep):
|
||||
+ with EnvironmentVarGuard() as environ, catch_warnings(record=True) as w:
|
||||
+ if sep != 'legacy':
|
||||
+ with self.assertRaises(ValueError):
|
||||
+ urllib.parse.parse_qsl(qs, separator=sep, max_num_fields=2)
|
||||
+ if sep:
|
||||
+ environ['PYTHON_URLLIB_QS_SEPARATOR'] = sep
|
||||
+ with self.assertRaises(ValueError):
|
||||
+ urllib.parse.parse_qsl(qs, max_num_fields=2)
|
||||
+
|
||||
+ def test_parse_qs_separator_priority(self):
|
||||
+ # env variable trumps config file
|
||||
+ with self._qsl_sep_config('~'), EnvironmentVarGuard() as environ:
|
||||
+ environ['PYTHON_URLLIB_QS_SEPARATOR'] = '!'
|
||||
+ result = urllib.parse.parse_qs('a=1!b=2~c=3')
|
||||
+ self.assertEqual(result, {'a': ['1'], 'b': ['2~c=3']})
|
||||
+ # argument trumps config file
|
||||
+ with self._qsl_sep_config('~'):
|
||||
+ result = urllib.parse.parse_qs('a=1$b=2~c=3', separator='$')
|
||||
+ self.assertEqual(result, {'a': ['1'], 'b': ['2~c=3']})
|
||||
+ # argument trumps env variable
|
||||
+ with EnvironmentVarGuard() as environ:
|
||||
+ environ['PYTHON_URLLIB_QS_SEPARATOR'] = '~'
|
||||
+ result = urllib.parse.parse_qs('a=1$b=2~c=3', separator='$')
|
||||
+ self.assertEqual(result, {'a': ['1'], 'b': ['2~c=3']})
|
||||
|
||||
def test_urlencode_sequences(self):
|
||||
# Other tests incidentally urlencode things; test non-covered cases:
|
||||
diff --git a/Lib/urllib/parse.py b/Lib/urllib/parse.py
|
||||
index fa8827a9fa7..57b8fcf8bbd 100644
|
||||
--- a/Lib/urllib/parse.py
|
||||
+++ b/Lib/urllib/parse.py
|
||||
@@ -28,6 +28,7 @@ test_urlparse.py provides a good indicator of parsing behavior.
|
||||
"""
|
||||
|
||||
import re
|
||||
+import os
|
||||
import sys
|
||||
import collections
|
||||
|
||||
@@ -644,7 +645,8 @@ def unquote(string, encoding='utf-8', errors='replace'):
|
||||
|
||||
|
||||
def parse_qs(qs, keep_blank_values=False, strict_parsing=False,
|
||||
- encoding='utf-8', errors='replace', max_num_fields=None):
|
||||
+ encoding='utf-8', errors='replace', max_num_fields=None,
|
||||
+ separator=None):
|
||||
"""Parse a query given as a string argument.
|
||||
|
||||
Arguments:
|
||||
@@ -673,7 +675,8 @@ def parse_qs(qs, keep_blank_values=False, strict_parsing=False,
|
||||
parsed_result = {}
|
||||
pairs = parse_qsl(qs, keep_blank_values, strict_parsing,
|
||||
encoding=encoding, errors=errors,
|
||||
- max_num_fields=max_num_fields)
|
||||
+ max_num_fields=max_num_fields,
|
||||
+ separator=separator)
|
||||
for name, value in pairs:
|
||||
if name in parsed_result:
|
||||
parsed_result[name].append(value)
|
||||
@@ -681,9 +684,16 @@ def parse_qs(qs, keep_blank_values=False, strict_parsing=False,
|
||||
parsed_result[name] = [value]
|
||||
return parsed_result
|
||||
|
||||
+class _QueryStringSeparatorWarning(RuntimeWarning):
|
||||
+ """Warning for using default `separator` in parse_qs or parse_qsl"""
|
||||
+
|
||||
+# The default "separator" for parse_qsl can be specified in a config file.
|
||||
+# It's cached after first read.
|
||||
+_QS_SEPARATOR_CONFIG_FILENAME = '/etc/python/urllib.cfg'
|
||||
+_default_qs_separator = None
|
||||
|
||||
def parse_qsl(qs, keep_blank_values=False, strict_parsing=False,
|
||||
- encoding='utf-8', errors='replace', max_num_fields=None):
|
||||
+ encoding='utf-8', errors='replace', max_num_fields=None, separator=None):
|
||||
"""Parse a query given as a string argument.
|
||||
|
||||
Arguments:
|
||||
@@ -710,15 +720,77 @@ def parse_qsl(qs, keep_blank_values=False, strict_parsing=False,
|
||||
"""
|
||||
qs, _coerce_result = _coerce_args(qs)
|
||||
|
||||
+ if isinstance(separator, bytes):
|
||||
+ separator = separator.decode('ascii')
|
||||
+
|
||||
+ if (not separator or (not isinstance(separator, (str, bytes)))) and separator is not None:
|
||||
+ raise ValueError("Separator must be of type string or bytes.")
|
||||
+
|
||||
+ # Used when both "&" and ";" act as separators. (Need a non-string value.)
|
||||
+ _legacy = object()
|
||||
+
|
||||
+ if separator is None:
|
||||
+ global _default_qs_separator
|
||||
+ separator = _default_qs_separator
|
||||
+ envvar_name = 'PYTHON_URLLIB_QS_SEPARATOR'
|
||||
+ if separator is None:
|
||||
+ # Set default separator from environment variable
|
||||
+ separator = os.environ.get(envvar_name)
|
||||
+ config_source = 'environment variable'
|
||||
+ if separator is None:
|
||||
+ # Set default separator from the configuration file
|
||||
+ try:
|
||||
+ file = open(_QS_SEPARATOR_CONFIG_FILENAME)
|
||||
+ except FileNotFoundError:
|
||||
+ pass
|
||||
+ else:
|
||||
+ with file:
|
||||
+ import configparser
|
||||
+ config = configparser.ConfigParser(
|
||||
+ interpolation=None,
|
||||
+ comment_prefixes=('#', ),
|
||||
+ )
|
||||
+ config.read_file(file)
|
||||
+ separator = config.get('parse_qs', envvar_name, fallback=None)
|
||||
+ _default_qs_separator = separator
|
||||
+ config_source = _QS_SEPARATOR_CONFIG_FILENAME
|
||||
+ if separator is None:
|
||||
+ # The default is '&', but warn if not specified explicitly
|
||||
+ if ';' in qs:
|
||||
+ from warnings import warn
|
||||
+ warn("The default separator of urllib.parse.parse_qsl and "
|
||||
+ + "parse_qs was changed to '&' to avoid a web cache "
|
||||
+ + "poisoning issue (CVE-2021-23336). "
|
||||
+ + "By default, semicolons no longer act as query field "
|
||||
+ + "separators. "
|
||||
+ + "See https://access.redhat.com/articles/5860431 for "
|
||||
+ + "more details.",
|
||||
+ _QueryStringSeparatorWarning, stacklevel=2)
|
||||
+ separator = '&'
|
||||
+ elif separator == 'legacy':
|
||||
+ separator = _legacy
|
||||
+ elif len(separator) != 1:
|
||||
+ raise ValueError(
|
||||
+ f'{envvar_name} (from {config_source}) must contain '
|
||||
+ + '1 character, or "legacy". See '
|
||||
+ + 'https://access.redhat.com/articles/5860431 for more details.'
|
||||
+ )
|
||||
+
|
||||
# If max_num_fields is defined then check that the number of fields
|
||||
# is less than max_num_fields. This prevents a memory exhaustion DOS
|
||||
# attack via post bodies with many fields.
|
||||
if max_num_fields is not None:
|
||||
- num_fields = 1 + qs.count('&') + qs.count(';')
|
||||
+ if separator is _legacy:
|
||||
+ num_fields = 1 + qs.count('&') + qs.count(';')
|
||||
+ else:
|
||||
+ num_fields = 1 + qs.count(separator)
|
||||
if max_num_fields < num_fields:
|
||||
raise ValueError('Max number of fields exceeded')
|
||||
|
||||
- pairs = [s2 for s1 in qs.split('&') for s2 in s1.split(';')]
|
||||
+ if separator is _legacy:
|
||||
+ pairs = [s2 for s1 in qs.split('&') for s2 in s1.split(';')]
|
||||
+ else:
|
||||
+ pairs = [s1 for s1 in qs.split(separator)]
|
||||
r = []
|
||||
for name_value in pairs:
|
||||
if not name_value and not strict_parsing:
|
||||
diff --git a/Misc/NEWS.d/next/Security/2021-02-14-15-59-16.bpo-42967.YApqDS.rst b/Misc/NEWS.d/next/Security/2021-02-14-15-59-16.bpo-42967.YApqDS.rst
|
||||
new file mode 100644
|
||||
index 00000000000..bc82c963067
|
||||
--- /dev/null
|
||||
+++ b/Misc/NEWS.d/next/Security/2021-02-14-15-59-16.bpo-42967.YApqDS.rst
|
||||
@@ -0,0 +1 @@
|
||||
+Make it possible to fix web cache poisoning vulnerability by allowing the user to choose a custom separator query args.
|
@ -14,7 +14,7 @@ URL: https://www.python.org/
|
||||
# WARNING When rebasing to a new Python version,
|
||||
# remember to update the python3-docs package as well
|
||||
Version: %{pybasever}.8
|
||||
Release: 31%{?dist}
|
||||
Release: 37%{?dist}
|
||||
License: Python
|
||||
|
||||
|
||||
@ -56,6 +56,15 @@ License: Python
|
||||
%bcond_with valgrind
|
||||
%endif
|
||||
|
||||
# https://fedoraproject.org/wiki/Changes/Python_Upstream_Architecture_Names
|
||||
# For a very long time we have converted "upstream architecture names" to "Fedora names".
|
||||
# This made sense at the time, see https://github.com/pypa/manylinux/issues/687#issuecomment-666362947
|
||||
# However, with manylinux wheels popularity growth, this is now a problem.
|
||||
# Wheels built on a Linux that doesn't do this were not compatible with ours and vice versa.
|
||||
# We now have a compatibility layer to workaround a problem,
|
||||
# but we also no longer use the legacy arch names in Fedora 34+.
|
||||
# This bcond controls the behavior. The defaults should be good for anybody.
|
||||
%bcond_without legacy_archnames
|
||||
|
||||
# ==================================
|
||||
# Notes from bootstraping Python 3.6
|
||||
@ -111,8 +120,21 @@ License: Python
|
||||
%global LDVERSION_optimized %{pybasever}%{ABIFLAGS_optimized}
|
||||
%global LDVERSION_debug %{pybasever}%{ABIFLAGS_debug}
|
||||
|
||||
%global SOABI_optimized cpython-%{pyshortver}%{ABIFLAGS_optimized}-%{_arch}-linux%{_gnu}
|
||||
%global SOABI_debug cpython-%{pyshortver}%{ABIFLAGS_debug}-%{_arch}-linux%{_gnu}
|
||||
# When we use the upstream arch triplets, we convert them from the legacy ones
|
||||
# This is reversed in prep when %%with legacy_archnames, so we keep both macros
|
||||
%global platform_triplet_legacy %{_arch}-linux%{_gnu}
|
||||
%global platform_triplet_upstream %{expand:%(echo %{platform_triplet_legacy} | sed -E \\
|
||||
-e 's/^arm(eb)?-linux-gnueabi$/arm\\1-linux-gnueabihf/' \\
|
||||
-e 's/^mips64(el)?-linux-gnu$/mips64\\1-linux-gnuabi64/' \\
|
||||
-e 's/^ppc(64)?(le)?-linux-gnu$/powerpc\\1\\2-linux-gnu/')}
|
||||
%if %{with legacy_archnames}
|
||||
%global platform_triplet %{platform_triplet_legacy}
|
||||
%else
|
||||
%global platform_triplet %{platform_triplet_upstream}
|
||||
%endif
|
||||
|
||||
%global SOABI_optimized cpython-%{pyshortver}%{ABIFLAGS_optimized}-%{platform_triplet}
|
||||
%global SOABI_debug cpython-%{pyshortver}%{ABIFLAGS_debug}-%{platform_triplet}
|
||||
|
||||
# All bytecode files are in a __pycache__ subdirectory, with a name
|
||||
# reflecting the version of the bytecode.
|
||||
@ -329,10 +351,6 @@ Patch251: 00251-change-user-install-location.patch
|
||||
# Original proposal: https://bugzilla.redhat.com/show_bug.cgi?id=1404918
|
||||
Patch262: 00262-pep538_coerce_legacy_c_locale.patch
|
||||
|
||||
# 00274 #
|
||||
# Upstream uses Debian-style architecture naming. Change to match Fedora.
|
||||
Patch274: 00274-fix-arch-names.patch
|
||||
|
||||
# 00294 #
|
||||
# Define TLS cipher suite on build time depending
|
||||
# on the OpenSSL default cipher suite selection.
|
||||
@ -519,6 +537,60 @@ Patch351: 00351-avoid-infinite-loop-in-the-tarfile-module.patch
|
||||
# Fixed upstream: https://bugs.python.org/issue41004
|
||||
Patch352: 00352-resolve-hash-collisions-for-ipv4interface-and-ipv6interface.patch
|
||||
|
||||
# 00353 #
|
||||
# Original names for architectures with different names downstream
|
||||
#
|
||||
# https://fedoraproject.org/wiki/Changes/Python_Upstream_Architecture_Names
|
||||
#
|
||||
# Pythons in RHEL/Fedora used different names for some architectures
|
||||
# than upstream and other distros (for example ppc64 vs. powerpc64).
|
||||
# This was patched in patch 274, now it is sedded if %%with legacy_archnames.
|
||||
#
|
||||
# That meant that an extension built with the default upstream settings
|
||||
# (on other distro or as an manylinux wheel) could not been found by Python
|
||||
# on RHEL/Fedora because it had a different suffix.
|
||||
# This patch adds the legacy names to importlib so Python is able
|
||||
# to import extensions with a legacy architecture name in its
|
||||
# file name.
|
||||
# It work both ways, so it support both %%with and %%without legacy_archnames.
|
||||
#
|
||||
# WARNING: This patch has no effect on Python built with bootstrap
|
||||
# enabled because Python/importlib_external.h is not regenerated
|
||||
# and therefore Python during bootstrap contains importlib from
|
||||
# upstream without this feature. It's possible to include
|
||||
# Python/importlib_external.h to this patch but it'd make rebasing
|
||||
# a nightmare because it's basically a binary file.
|
||||
Patch353: 00353-architecture-names-upstream-downstream.patch
|
||||
|
||||
# 00354 #
|
||||
# Reject control chars in HTTP method in http.client to prevent
|
||||
# HTTP header injection
|
||||
# Fixed ustream: https://bugs.python.org/issue39603
|
||||
Patch354: 00354-cve-2020-26116-http-request-method-crlf-injection-in-httplib.patch
|
||||
|
||||
# 00355 #
|
||||
# No longer call eval() on content received via HTTP in the CJK codec tests
|
||||
# Fixed upstream: https://bugs.python.org/issue41944
|
||||
Patch355: 00355-CVE-2020-27619.patch
|
||||
|
||||
# 00356 #
|
||||
# options -a and -k for pathfix.py used in %%py3_shebang_fix
|
||||
# Upstream: https://github.com/python/cpython/commit/c71c54c62600fd721baed3c96709e3d6e9c33817
|
||||
Patch356: 00356-k_and_a_options_for_pathfix.patch
|
||||
|
||||
# 00357 #
|
||||
# CVE-2021-3177 stack-based buffer overflow in PyCArg_repr in _ctypes/callproc.c
|
||||
# Upstream: https://bugs.python.org/issue42938
|
||||
# Main BZ: https://bugzilla.redhat.com/show_bug.cgi?id=1918168
|
||||
Patch357: 00357-CVE-2021-3177.patch
|
||||
|
||||
# 00359 #
|
||||
# CVE-2021-23336 python: Web Cache Poisoning via urllib.parse.parse_qsl and
|
||||
# urllib.parse.parse_qs by using a semicolon in query parameters
|
||||
# Upstream: https://bugs.python.org/issue42967
|
||||
# Main BZ: https://bugzilla.redhat.com/show_bug.cgi?id=1928904
|
||||
Patch359: 00359-CVE-2021-23336.patch
|
||||
|
||||
# (New patches go here ^^^)
|
||||
#
|
||||
# When adding new patches to "python" and "python3" in Fedora, EL, etc.,
|
||||
@ -817,7 +889,6 @@ rm Lib/ensurepip/_bundled/*.whl
|
||||
|
||||
%patch251 -p1
|
||||
%patch262 -p1
|
||||
%patch274 -p1
|
||||
%patch294 -p1
|
||||
%patch316 -p1
|
||||
%patch317 -p1
|
||||
@ -841,11 +912,23 @@ rm Lib/ensurepip/_bundled/*.whl
|
||||
git apply %{PATCH351}
|
||||
|
||||
%patch352 -p1
|
||||
%patch353 -p1
|
||||
%patch354 -p1
|
||||
%patch355 -p1
|
||||
%patch356 -p1
|
||||
%patch357 -p1
|
||||
%patch359 -p1
|
||||
|
||||
# Remove files that should be generated by the build
|
||||
# (This is after patching, so that we can use patches directly from upstream)
|
||||
rm configure pyconfig.h.in
|
||||
|
||||
# When we use the legacy arch names, we need to change them in configure.ac
|
||||
%if %{with legacy_archnames}
|
||||
sed -i configure.ac \
|
||||
-e 's/\b%{platform_triplet_upstream}\b/%{platform_triplet_legacy}/'
|
||||
%endif
|
||||
|
||||
|
||||
# ======================================================
|
||||
# Configuring and building the code:
|
||||
@ -926,6 +1009,9 @@ BuildPython() {
|
||||
$ExtraConfigArgs \
|
||||
%{nil}
|
||||
|
||||
# Regenerate generated importlib frozen modules (see patch 353)
|
||||
%make_build CFLAGS_NODIST="$CFLAGS_NODIST $MoreCFlags" regen-importlib
|
||||
|
||||
# Invoke the build
|
||||
%make_build CFLAGS_NODIST="$CFLAGS_NODIST $MoreCFlags"
|
||||
|
||||
@ -1125,7 +1211,7 @@ do
|
||||
LD_LIBRARY_PATH=./build/optimized ./build/optimized/python \
|
||||
Tools/scripts/pathfix.py \
|
||||
-i "%{_libexecdir}/platform-python${LDVersion}" -pn \
|
||||
%{buildroot}%{pylibdir}/config-${LDVersion}-%{_arch}-linux%{_gnu}/python-config.py
|
||||
%{buildroot}%{pylibdir}/config-${LDVersion}-%{platform_triplet}/python-config.py
|
||||
done
|
||||
|
||||
# Remove tests for python3-tools which was removed in
|
||||
@ -1545,8 +1631,8 @@ fi
|
||||
# "Makefile" and the config-32/64.h file are needed by
|
||||
# distutils/sysconfig.py:_init_posix(), so we include them in the core
|
||||
# package, along with their parent directories (bug 531901):
|
||||
%dir %{pylibdir}/config-%{LDVERSION_optimized}-%{_arch}-linux%{_gnu}/
|
||||
%{pylibdir}/config-%{LDVERSION_optimized}-%{_arch}-linux%{_gnu}/Makefile
|
||||
%dir %{pylibdir}/config-%{LDVERSION_optimized}-%{platform_triplet}/
|
||||
%{pylibdir}/config-%{LDVERSION_optimized}-%{platform_triplet}/Makefile
|
||||
%dir %{_includedir}/python%{LDVERSION_optimized}/
|
||||
%{_includedir}/python%{LDVERSION_optimized}/%{_pyconfig_h}
|
||||
|
||||
@ -1560,8 +1646,8 @@ fi
|
||||
%{_bindir}/2to3
|
||||
# TODO: Remove 2to3-3.7 once rebased to 3.7
|
||||
%{_bindir}/2to3-%{pybasever}
|
||||
%{pylibdir}/config-%{LDVERSION_optimized}-%{_arch}-linux%{_gnu}/*
|
||||
%exclude %{pylibdir}/config-%{LDVERSION_optimized}-%{_arch}-linux%{_gnu}/Makefile
|
||||
%{pylibdir}/config-%{LDVERSION_optimized}-%{platform_triplet}/*
|
||||
%exclude %{pylibdir}/config-%{LDVERSION_optimized}-%{platform_triplet}/Makefile
|
||||
%exclude %{pylibdir}/distutils/command/wininst-*.exe
|
||||
%{_includedir}/python%{LDVERSION_optimized}/*.h
|
||||
%exclude %{_includedir}/python%{LDVERSION_optimized}/%{_pyconfig_h}
|
||||
@ -1709,7 +1795,7 @@ fi
|
||||
%{_libdir}/%{py_INSTSONAME_debug}
|
||||
|
||||
# Analog of the -devel subpackage's files:
|
||||
%{pylibdir}/config-%{LDVERSION_debug}-%{_arch}-linux%{_gnu}
|
||||
%{pylibdir}/config-%{LDVERSION_debug}-%{platform_triplet}
|
||||
%{_includedir}/python%{LDVERSION_debug}
|
||||
|
||||
%exclude %{_bindir}/python%{LDVERSION_debug}-config
|
||||
@ -1757,6 +1843,31 @@ fi
|
||||
# ======================================================
|
||||
|
||||
%changelog
|
||||
* Thu Mar 04 2021 Petr Viktorin <pviktori@redhat.com> - 3.6.8-37
|
||||
- Fix for CVE-2021-23336
|
||||
Resolves: rhbz#1928904
|
||||
|
||||
* Fri Jan 22 2021 Lumír Balhar <lbalhar@redhat.com> - 3.6.8-36
|
||||
- Fix for CVE-2021-3177
|
||||
Resolves: rhbz#1918168
|
||||
|
||||
* Mon Jan 18 2021 Lumír Balhar <lbalhar@redhat.com> - 3.6.8-35
|
||||
- New options -a and -k for pathfix.py script backported from upstream
|
||||
Resolves: rhbz#1917691
|
||||
|
||||
* Fri Dec 04 2020 Charalampos Stratakis <cstratak@redhat.com> - 3.6.8-34
|
||||
- Security fix for CVE-2020-27619: eval() call on content received via HTTP in the CJK codec tests
|
||||
Resolves: rhbz#1890237
|
||||
|
||||
* Tue Nov 24 2020 Lumír Balhar <lbalhar@redhat.com> - 3.6.8-33
|
||||
- Add support for upstream architecture names
|
||||
https://fedoraproject.org/wiki/Changes/Python_Upstream_Architecture_Names
|
||||
Resolves: rhbz#1868003
|
||||
|
||||
* Mon Nov 09 2020 Charalampos Stratakis <cstratak@redhat.com> - 3.6.8-32
|
||||
- Security fix for CVE-2020-26116: Reject control chars in HTTP method in http.client
|
||||
Resolves: rhbz#1883257
|
||||
|
||||
* Mon Aug 17 2020 Tomas Orsava <torsava@redhat.com> - 3.6.8-31
|
||||
- Avoid infinite loop when reading specially crafted TAR files (CVE-2019-20907)
|
||||
Resolves: rhbz#1856481
|
||||
|
Loading…
Reference in New Issue
Block a user