175 lines
5.6 KiB
Diff
175 lines
5.6 KiB
Diff
From 71340653cd375c01ada053c63d7c55d0ca475b47 Mon Sep 17 00:00:00 2001
|
||
From: ph10 <ph10@6239d852-aaf2-0410-a92c-79f79f948069>
|
||
Date: Fri, 3 Aug 2018 09:38:36 +0000
|
||
Subject: [PATCH] Make /x more Perl-compatible by recognizing all of Unicode's
|
||
"Pattern White Space" characters, not just the ASCII ones.
|
||
MIME-Version: 1.0
|
||
Content-Type: text/plain; charset=UTF-8
|
||
Content-Transfer-Encoding: 8bit
|
||
|
||
git-svn-id: svn://vcs.exim.org/pcre2/code/trunk@977 6239d852-aaf2-0410-a92c-79f79f948069
|
||
|
||
Petr Písař: Ported to 10.31.
|
||
|
||
Signed-off-by: Petr Písař <ppisar@redhat.com>
|
||
---
|
||
src/pcre2_compile.c | 25 +++++++++++++++++++------
|
||
testdata/testinput4 | 15 +++++++++++++++
|
||
testdata/testinput5 | 13 +++++++++++++
|
||
testdata/testoutput4 | 18 ++++++++++++++++++
|
||
testdata/testoutput5 | 16 ++++++++++++++++
|
||
5 files changed, 81 insertions(+), 6 deletions(-)
|
||
|
||
diff --git a/src/pcre2_compile.c b/src/pcre2_compile.c
|
||
index 7ff8b4c..1d62a38 100644
|
||
--- a/src/pcre2_compile.c
|
||
+++ b/src/pcre2_compile.c
|
||
@@ -2434,11 +2434,17 @@ while (ptr < ptrend)
|
||
/* EITHER: not both options set */
|
||
((options & (PCRE2_EXTENDED | PCRE2_ALT_VERBNAMES)) !=
|
||
(PCRE2_EXTENDED | PCRE2_ALT_VERBNAMES)) ||
|
||
- /* OR: character > 255 */
|
||
- c > 255 ||
|
||
- /* OR: not a # comment or white space */
|
||
- (c != CHAR_NUMBER_SIGN && (cb->ctypes[c] & ctype_space) == 0)
|
||
- ))
|
||
+#ifdef SUPPORT_UNICODE
|
||
+ /* OR: character > 255 AND not Unicode Pattern White Space */
|
||
+ (c > 255 && (c|1) != 0x200f && (c|1) != 0x2029) ||
|
||
+#endif
|
||
+ /* OR: not a # comment or isspace() white space */
|
||
+ (c < 256 && c != CHAR_NUMBER_SIGN && (cb->ctypes[c] & ctype_space) == 0
|
||
+#ifdef SUPPORT_UNICODE
|
||
+ /* and not CHAR_NEL when Unicode is supported */
|
||
+ && c != CHAR_NEL
|
||
+#endif
|
||
+ )))
|
||
{
|
||
PCRE2_SIZE verbnamelength;
|
||
|
||
@@ -2510,11 +2516,18 @@ while (ptr < ptrend)
|
||
|
||
/* Skip over whitespace and # comments in extended mode. Note that c is a
|
||
character, not a code unit, so we must not use MAX_255 to test its size
|
||
- because MAX_255 tests code units and is assumed TRUE in 8-bit mode. */
|
||
+ because MAX_255 tests code units and is assumed TRUE in 8-bit mode. The
|
||
+ whitespace characters are those designated as "Pattern White Space" by
|
||
+ Unicode, which are the isspace() characters plus CHAR_NEL (newline), which is
|
||
+ U+0085 in Unicode, plus U+200E, U+200F, U+2028, and U+2029. These are a
|
||
+ subset of space characters that match \h and \v. */
|
||
|
||
if ((options & PCRE2_EXTENDED) != 0)
|
||
{
|
||
if (c < 256 && (cb->ctypes[c] & ctype_space) != 0) continue;
|
||
+#ifdef SUPPORT_UNICODE
|
||
+ if (c == CHAR_NEL || (c|1) == 0x200f || (c|1) == 0x2029) continue;
|
||
+#endif
|
||
if (c == CHAR_NUMBER_SIGN)
|
||
{
|
||
while (ptr < ptrend)
|
||
diff --git a/testdata/testinput4 b/testdata/testinput4
|
||
index 0ef7b8e..6884f60 100644
|
||
--- a/testdata/testinput4
|
||
+++ b/testdata/testinput4
|
||
@@ -2300,5 +2300,20 @@
|
||
\x{123}\x{122}\x{123}
|
||
\= Expect no match
|
||
\x{123}\x{124}\x{123}
|
||
+
|
||
+# Test the full list of Unicode "Pattern White Space" characters that are to
|
||
+# be ignored by /x. The pattern lines below may show up oddly in text editors
|
||
+# or when listed to the screen. Note that characters such as U+2002, which are
|
||
+# matched as space by \h and \v are *not* "Pattern White Space".
|
||
+
|
||
+/A
B/x,utf
|
||
+ AB
|
||
+
|
||
+/A B/x,utf
|
||
+ A\x{2002}B
|
||
+\= Expect no match
|
||
+ AB
|
||
+
|
||
+# -------
|
||
|
||
# End of testinput4
|
||
diff --git a/testdata/testinput5 b/testdata/testinput5
|
||
index 0366136..ebeee07 100644
|
||
--- a/testdata/testinput5
|
||
+++ b/testdata/testinput5
|
||
@@ -2059,5 +2059,18 @@
|
||
\x{1F1E6}\x{1F1E7}\x{1F1E7}B
|
||
\x{1F1E6}\x{1F1E7}\x{1F1E7}\x{1F1E6}B
|
||
|
||
+# This tests the non-UTF Unicode NEL pattern whitespace character, only
|
||
+# recognized by PCRE2 with /x when there is Unicode support.
|
||
+
|
||
+/A
|
||
+
|
||
?B/x
|
||
+ AB
|
||
+
|
||
+# This tests Unicode Pattern White Space characters in verb names when they
|
||
+# are being processed with PCRE2_EXTENDED. Note: there are UTF-8 characters
|
||
+# with code points greater than 255 between A, B, and C in the pattern.
|
||
+
|
||
+/(*: AB
C)abc/x,utf,mark,alt_verbnames
|
||
+ abc
|
||
|
||
# End of testinput5
|
||
diff --git a/testdata/testoutput4 b/testdata/testoutput4
|
||
index 6056e6d..51c8219 100644
|
||
--- a/testdata/testoutput4
|
||
+++ b/testdata/testoutput4
|
||
@@ -3728,5 +3728,23 @@ No match
|
||
\= Expect no match
|
||
\x{123}\x{124}\x{123}
|
||
No match
|
||
+
|
||
+# Test the full list of Unicode "Pattern White Space" characters that are to
|
||
+# be ignored by /x. The pattern lines below may show up oddly in text editors
|
||
+# or when listed to the screen. Note that characters such as U+2002, which are
|
||
+# matched as space by \h and \v are *not* "Pattern White Space".
|
||
+
|
||
+/A
B/x,utf
|
||
+ AB
|
||
+ 0: AB
|
||
+
|
||
+/A B/x,utf
|
||
+ A\x{2002}B
|
||
+ 0: A\x{2002}B
|
||
+\= Expect no match
|
||
+ AB
|
||
+No match
|
||
+
|
||
+# -------
|
||
|
||
# End of testinput4
|
||
diff --git a/testdata/testoutput5 b/testdata/testoutput5
|
||
index 4b3171c..1392e98 100644
|
||
--- a/testdata/testoutput5
|
||
+++ b/testdata/testoutput5
|
||
@@ -4700,5 +4700,21 @@ Callout 0: last capture = 1
|
||
1: \x{1f1e6}\x{1f1e7}
|
||
2: \x{1f1e7}\x{1f1e6}
|
||
|
||
+# This tests the non-UTF Unicode NEL pattern whitespace character, only
|
||
+# recognized by PCRE2 with /x when there is Unicode support.
|
||
+
|
||
+/A
|
||
+
|
||
?B/x
|
||
+ AB
|
||
+ 0: AB
|
||
+
|
||
+# This tests Unicode Pattern White Space characters in verb names when they
|
||
+# are being processed with PCRE2_EXTENDED. Note: there are UTF-8 characters
|
||
+# with code points greater than 255 between A, B, and C in the pattern.
|
||
+
|
||
+/(*: AB
C)abc/x,utf,mark,alt_verbnames
|
||
+ abc
|
||
+ 0: abc
|
||
+MK: ABC
|
||
|
||
# End of testinput5
|
||
--
|
||
2.14.4
|
||
|