forked from rpms/glibc
261 lines
9.7 KiB
Diff
261 lines
9.7 KiB
Diff
commit 5e0a7ecb6629461b28adc1a5aabcc0ede122f201
|
|
Author: Wilco Dijkstra <wdijkstr@arm.com>
|
|
Date: Wed Jun 12 11:38:52 2019 +0100
|
|
|
|
Improve performance of strstr
|
|
|
|
This patch significantly improves performance of strstr using a novel
|
|
modified Horspool algorithm. Needles up to size 256 use a bad-character
|
|
table indexed by hashed pairs of characters to quickly skip past mismatches.
|
|
Long needles use a self-adapting filtering step to avoid comparing the whole
|
|
needle repeatedly.
|
|
|
|
By limiting the needle length to 256, the shift table only requires 8 bits
|
|
per entry, lowering preprocessing overhead and minimizing cache effects.
|
|
This limit also implies worst-case performance is linear.
|
|
|
|
Small needles up to size 3 use a dedicated linear search. Very long needles
|
|
use the Two-Way algorithm.
|
|
|
|
The performance gain using the improved bench-strstr on Cortex-A72 is 5.8
|
|
times basic_strstr and 3.7 times twoway_strstr.
|
|
|
|
Tested against GLIBC testsuite, randomized tests and the GNULIB strstr test
|
|
(https://git.savannah.gnu.org/cgit/gnulib.git/tree/tests/test-strstr.c).
|
|
|
|
Reviewed-by: Szabolcs Nagy <szabolcs.nagy@arm.com>
|
|
|
|
* string/str-two-way.h (two_way_short_needle): Add inline to avoid
|
|
warning.
|
|
(two_way_long_needle): Block inlining.
|
|
* string/strstr.c (strstr2): Add new function.
|
|
(strstr3): Likewise.
|
|
(STRSTR): Completely rewrite strstr to improve performance.
|
|
|
|
diff --git a/string/str-two-way.h b/string/str-two-way.h
|
|
index 523d946c59412e1f..358959bef0fd6f74 100644
|
|
--- a/string/str-two-way.h
|
|
+++ b/string/str-two-way.h
|
|
@@ -221,7 +221,7 @@ critical_factorization (const unsigned char *needle, size_t needle_len,
|
|
most 2 * HAYSTACK_LEN - NEEDLE_LEN comparisons occur in searching.
|
|
If AVAILABLE modifies HAYSTACK_LEN (as in strstr), then at most 3 *
|
|
HAYSTACK_LEN - NEEDLE_LEN comparisons occur in searching. */
|
|
-static RETURN_TYPE
|
|
+static inline RETURN_TYPE
|
|
two_way_short_needle (const unsigned char *haystack, size_t haystack_len,
|
|
const unsigned char *needle, size_t needle_len)
|
|
{
|
|
@@ -382,8 +382,11 @@ two_way_short_needle (const unsigned char *haystack, size_t haystack_len,
|
|
and sublinear performance O(HAYSTACK_LEN / NEEDLE_LEN) is possible.
|
|
If AVAILABLE modifies HAYSTACK_LEN (as in strstr), then at most 3 *
|
|
HAYSTACK_LEN - NEEDLE_LEN comparisons occur in searching, and
|
|
- sublinear performance is not possible. */
|
|
-static RETURN_TYPE
|
|
+ sublinear performance is not possible.
|
|
+
|
|
+ Since this function is large and complex, block inlining to avoid
|
|
+ slowing down the common case of small needles. */
|
|
+__attribute__((noinline)) static RETURN_TYPE
|
|
two_way_long_needle (const unsigned char *haystack, size_t haystack_len,
|
|
const unsigned char *needle, size_t needle_len)
|
|
{
|
|
diff --git a/string/strstr.c b/string/strstr.c
|
|
index f74d7189ed1319f6..7ffb18ab4285b060 100644
|
|
--- a/string/strstr.c
|
|
+++ b/string/strstr.c
|
|
@@ -16,29 +16,17 @@
|
|
License along with the GNU C Library; if not, see
|
|
<http://www.gnu.org/licenses/>. */
|
|
|
|
-/* This particular implementation was written by Eric Blake, 2008. */
|
|
-
|
|
#ifndef _LIBC
|
|
# include <config.h>
|
|
#endif
|
|
|
|
-/* Specification of strstr. */
|
|
#include <string.h>
|
|
|
|
-#include <stdbool.h>
|
|
-
|
|
-#ifndef _LIBC
|
|
-# define __builtin_expect(expr, val) (expr)
|
|
-#endif
|
|
-
|
|
#define RETURN_TYPE char *
|
|
#define AVAILABLE(h, h_l, j, n_l) \
|
|
(((j) + (n_l) <= (h_l)) \
|
|
|| ((h_l) += __strnlen ((void*)((h) + (h_l)), (n_l) + 512), \
|
|
(j) + (n_l) <= (h_l)))
|
|
-#define CHECK_EOL (1)
|
|
-#define RET0_IF_0(a) if (!a) goto ret0
|
|
-#define FASTSEARCH(S,C,N) (void*) strchr ((void*)(S), (C))
|
|
#include "str-two-way.h"
|
|
|
|
#undef strstr
|
|
@@ -47,47 +35,128 @@
|
|
#define STRSTR strstr
|
|
#endif
|
|
|
|
-/* Return the first occurrence of NEEDLE in HAYSTACK. Return HAYSTACK
|
|
- if NEEDLE is empty, otherwise NULL if NEEDLE is not found in
|
|
- HAYSTACK. */
|
|
-char *
|
|
-STRSTR (const char *haystack, const char *needle)
|
|
+static inline char *
|
|
+strstr2 (const unsigned char *hs, const unsigned char *ne)
|
|
{
|
|
- size_t needle_len; /* Length of NEEDLE. */
|
|
- size_t haystack_len; /* Known minimum length of HAYSTACK. */
|
|
-
|
|
- /* Handle empty NEEDLE special case. */
|
|
- if (needle[0] == '\0')
|
|
- return (char *) haystack;
|
|
+ uint32_t h1 = (ne[0] << 16) | ne[1];
|
|
+ uint32_t h2 = 0;
|
|
+ for (int c = hs[0]; h1 != h2 && c != 0; c = *++hs)
|
|
+ h2 = (h2 << 16) | c;
|
|
+ return h1 == h2 ? (char *)hs - 2 : NULL;
|
|
+}
|
|
|
|
- /* Skip until we find the first matching char from NEEDLE. */
|
|
- haystack = strchr (haystack, needle[0]);
|
|
- if (haystack == NULL || needle[1] == '\0')
|
|
- return (char *) haystack;
|
|
+static inline char *
|
|
+strstr3 (const unsigned char *hs, const unsigned char *ne)
|
|
+{
|
|
+ uint32_t h1 = ((uint32_t)ne[0] << 24) | (ne[1] << 16) | (ne[2] << 8);
|
|
+ uint32_t h2 = 0;
|
|
+ for (int c = hs[0]; h1 != h2 && c != 0; c = *++hs)
|
|
+ h2 = (h2 | c) << 8;
|
|
+ return h1 == h2 ? (char *)hs - 3 : NULL;
|
|
+}
|
|
|
|
- /* Ensure HAYSTACK length is at least as long as NEEDLE length.
|
|
- Since a match may occur early on in a huge HAYSTACK, use strnlen
|
|
+/* Hash character pairs so a small shift table can be used. All bits of
|
|
+ p[0] are included, but not all bits from p[-1]. So if two equal hashes
|
|
+ match on p[-1], p[0] matches too. Hash collisions are harmless and result
|
|
+ in smaller shifts. */
|
|
+#define hash2(p) (((size_t)(p)[0] - ((size_t)(p)[-1] << 3)) % sizeof (shift))
|
|
+
|
|
+/* Fast strstr algorithm with guaranteed linear-time performance.
|
|
+ Small needles up to size 3 use a dedicated linear search. Longer needles
|
|
+ up to size 256 use a novel modified Horspool algorithm. It hashes pairs
|
|
+ of characters to quickly skip past mismatches. The main search loop only
|
|
+ exits if the last 2 characters match, avoiding unnecessary calls to memcmp
|
|
+ and allowing for a larger skip if there is no match. A self-adapting
|
|
+ filtering check is used to quickly detect mismatches in long needles.
|
|
+ By limiting the needle length to 256, the shift table can be reduced to 8
|
|
+ bits per entry, lowering preprocessing overhead and minimizing cache effects.
|
|
+ The limit also implies worst-case performance is linear.
|
|
+ Needles larger than 256 characters use the linear-time Two-Way algorithm. */
|
|
+char *
|
|
+STRSTR (const char *haystack, const char *needle)
|
|
+{
|
|
+ const unsigned char *hs = (const unsigned char *) haystack;
|
|
+ const unsigned char *ne = (const unsigned char *) needle;
|
|
+
|
|
+ /* Handle short needle special cases first. */
|
|
+ if (ne[0] == '\0')
|
|
+ return (char *)hs;
|
|
+ hs = (const unsigned char *)strchr ((const char*)hs, ne[0]);
|
|
+ if (hs == NULL || ne[1] == '\0')
|
|
+ return (char*)hs;
|
|
+ if (ne[2] == '\0')
|
|
+ return strstr2 (hs, ne);
|
|
+ if (ne[3] == '\0')
|
|
+ return strstr3 (hs, ne);
|
|
+
|
|
+ /* Ensure haystack length is at least as long as needle length.
|
|
+ Since a match may occur early on in a huge haystack, use strnlen
|
|
and read ahead a few cachelines for improved performance. */
|
|
- needle_len = strlen (needle);
|
|
- haystack_len = __strnlen (haystack, needle_len + 256);
|
|
- if (haystack_len < needle_len)
|
|
+ size_t ne_len = strlen ((const char*)ne);
|
|
+ size_t hs_len = __strnlen ((const char*)hs, ne_len | 512);
|
|
+ if (hs_len < ne_len)
|
|
return NULL;
|
|
|
|
- /* Check whether we have a match. This improves performance since we avoid
|
|
- the initialization overhead of the two-way algorithm. */
|
|
- if (memcmp (haystack, needle, needle_len) == 0)
|
|
- return (char *) haystack;
|
|
-
|
|
- /* Perform the search. Abstract memory is considered to be an array
|
|
- of 'unsigned char' values, not an array of 'char' values. See
|
|
- ISO C 99 section 6.2.6.1. */
|
|
- if (needle_len < LONG_NEEDLE_THRESHOLD)
|
|
- return two_way_short_needle ((const unsigned char *) haystack,
|
|
- haystack_len,
|
|
- (const unsigned char *) needle, needle_len);
|
|
- return two_way_long_needle ((const unsigned char *) haystack, haystack_len,
|
|
- (const unsigned char *) needle, needle_len);
|
|
+ /* Check whether we have a match. This improves performance since we
|
|
+ avoid initialization overheads. */
|
|
+ if (memcmp (hs, ne, ne_len) == 0)
|
|
+ return (char *) hs;
|
|
+
|
|
+ /* Use Two-Way algorithm for very long needles. */
|
|
+ if (__glibc_unlikely (ne_len > 256))
|
|
+ return two_way_long_needle (hs, hs_len, ne, ne_len);
|
|
+
|
|
+ const unsigned char *end = hs + hs_len - ne_len;
|
|
+ uint8_t shift[256];
|
|
+ size_t tmp, shift1;
|
|
+ size_t m1 = ne_len - 1;
|
|
+ size_t offset = 0;
|
|
+
|
|
+ /* Initialize bad character shift hash table. */
|
|
+ memset (shift, 0, sizeof (shift));
|
|
+ for (int i = 1; i < m1; i++)
|
|
+ shift[hash2 (ne + i)] = i;
|
|
+ /* Shift1 is the amount we can skip after matching the hash of the
|
|
+ needle end but not the full needle. */
|
|
+ shift1 = m1 - shift[hash2 (ne + m1)];
|
|
+ shift[hash2 (ne + m1)] = m1;
|
|
+
|
|
+ while (1)
|
|
+ {
|
|
+ if (__glibc_unlikely (hs > end))
|
|
+ {
|
|
+ end += __strnlen ((const char*)end + m1 + 1, 2048);
|
|
+ if (hs > end)
|
|
+ return NULL;
|
|
+ }
|
|
+
|
|
+ /* Skip past character pairs not in the needle. */
|
|
+ do
|
|
+ {
|
|
+ hs += m1;
|
|
+ tmp = shift[hash2 (hs)];
|
|
+ }
|
|
+ while (tmp == 0 && hs <= end);
|
|
+
|
|
+ /* If the match is not at the end of the needle, shift to the end
|
|
+ and continue until we match the hash of the needle end. */
|
|
+ hs -= tmp;
|
|
+ if (tmp < m1)
|
|
+ continue;
|
|
+
|
|
+ /* Hash of the last 2 characters matches. If the needle is long,
|
|
+ try to quickly filter out mismatches. */
|
|
+ if (m1 < 15 || memcmp (hs + offset, ne + offset, 8) == 0)
|
|
+ {
|
|
+ if (memcmp (hs, ne, m1) == 0)
|
|
+ return (void *) hs;
|
|
+
|
|
+ /* Adjust filter offset when it doesn't find the mismatch. */
|
|
+ offset = (offset >= 8 ? offset : m1) - 8;
|
|
+ }
|
|
+
|
|
+ /* Skip based on matching the hash of the needle end. */
|
|
+ hs += shift1;
|
|
+ }
|
|
}
|
|
libc_hidden_builtin_def (strstr)
|
|
-
|
|
-#undef LONG_NEEDLE_THRESHOLD
|