commit d0987c7014d33e96a7a0d170fea8bcc97163cead Author: Florian Weimer Date: Thu Nov 23 08:34:30 2023 +0100 stdlib: Add another workaround to the insertion sort phase of qsort If the comparison function returns negative values incorrectly, it was possible that we decrement tmp_ptr past the start of the array. Improves commit e4d8117b82065dc72e8df80097360e7c05a349b9 ("stdlib: Avoid another self-comparison in qsort"). diff --git a/stdlib/qsort.c b/stdlib/qsort.c index be01fb5598de2257..6f28abbc7f9719fb 100644 --- a/stdlib/qsort.c +++ b/stdlib/qsort.c @@ -238,8 +238,17 @@ insertion_sort_qsort_partitions (void *const pbase, size_t total_elems, while ((run_ptr += size) <= end_ptr) { tmp_ptr = run_ptr - size; - while (run_ptr != tmp_ptr && cmp (run_ptr, tmp_ptr, arg) < 0) - tmp_ptr -= size; + /* The initial pointer comparison avoids a call to cmp if the + pointer arguments are identical (the call returns zero with a + correctly implemented comparison function). The final + pointer comparison cannot be reached because the element at + base_ptr is the smallest element, but it prevents the loop + from running beyond the start of the array with a broken + comparison function. */ + while (run_ptr != tmp_ptr + && cmp (run_ptr, tmp_ptr, arg) < 0 + && run_ptr != base_ptr) + tmp_ptr -= size; tmp_ptr += size; if (tmp_ptr != run_ptr)