Skip to content

Commit b68e2eb

Browse files
authored
[libc++] Vectorize mismatch (#73255)
``` --------------------------------------------------- Benchmark old new --------------------------------------------------- bm_mismatch<char>/1 0.835 ns 2.37 ns bm_mismatch<char>/2 1.44 ns 2.60 ns bm_mismatch<char>/3 2.06 ns 2.83 ns bm_mismatch<char>/4 2.60 ns 3.29 ns bm_mismatch<char>/5 3.15 ns 3.77 ns bm_mismatch<char>/6 3.82 ns 4.17 ns bm_mismatch<char>/7 4.29 ns 4.52 ns bm_mismatch<char>/8 4.78 ns 4.86 ns bm_mismatch<char>/16 9.06 ns 7.54 ns bm_mismatch<char>/64 31.7 ns 19.1 ns bm_mismatch<char>/512 249 ns 8.16 ns bm_mismatch<char>/4096 1956 ns 44.2 ns bm_mismatch<char>/32768 15498 ns 501 ns bm_mismatch<char>/262144 123965 ns 4479 ns bm_mismatch<char>/1048576 495668 ns 21306 ns bm_mismatch<short>/1 0.710 ns 2.12 ns bm_mismatch<short>/2 1.03 ns 2.66 ns bm_mismatch<short>/3 1.29 ns 3.56 ns bm_mismatch<short>/4 1.68 ns 4.29 ns bm_mismatch<short>/5 1.96 ns 5.18 ns bm_mismatch<short>/6 2.59 ns 5.91 ns bm_mismatch<short>/7 2.86 ns 6.63 ns bm_mismatch<short>/8 3.19 ns 7.33 ns bm_mismatch<short>/16 5.48 ns 13.0 ns bm_mismatch<short>/64 16.6 ns 4.06 ns bm_mismatch<short>/512 130 ns 13.8 ns bm_mismatch<short>/4096 985 ns 93.8 ns bm_mismatch<short>/32768 7846 ns 1002 ns bm_mismatch<short>/262144 63217 ns 10637 ns bm_mismatch<short>/1048576 251782 ns 42471 ns bm_mismatch<int>/1 0.716 ns 1.91 ns bm_mismatch<int>/2 1.21 ns 2.49 ns bm_mismatch<int>/3 1.38 ns 3.46 ns bm_mismatch<int>/4 1.71 ns 4.04 ns bm_mismatch<int>/5 2.00 ns 4.98 ns bm_mismatch<int>/6 2.43 ns 5.67 ns bm_mismatch<int>/7 3.05 ns 6.38 ns bm_mismatch<int>/8 3.22 ns 7.09 ns bm_mismatch<int>/16 5.18 ns 12.8 ns bm_mismatch<int>/64 16.6 ns 5.28 ns bm_mismatch<int>/512 129 ns 25.2 ns bm_mismatch<int>/4096 1009 ns 201 ns bm_mismatch<int>/32768 7776 ns 2144 ns bm_mismatch<int>/262144 62371 ns 20551 ns bm_mismatch<int>/1048576 254750 ns 90097 ns ```
1 parent 57146da commit b68e2eb

File tree

12 files changed

+413
-189
lines changed

12 files changed

+413
-189
lines changed

libcxx/benchmarks/CMakeLists.txt

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -183,6 +183,7 @@ set(BENCHMARK_TESTS
183183
algorithms/make_heap_then_sort_heap.bench.cpp
184184
algorithms/min.bench.cpp
185185
algorithms/min_max_element.bench.cpp
186+
algorithms/mismatch.bench.cpp
186187
algorithms/pop_heap.bench.cpp
187188
algorithms/pstl.stable_sort.bench.cpp
188189
algorithms/push_heap.bench.cpp
Lines changed: 31 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,31 @@
1+
//===----------------------------------------------------------------------===//
2+
//
3+
// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4+
// See https://llvm.org/LICENSE.txt for license information.
5+
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6+
//
7+
//===----------------------------------------------------------------------===//
8+
9+
#include <algorithm>
10+
#include <benchmark/benchmark.h>
11+
#include <random>
12+
13+
// TODO: Look into benchmarking aligned and unaligned memory explicitly
14+
// (currently things happen to be aligned because they are malloced that way)
15+
template <class T>
16+
static void bm_mismatch(benchmark::State& state) {
17+
std::vector<T> vec1(state.range(), '1');
18+
std::vector<T> vec2(state.range(), '1');
19+
std::mt19937_64 rng(std::random_device{}());
20+
21+
vec1.back() = '2';
22+
for (auto _ : state) {
23+
benchmark::DoNotOptimize(vec1);
24+
benchmark::DoNotOptimize(std::mismatch(vec1.begin(), vec1.end(), vec2.begin()));
25+
}
26+
}
27+
BENCHMARK(bm_mismatch<char>)->DenseRange(1, 8)->Range(16, 1 << 20);
28+
BENCHMARK(bm_mismatch<short>)->DenseRange(1, 8)->Range(16, 1 << 20);
29+
BENCHMARK(bm_mismatch<int>)->DenseRange(1, 8)->Range(16, 1 << 20);
30+
31+
BENCHMARK_MAIN();

libcxx/docs/ReleaseNotes/19.rst

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -51,6 +51,8 @@ Improvements and New Features
5151
- The performance of growing ``std::vector`` has been improved for trivially relocatable types.
5252
- The performance of ``ranges::fill`` and ``ranges::fill_n`` has been improved for ``vector<bool>::iterator``\s,
5353
resulting in a performance increase of up to 1400x.
54+
- The ``std::mismatch`` algorithm has been optimized for integral types, which can lead up to 40x performance
55+
improvements.
5456

5557
Deprecations and Removals
5658
-------------------------

libcxx/include/CMakeLists.txt

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -217,6 +217,7 @@ set(files
217217
__algorithm/shift_right.h
218218
__algorithm/shuffle.h
219219
__algorithm/sift_down.h
220+
__algorithm/simd_utils.h
220221
__algorithm/sort.h
221222
__algorithm/sort_heap.h
222223
__algorithm/stable_partition.h

libcxx/include/__algorithm/mismatch.h

Lines changed: 77 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -11,23 +11,93 @@
1111
#define _LIBCPP___ALGORITHM_MISMATCH_H
1212

1313
#include <__algorithm/comp.h>
14+
#include <__algorithm/simd_utils.h>
15+
#include <__algorithm/unwrap_iter.h>
1416
#include <__config>
15-
#include <__iterator/iterator_traits.h>
17+
#include <__functional/identity.h>
18+
#include <__type_traits/invoke.h>
19+
#include <__type_traits/is_constant_evaluated.h>
20+
#include <__type_traits/is_equality_comparable.h>
21+
#include <__type_traits/operation_traits.h>
22+
#include <__utility/move.h>
1623
#include <__utility/pair.h>
24+
#include <__utility/unreachable.h>
1725

1826
#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
1927
# pragma GCC system_header
2028
#endif
2129

30+
_LIBCPP_PUSH_MACROS
31+
#include <__undef_macros>
32+
2233
_LIBCPP_BEGIN_NAMESPACE_STD
2334

35+
template <class _Iter1, class _Sent1, class _Iter2, class _Pred, class _Proj1, class _Proj2>
36+
_LIBCPP_NODISCARD _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 pair<_Iter1, _Iter2>
37+
__mismatch_loop(_Iter1 __first1, _Sent1 __last1, _Iter2 __first2, _Pred& __pred, _Proj1& __proj1, _Proj2& __proj2) {
38+
while (__first1 != __last1) {
39+
if (!std::__invoke(__pred, std::__invoke(__proj1, *__first1), std::__invoke(__proj2, *__first2)))
40+
break;
41+
++__first1;
42+
++__first2;
43+
}
44+
return std::make_pair(std::move(__first1), std::move(__first2));
45+
}
46+
47+
template <class _Iter1, class _Sent1, class _Iter2, class _Pred, class _Proj1, class _Proj2>
48+
_LIBCPP_NODISCARD _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 pair<_Iter1, _Iter2>
49+
__mismatch(_Iter1 __first1, _Sent1 __last1, _Iter2 __first2, _Pred& __pred, _Proj1& __proj1, _Proj2& __proj2) {
50+
return std::__mismatch_loop(__first1, __last1, __first2, __pred, __proj1, __proj2);
51+
}
52+
53+
#if _LIBCPP_VECTORIZE_ALGORITHMS
54+
55+
template <class _Tp,
56+
class _Pred,
57+
class _Proj1,
58+
class _Proj2,
59+
__enable_if_t<is_integral<_Tp>::value && __desugars_to<__equal_tag, _Pred, _Tp, _Tp>::value &&
60+
__is_identity<_Proj1>::value && __is_identity<_Proj2>::value,
61+
int> = 0>
62+
_LIBCPP_NODISCARD _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 pair<_Tp*, _Tp*>
63+
__mismatch(_Tp* __first1, _Tp* __last1, _Tp* __first2, _Pred& __pred, _Proj1& __proj1, _Proj2& __proj2) {
64+
constexpr size_t __unroll_count = 4;
65+
constexpr size_t __vec_size = __native_vector_size<_Tp>;
66+
using __vec = __simd_vector<_Tp, __vec_size>;
67+
if (!__libcpp_is_constant_evaluated()) {
68+
while (static_cast<size_t>(__last1 - __first1) >= __unroll_count * __vec_size) [[__unlikely__]] {
69+
__vec __lhs[__unroll_count];
70+
__vec __rhs[__unroll_count];
71+
72+
for (size_t __i = 0; __i != __unroll_count; ++__i) {
73+
__lhs[__i] = std::__load_vector<__vec>(__first1 + __i * __vec_size);
74+
__rhs[__i] = std::__load_vector<__vec>(__first2 + __i * __vec_size);
75+
}
76+
77+
for (size_t __i = 0; __i != __unroll_count; ++__i) {
78+
if (auto __cmp_res = __lhs[__i] == __rhs[__i]; !std::__all_of(__cmp_res)) {
79+
auto __offset = __i * __vec_size + std::__find_first_not_set(__cmp_res);
80+
return {__first1 + __offset, __first2 + __offset};
81+
}
82+
}
83+
84+
__first1 += __unroll_count * __vec_size;
85+
__first2 += __unroll_count * __vec_size;
86+
}
87+
}
88+
// TODO: Consider vectorizing the tail
89+
return std::__mismatch_loop(__first1, __last1, __first2, __pred, __proj1, __proj2);
90+
}
91+
92+
#endif // _LIBCPP_VECTORIZE_ALGORITHMS
93+
2494
template <class _InputIterator1, class _InputIterator2, class _BinaryPredicate>
2595
_LIBCPP_NODISCARD_EXT inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 pair<_InputIterator1, _InputIterator2>
2696
mismatch(_InputIterator1 __first1, _InputIterator1 __last1, _InputIterator2 __first2, _BinaryPredicate __pred) {
27-
for (; __first1 != __last1; ++__first1, (void)++__first2)
28-
if (!__pred(*__first1, *__first2))
29-
break;
30-
return pair<_InputIterator1, _InputIterator2>(__first1, __first2);
97+
__identity __proj;
98+
auto __res = std::__mismatch(
99+
std::__unwrap_iter(__first1), std::__unwrap_iter(__last1), std::__unwrap_iter(__first2), __pred, __proj, __proj);
100+
return std::make_pair(std::__rewrap_iter(__first1, __res.first), std::__rewrap_iter(__first2, __res.second));
31101
}
32102

33103
template <class _InputIterator1, class _InputIterator2>
@@ -59,4 +129,6 @@ mismatch(_InputIterator1 __first1, _InputIterator1 __last1, _InputIterator2 __fi
59129

60130
_LIBCPP_END_NAMESPACE_STD
61131

132+
_LIBCPP_POP_MACROS
133+
62134
#endif // _LIBCPP___ALGORITHM_MISMATCH_H
Lines changed: 123 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,123 @@
1+
//===----------------------------------------------------------------------===//
2+
//
3+
// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4+
// See https://llvm.org/LICENSE.txt for license information.
5+
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6+
//
7+
//===----------------------------------------------------------------------===//
8+
9+
#ifndef _LIBCPP___ALGORITHM_SIMD_UTILS_H
10+
#define _LIBCPP___ALGORITHM_SIMD_UTILS_H
11+
12+
#include <__bit/bit_cast.h>
13+
#include <__bit/countr.h>
14+
#include <__config>
15+
#include <__type_traits/is_arithmetic.h>
16+
#include <__type_traits/is_same.h>
17+
#include <__utility/integer_sequence.h>
18+
#include <cstddef>
19+
#include <cstdint>
20+
21+
#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
22+
# pragma GCC system_header
23+
#endif
24+
25+
// TODO: Find out how altivec changes things and allow vectorizations there too.
26+
#if _LIBCPP_STD_VER >= 14 && defined(_LIBCPP_CLANG_VER) && _LIBCPP_CLANG_VER >= 1700 && !defined(__ALTIVEC__)
27+
# define _LIBCPP_HAS_ALGORITHM_VECTOR_UTILS 1
28+
#else
29+
# define _LIBCPP_HAS_ALGORITHM_VECTOR_UTILS 0
30+
#endif
31+
32+
#if _LIBCPP_HAS_ALGORITHM_VECTOR_UTILS && !defined(__OPTIMIZE_SIZE__)
33+
# define _LIBCPP_VECTORIZE_ALGORITHMS 1
34+
#else
35+
# define _LIBCPP_VECTORIZE_ALGORITHMS 0
36+
#endif
37+
38+
#if _LIBCPP_HAS_ALGORITHM_VECTOR_UTILS
39+
40+
_LIBCPP_BEGIN_NAMESPACE_STD
41+
42+
// This isn't specialized for 64 byte vectors on purpose. They have the potential to significantly reduce performance
43+
// in mixed simd/non-simd workloads and don't provide any performance improvement for currently vectorized algorithms
44+
// as far as benchmarks are concerned.
45+
# if defined(__AVX__)
46+
template <class _Tp>
47+
inline constexpr size_t __native_vector_size = 32 / sizeof(_Tp);
48+
# elif defined(__SSE__) || defined(__ARM_NEON__)
49+
template <class _Tp>
50+
inline constexpr size_t __native_vector_size = 16 / sizeof(_Tp);
51+
# elif defined(__MMX__)
52+
template <class _Tp>
53+
inline constexpr size_t __native_vector_size = 8 / sizeof(_Tp);
54+
# else
55+
template <class _Tp>
56+
inline constexpr size_t __native_vector_size = 1;
57+
# endif
58+
59+
template <class _ArithmeticT, size_t _Np>
60+
using __simd_vector __attribute__((__ext_vector_type__(_Np))) = _ArithmeticT;
61+
62+
template <class _VecT>
63+
inline constexpr size_t __simd_vector_size_v = []<bool _False = false>() -> size_t {
64+
static_assert(_False, "Not a vector!");
65+
}();
66+
67+
template <class _Tp, size_t _Np>
68+
inline constexpr size_t __simd_vector_size_v<__simd_vector<_Tp, _Np>> = _Np;
69+
70+
template <class _Tp, size_t _Np>
71+
_LIBCPP_HIDE_FROM_ABI _Tp __simd_vector_underlying_type_impl(__simd_vector<_Tp, _Np>) {
72+
return _Tp{};
73+
}
74+
75+
template <class _VecT>
76+
using __simd_vector_underlying_type_t = decltype(std::__simd_vector_underlying_type_impl(_VecT{}));
77+
78+
// This isn't inlined without always_inline when loading chars.
79+
template <class _VecT, class _Tp>
80+
_LIBCPP_NODISCARD _LIBCPP_ALWAYS_INLINE _LIBCPP_HIDE_FROM_ABI _VecT __load_vector(const _Tp* __ptr) noexcept {
81+
return [=]<size_t... _Indices>(index_sequence<_Indices...>) _LIBCPP_ALWAYS_INLINE noexcept {
82+
return _VecT{__ptr[_Indices]...};
83+
}(make_index_sequence<__simd_vector_size_v<_VecT>>{});
84+
}
85+
86+
template <class _Tp, size_t _Np>
87+
_LIBCPP_NODISCARD _LIBCPP_HIDE_FROM_ABI bool __all_of(__simd_vector<_Tp, _Np> __vec) noexcept {
88+
return __builtin_reduce_and(__builtin_convertvector(__vec, __simd_vector<bool, _Np>));
89+
}
90+
91+
template <class _Tp, size_t _Np>
92+
_LIBCPP_NODISCARD _LIBCPP_HIDE_FROM_ABI size_t __find_first_set(__simd_vector<_Tp, _Np> __vec) noexcept {
93+
using __mask_vec = __simd_vector<bool, _Np>;
94+
95+
// This has MSan disabled du to https://github.com/llvm/llvm-project/issues/85876
96+
auto __impl = [&]<class _MaskT>(_MaskT) _LIBCPP_NO_SANITIZE("memory") noexcept {
97+
return std::__countr_zero(__builtin_bit_cast(_MaskT, __builtin_convertvector(__vec, __mask_vec)));
98+
};
99+
100+
if constexpr (sizeof(__mask_vec) == sizeof(uint8_t)) {
101+
return __impl(uint8_t{});
102+
} else if constexpr (sizeof(__mask_vec) == sizeof(uint16_t)) {
103+
return __impl(uint16_t{});
104+
} else if constexpr (sizeof(__mask_vec) == sizeof(uint32_t)) {
105+
return __impl(uint32_t{});
106+
} else if constexpr (sizeof(__mask_vec) == sizeof(uint64_t)) {
107+
return __impl(uint64_t{});
108+
} else {
109+
static_assert(sizeof(__mask_vec) == 0, "unexpected required size for mask integer type");
110+
return 0;
111+
}
112+
}
113+
114+
template <class _Tp, size_t _Np>
115+
_LIBCPP_NODISCARD _LIBCPP_HIDE_FROM_ABI size_t __find_first_not_set(__simd_vector<_Tp, _Np> __vec) noexcept {
116+
return std::__find_first_set(~__vec);
117+
}
118+
119+
_LIBCPP_END_NAMESPACE_STD
120+
121+
#endif // _LIBCPP_HAS_ALGORITHM_VECTOR_UTILS
122+
123+
#endif // _LIBCPP___ALGORITHM_SIMD_UTILS_H

libcxx/include/__bit/bit_cast.h

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -19,6 +19,15 @@
1919

2020
_LIBCPP_BEGIN_NAMESPACE_STD
2121

22+
#ifndef _LIBCPP_CXX03_LANG
23+
24+
template <class _ToType, class _FromType>
25+
_LIBCPP_NODISCARD _LIBCPP_HIDE_FROM_ABI constexpr _ToType __bit_cast(const _FromType& __from) noexcept {
26+
return __builtin_bit_cast(_ToType, __from);
27+
}
28+
29+
#endif // _LIBCPP_CXX03_LANG
30+
2231
#if _LIBCPP_STD_VER >= 20
2332

2433
template <class _ToType, class _FromType>

libcxx/include/__bit/countr.h

Lines changed: 9 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -35,10 +35,8 @@ _LIBCPP_NODISCARD inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR int __libcpp_ct
3535
return __builtin_ctzll(__x);
3636
}
3737

38-
#if _LIBCPP_STD_VER >= 20
39-
40-
template <__libcpp_unsigned_integer _Tp>
41-
_LIBCPP_NODISCARD_EXT _LIBCPP_HIDE_FROM_ABI constexpr int countr_zero(_Tp __t) noexcept {
38+
template <class _Tp>
39+
_LIBCPP_NODISCARD _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX14 int __countr_zero(_Tp __t) _NOEXCEPT {
4240
if (__t == 0)
4341
return numeric_limits<_Tp>::digits;
4442

@@ -59,6 +57,13 @@ _LIBCPP_NODISCARD_EXT _LIBCPP_HIDE_FROM_ABI constexpr int countr_zero(_Tp __t) n
5957
}
6058
}
6159

60+
#if _LIBCPP_STD_VER >= 20
61+
62+
template <__libcpp_unsigned_integer _Tp>
63+
_LIBCPP_NODISCARD_EXT _LIBCPP_HIDE_FROM_ABI constexpr int countr_zero(_Tp __t) noexcept {
64+
return std::__countr_zero(__t);
65+
}
66+
6267
template <__libcpp_unsigned_integer _Tp>
6368
_LIBCPP_NODISCARD_EXT _LIBCPP_HIDE_FROM_ABI constexpr int countr_one(_Tp __t) noexcept {
6469
return __t != numeric_limits<_Tp>::max() ? std::countr_zero(static_cast<_Tp>(~__t)) : numeric_limits<_Tp>::digits;

libcxx/include/libcxx.imp

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -217,6 +217,7 @@
217217
{ include: [ "<__algorithm/shift_right.h>", "private", "<algorithm>", "public" ] },
218218
{ include: [ "<__algorithm/shuffle.h>", "private", "<algorithm>", "public" ] },
219219
{ include: [ "<__algorithm/sift_down.h>", "private", "<algorithm>", "public" ] },
220+
{ include: [ "<__algorithm/simd_utils.h>", "private", "<algorithm>", "public" ] },
220221
{ include: [ "<__algorithm/sort.h>", "private", "<algorithm>", "public" ] },
221222
{ include: [ "<__algorithm/sort_heap.h>", "private", "<algorithm>", "public" ] },
222223
{ include: [ "<__algorithm/stable_partition.h>", "private", "<algorithm>", "public" ] },

libcxx/include/module.modulemap

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -697,7 +697,10 @@ module std_private_algorithm_minmax [system
697697
export *
698698
}
699699
module std_private_algorithm_minmax_element [system] { header "__algorithm/minmax_element.h" }
700-
module std_private_algorithm_mismatch [system] { header "__algorithm/mismatch.h" }
700+
module std_private_algorithm_mismatch [system] {
701+
header "__algorithm/mismatch.h"
702+
export std_private_algorithm_simd_utils
703+
}
701704
module std_private_algorithm_move [system] { header "__algorithm/move.h" }
702705
module std_private_algorithm_move_backward [system] { header "__algorithm/move_backward.h" }
703706
module std_private_algorithm_next_permutation [system] { header "__algorithm/next_permutation.h" }
@@ -1048,6 +1051,7 @@ module std_private_algorithm_sort [system
10481051
header "__algorithm/sort.h"
10491052
export std_private_debug_utils_strict_weak_ordering_check
10501053
}
1054+
module std_private_algorithm_simd_utils [system] { header "__algorithm/simd_utils.h" }
10511055
module std_private_algorithm_sort_heap [system] { header "__algorithm/sort_heap.h" }
10521056
module std_private_algorithm_stable_partition [system] { header "__algorithm/stable_partition.h" }
10531057
module std_private_algorithm_stable_sort [system] { header "__algorithm/stable_sort.h" }

0 commit comments

Comments
 (0)