Add fast mb_strcut implementation for UTF-8

The old implementation decodes the entire string to pick out the
part which should be returned by mb_strcut. This creates significant
performance overhead. The new specialized implementation of mb_strcut
for UTF-8 usually only examines a few bytes around the starting and
ending cut points, meaning it generally runs in constant time.

For UTF-8 strings just a few bytes long, the new implementation is
around 10% faster (according to microbenchmarks which I ran locally).
For strings around 10,000 bytes in length, it is 50-300x faster.
(Yes, that is 300x and not 300%.)

At the same time, I also added many more unit tests for mb_strcut.
This will help to avoid unintended behavior changes as the function
undergoes further performance work.

The new implementation behaves identically to the old one on VALID
UTF-8 strings; a fuzzer was used to help ensure this is the case.
On invalid UTF-8 strings, there is a difference: the old implementation
would convert invalid UTF-8 byte sequences to error markers ('?'),
but the new implementation just cuts a subsequence of bytes out of the
source string without performing any conversion on it.

Author: alexdowad

UPGRADING

@@ -55,6 +55,13 @@ PHP 8.4 UPGRADE NOTES
 5. Changed Functions
 ========================================
 
+- MBString:
+  . After performance optimization, mb_strcut returns slightly different
+    output for invalid UTF-8 strings. Rather than converting invalid UTF-8
+    byte sequences to an error marker (usually the string '?'), any
+    invalid sequences embedded in the requested range of bytes are cut
+    out and returned as is.
+
 - Standard:
   . The internal implementation for rounding to integers has been rewritten
     to be easier to verify for correctness and to be easier to maintain.

ext/mbstring/libmbfl/filters/mbfilter_7bit.c

@@ -65,6 +65,7 @@ const mbfl_encoding mbfl_encoding_7bit = {
 	&vtbl_wchar_7bit,
 	mb_7bit_to_wchar,
 	mb_wchar_to_7bit,
+	NULL,
 	NULL
 };
 

ext/mbstring/libmbfl/filters/mbfilter_base64.c

@@ -45,6 +45,7 @@ const mbfl_encoding mbfl_encoding_base64 = {
 	NULL,
 	mb_base64_to_wchar,
 	mb_wchar_to_base64,
+	NULL,
 	NULL
 };
 

ext/mbstring/libmbfl/filters/mbfilter_cjk.c

@@ -4392,7 +4392,8 @@ const mbfl_encoding mbfl_encoding_jis = {
 	&vtbl_wchar_jis,
 	mb_iso2022jp_to_wchar,
 	mb_wchar_to_jis,
-	mb_check_jis
+	mb_check_jis,
+	NULL
 };
 
 static const struct mbfl_convert_vtbl vtbl_2022jp_wchar = {
@@ -4426,7 +4427,8 @@ const mbfl_encoding mbfl_encoding_2022jp = {
 	&vtbl_wchar_2022jp,
 	mb_iso2022jp_to_wchar,
 	mb_wchar_to_iso2022jp,
-	mb_check_iso2022jp
+	mb_check_iso2022jp,
+	NULL
 };
 
 static const char *mbfl_encoding_2022jp_kddi_aliases[] = {"ISO-2022-JP-KDDI", NULL};
@@ -4462,6 +4464,7 @@ const mbfl_encoding mbfl_encoding_2022jp_kddi = {
 	&vtbl_wchar_2022jp_kddi,
 	mb_iso2022jp_kddi_to_wchar,
 	mb_wchar_to_iso2022jp_kddi,
+	NULL,
 	NULL
 };
 
@@ -4496,6 +4499,7 @@ const mbfl_encoding mbfl_encoding_2022jp_2004 = {
 	&vtbl_wchar_2022jp_2004,
 	mb_iso2022jp2004_to_wchar,
 	mb_wchar_to_iso2022jp2004,
+	NULL,
 	NULL
 };
 
@@ -4581,6 +4585,7 @@ const mbfl_encoding mbfl_encoding_cp50220 = {
 	&vtbl_wchar_cp50220,
 	mb_cp5022x_to_wchar,
 	mb_wchar_to_cp50220,
+	NULL,
 	NULL
 };
 
@@ -4595,6 +4600,7 @@ const mbfl_encoding mbfl_encoding_cp50221 = {
 	&vtbl_wchar_cp50221,
 	mb_cp5022x_to_wchar,
 	mb_wchar_to_cp50221,
+	NULL,
 	NULL
 };
 
@@ -4609,6 +4615,7 @@ const mbfl_encoding mbfl_encoding_cp50222 = {
 	&vtbl_wchar_cp50222,
 	mb_cp5022x_to_wchar,
 	mb_wchar_to_cp50222,
+	NULL,
 	NULL
 };
 
@@ -4645,6 +4652,7 @@ const mbfl_encoding mbfl_encoding_2022jpms = {
 	&vtbl_wchar_2022jpms,
 	mb_iso2022jpms_to_wchar,
 	mb_wchar_to_iso2022jpms,
+	NULL,
 	NULL
 };
 
@@ -4687,6 +4695,7 @@ const mbfl_encoding mbfl_encoding_2022kr = {
 	&vtbl_wchar_2022kr,
 	mb_iso2022kr_to_wchar,
 	mb_wchar_to_iso2022kr,
+	NULL,
 	NULL
 };
 
@@ -7832,6 +7841,7 @@ const mbfl_encoding mbfl_encoding_sjis = {
 	&vtbl_wchar_sjis,
 	mb_sjis_to_wchar,
 	mb_wchar_to_sjis,
+	NULL,
 	NULL
 };
 
@@ -7868,6 +7878,7 @@ const mbfl_encoding mbfl_encoding_sjis_mac = {
 	&vtbl_wchar_sjis_mac,
 	mb_sjismac_to_wchar,
 	mb_wchar_to_sjismac,
+	NULL,
 	NULL
 };
 
@@ -7906,6 +7917,7 @@ const mbfl_encoding mbfl_encoding_sjis_docomo = {
 	&vtbl_wchar_sjis_docomo,
 	mb_sjis_docomo_to_wchar,
 	mb_wchar_to_sjis_docomo,
+	NULL,
 	NULL
 };
 
@@ -7940,6 +7952,7 @@ const mbfl_encoding mbfl_encoding_sjis_kddi = {
 	&vtbl_wchar_sjis_kddi,
 	mb_sjis_kddi_to_wchar,
 	mb_wchar_to_sjis_kddi,
+	NULL,
 	NULL
 };
 
@@ -7974,6 +7987,7 @@ const mbfl_encoding mbfl_encoding_sjis_sb = {
 	&vtbl_wchar_sjis_sb,
 	mb_sjis_sb_to_wchar,
 	mb_wchar_to_sjis_sb,
+	NULL,
 	NULL
 };
 
@@ -8017,6 +8031,7 @@ const mbfl_encoding mbfl_encoding_sjis2004 = {
 	&vtbl_wchar_sjis2004,
 	mb_sjis2004_to_wchar,
 	mb_wchar_to_sjis2004,
+	NULL,
 	NULL
 };
 
@@ -8103,6 +8118,7 @@ const mbfl_encoding mbfl_encoding_cp932 = {
 	&vtbl_wchar_cp932,
 	mb_cp932_to_wchar,
 	mb_wchar_to_cp932,
+	NULL,
 	NULL
 };
 
@@ -8137,6 +8153,7 @@ const mbfl_encoding mbfl_encoding_sjiswin = {
 	&vtbl_wchar_sjiswin,
 	mb_cp932_to_wchar,
 	mb_wchar_to_sjiswin,
+	NULL,
 	NULL
 };
 
@@ -10346,6 +10363,7 @@ const mbfl_encoding mbfl_encoding_euc_jp = {
 	&vtbl_wchar_eucjp,
 	mb_eucjp_to_wchar,
 	mb_wchar_to_eucjp,
+	NULL,
 	NULL
 };
 
@@ -10382,6 +10400,7 @@ const mbfl_encoding mbfl_encoding_eucjp2004 = {
 	&vtbl_wchar_eucjp2004,
 	mb_eucjp2004_to_wchar,
 	mb_wchar_to_eucjp2004,
+	NULL,
 	NULL
 };
 
@@ -10418,6 +10437,7 @@ const mbfl_encoding mbfl_encoding_eucjp_win = {
 	&vtbl_wchar_eucjpwin,
 	mb_eucjpwin_to_wchar,
 	mb_wchar_to_eucjpwin,
+	NULL,
 	NULL
 };
 
@@ -10454,6 +10474,7 @@ const mbfl_encoding mbfl_encoding_cp51932 = {
 	&vtbl_wchar_cp51932,
 	mb_cp51932_to_wchar,
 	mb_wchar_to_cp51932,
+	NULL,
 	NULL
 };
 
@@ -10509,6 +10530,7 @@ const mbfl_encoding mbfl_encoding_euc_cn = {
 	&vtbl_wchar_euccn,
 	mb_euccn_to_wchar,
 	mb_wchar_to_euccn,
+	NULL,
 	NULL
 };
 
@@ -10545,6 +10567,7 @@ const mbfl_encoding mbfl_encoding_euc_tw = {
 	&vtbl_wchar_euctw,
 	mb_euctw_to_wchar,
 	mb_wchar_to_euctw,
+	NULL,
 	NULL
 };
 
@@ -10581,6 +10604,7 @@ const mbfl_encoding mbfl_encoding_euc_kr = {
 	&vtbl_wchar_euckr,
 	mb_euckr_to_wchar,
 	mb_wchar_to_euckr,
+	NULL,
 	NULL
 };
 
@@ -10640,6 +10664,7 @@ const mbfl_encoding mbfl_encoding_uhc = {
 	&vtbl_wchar_uhc,
 	mb_uhc_to_wchar,
 	mb_wchar_to_uhc,
+	NULL,
 	NULL
 };
 
@@ -11555,6 +11580,7 @@ const mbfl_encoding mbfl_encoding_gb18030 = {
 	&vtbl_wchar_gb18030,
 	mb_gb18030_to_wchar,
 	mb_wchar_to_gb18030,
+	NULL,
 	NULL
 };
 
@@ -11591,6 +11617,7 @@ const mbfl_encoding mbfl_encoding_cp936 = {
 	&vtbl_wchar_cp936,
 	mb_cp936_to_wchar,
 	mb_wchar_to_cp936,
+	NULL,
 	NULL
 };
 
@@ -12160,6 +12187,7 @@ const mbfl_encoding mbfl_encoding_big5 = {
 	&vtbl_wchar_big5,
 	mb_big5_to_wchar,
 	mb_wchar_to_big5,
+	NULL,
 	NULL
 };
 
@@ -12194,6 +12222,7 @@ const mbfl_encoding mbfl_encoding_cp950 = {
 	&vtbl_wchar_cp950,
 	mb_cp950_to_wchar,
 	mb_wchar_to_cp950,
+	NULL,
 	NULL
 };
 
@@ -12567,5 +12596,6 @@ const mbfl_encoding mbfl_encoding_hz = {
 	&vtbl_wchar_hz,
 	mb_hz_to_wchar,
 	mb_wchar_to_hz,
+	NULL,
 	NULL
 };

ext/mbstring/libmbfl/filters/mbfilter_htmlent.c

@@ -67,6 +67,7 @@ const mbfl_encoding mbfl_encoding_html_ent = {
 	&vtbl_wchar_html,
 	mb_htmlent_to_wchar,
 	mb_wchar_to_htmlent,
+	NULL,
 	NULL
 };
 

ext/mbstring/libmbfl/filters/mbfilter_qprint.c

@@ -46,6 +46,7 @@ const mbfl_encoding mbfl_encoding_qprint = {
 	NULL,
 	mb_qprint_to_wchar,
 	mb_wchar_to_qprint,
+	NULL,
 	NULL
 };
 

ext/mbstring/libmbfl/filters/mbfilter_singlebyte.c

@@ -87,6 +87,7 @@ static int mbfl_conv_reverselookup_table(int c, mbfl_convert_filter *filter, int
 		&vtbl_wchar_##id, \
 		mb_##id##_to_wchar, \
 		mb_wchar_to_##id, \
+		NULL, \
 		NULL \
 	}
 

ext/mbstring/libmbfl/filters/mbfilter_ucs2.c

@@ -57,6 +57,7 @@ const mbfl_encoding mbfl_encoding_ucs2 = {
 	&vtbl_wchar_ucs2,
 	mb_ucs2_to_wchar,
 	mb_wchar_to_ucs2be,
+	NULL,
 	NULL
 };
 
@@ -71,6 +72,7 @@ const mbfl_encoding mbfl_encoding_ucs2be = {
 	&vtbl_wchar_ucs2be,
 	mb_ucs2be_to_wchar,
 	mb_wchar_to_ucs2be,
+	NULL,
 	NULL
 };
 
@@ -85,6 +87,7 @@ const mbfl_encoding mbfl_encoding_ucs2le = {
 	&vtbl_wchar_ucs2le,
 	mb_ucs2le_to_wchar,
 	mb_wchar_to_ucs2le,
+	NULL,
 	NULL
 };
 

ext/mbstring/libmbfl/filters/mbfilter_ucs4.c

@@ -57,6 +57,7 @@ const mbfl_encoding mbfl_encoding_ucs4 = {
 	&vtbl_wchar_ucs4,
 	mb_ucs4_to_wchar,
 	mb_wchar_to_ucs4be,
+	NULL,
 	NULL
 };
 
@@ -71,6 +72,7 @@ const mbfl_encoding mbfl_encoding_ucs4be = {
 	&vtbl_wchar_ucs4be,
 	mb_ucs4be_to_wchar,
 	mb_wchar_to_ucs4be,
+	NULL,
 	NULL
 };
 
@@ -85,6 +87,7 @@ const mbfl_encoding mbfl_encoding_ucs4le = {
 	&vtbl_wchar_ucs4le,
 	mb_ucs4le_to_wchar,
 	mb_wchar_to_ucs4le,
+	NULL,
 	NULL
 };
 

ext/mbstring/libmbfl/filters/mbfilter_utf16.c

@@ -189,6 +189,7 @@ const mbfl_encoding mbfl_encoding_utf16 = {
 	&vtbl_wchar_utf16,
 	mb_utf16_to_wchar,
 	mb_wchar_to_utf16be,
+	NULL,
 	NULL
 };
 
@@ -203,6 +204,7 @@ const mbfl_encoding mbfl_encoding_utf16be = {
 	&vtbl_wchar_utf16be,
 	mb_utf16be_to_wchar,
 	mb_wchar_to_utf16be,
+	NULL,
 	NULL
 };
 
@@ -217,6 +219,7 @@ const mbfl_encoding mbfl_encoding_utf16le = {
 	&vtbl_wchar_utf16le,
 	mb_utf16le_to_wchar,
 	mb_wchar_to_utf16le,
+	NULL,
 	NULL
 };
 

ext/mbstring/libmbfl/filters/mbfilter_utf32.c

@@ -50,6 +50,7 @@ const mbfl_encoding mbfl_encoding_utf32 = {
 	&vtbl_wchar_utf32,
 	mb_utf32_to_wchar,
 	mb_wchar_to_utf32be,
+	NULL,
 	NULL
 };
 
@@ -64,6 +65,7 @@ const mbfl_encoding mbfl_encoding_utf32be = {
 	&vtbl_wchar_utf32be,
 	mb_utf32be_to_wchar,
 	mb_wchar_to_utf32be,
+	NULL,
 	NULL
 };
 
@@ -78,6 +80,7 @@ const mbfl_encoding mbfl_encoding_utf32le = {
 	&vtbl_wchar_utf32le,
 	mb_utf32le_to_wchar,
 	mb_wchar_to_utf32le,
+	NULL,
 	NULL
 };