diff --git a/Zend/zend_string.c b/Zend/zend_string.c index b1b4a0e17a69..d9744cf142e2 100644 --- a/Zend/zend_string.c +++ b/Zend/zend_string.c @@ -18,6 +18,7 @@ #include "zend.h" #include "zend_globals.h" #include "zend_multiply.h" +#include "zend_simd.h" #ifdef HAVE_VALGRIND # include "valgrind/callgrind.h" @@ -63,6 +64,35 @@ static zend_always_inline void zend_bin2hex_impl(char *out, const unsigned char } } +#ifdef XSSE2 +static zend_never_inline void zend_bin2hex_simd(char *out, const unsigned char *in, size_t in_len) +{ + const __m128i nibble_mask = _mm_set1_epi8(0x0f); + const __m128i nine = _mm_set1_epi8(9); + const __m128i zero_digit = _mm_set1_epi8('0'); + const __m128i alpha_offset = _mm_set1_epi8('a' - '0' - 10); + size_t i = 0; + + for (; i + sizeof(__m128i) <= in_len; i += sizeof(__m128i)) { + __m128i v = _mm_loadu_si128((const __m128i *) (in + i)); + __m128i hi_nib = _mm_and_si128(_mm_srli_epi16(v, 4), nibble_mask); + __m128i lo_nib = _mm_and_si128(v, nibble_mask); + + /* Signed cmpgt is safe here: every threshold is < 0x80, so bytes >= 0x80 + compare as negative and correctly fail every range test. */ + __m128i hi_hex = _mm_add_epi8(_mm_add_epi8(hi_nib, zero_digit), + _mm_and_si128(_mm_cmpgt_epi8(hi_nib, nine), alpha_offset)); + __m128i lo_hex = _mm_add_epi8(_mm_add_epi8(lo_nib, zero_digit), + _mm_and_si128(_mm_cmpgt_epi8(lo_nib, nine), alpha_offset)); + + _mm_storeu_si128((__m128i *) (out + i * 2), _mm_unpacklo_epi8(hi_hex, lo_hex)); + _mm_storeu_si128((__m128i *) (out + i * 2 + sizeof(__m128i)), _mm_unpackhi_epi8(hi_hex, lo_hex)); + } + + zend_bin2hex_impl(out + i * 2, in + i, in_len - i, zend_hexconvtab); +} +#endif + ZEND_API zend_ulong ZEND_FASTCALL zend_string_hash_func(zend_string *str) { return ZSTR_H(str) = zend_hash_func(ZSTR_VAL(str), ZSTR_LEN(str)); @@ -75,6 +105,12 @@ ZEND_API zend_ulong ZEND_FASTCALL zend_hash_func(const char *str, size_t len) ZEND_API void ZEND_FASTCALL zend_bin2hex(char *out, const unsigned char *in, size_t in_len) { +#ifdef XSSE2 + if (in_len >= sizeof(__m128i)) { + zend_bin2hex_simd(out, in, in_len); + return; + } +#endif zend_bin2hex_impl(out, in, in_len, zend_hexconvtab); } diff --git a/ext/standard/string.c b/ext/standard/string.c index 441890ea2902..2fbfec7074f2 100644 --- a/ext/standard/string.c +++ b/ext/standard/string.c @@ -50,6 +50,61 @@ static MUTEX_T locale_mutex = NULL; #endif +#ifdef XSSE2 +static zend_never_inline bool php_hex2bin_simd_chunk(unsigned char *out16, const unsigned char *in32) +{ + const __m128i lo_bound_digit = _mm_set1_epi8(0x2f); /* '0' - 1 */ + const __m128i hi_bound_digit = _mm_set1_epi8(0x3a); /* '9' + 1 */ + const __m128i lo_bound_upper = _mm_set1_epi8(0x40); /* 'A' - 1 */ + const __m128i hi_bound_upper = _mm_set1_epi8(0x47); /* 'F' + 1 */ + const __m128i lo_bound_lower = _mm_set1_epi8(0x60); /* 'a' - 1 */ + const __m128i hi_bound_lower = _mm_set1_epi8(0x67); /* 'f' + 1 */ + const __m128i letter_bit_mask = _mm_set1_epi8(0x40); /* set in 'A'-'F' and 'a'-'f', clear in '0'-'9' */ + const __m128i low_nibble_mask = _mm_set1_epi8(0x0f); + const __m128i letter_nib_offset = _mm_set1_epi8(9); + const __m128i word_lo_mask = _mm_set1_epi16(0x00ff); + const __m128i nibble_hi_mask = _mm_set1_epi8((char) 0xf0); + const __m128i zero = _mm_setzero_si128(); + __m128i packed[2]; + bool chunk_valid = true; + + for (int half = 0; half < 2; half++) { + __m128i c = _mm_loadu_si128((const __m128i *) (in32 + half * sizeof(__m128i))); + + /* Signed cmpgt is safe here: every threshold is < 0x80, so bytes >= 0x80 + compare as negative and correctly fail every range test. */ + __m128i is_digit = _mm_and_si128(_mm_cmpgt_epi8(c, lo_bound_digit), _mm_cmpgt_epi8(hi_bound_digit, c)); + __m128i is_upper = _mm_and_si128(_mm_cmpgt_epi8(c, lo_bound_upper), _mm_cmpgt_epi8(hi_bound_upper, c)); + __m128i is_lower = _mm_and_si128(_mm_cmpgt_epi8(c, lo_bound_lower), _mm_cmpgt_epi8(hi_bound_lower, c)); + __m128i valid = _mm_or_si128(_mm_or_si128(is_digit, is_upper), is_lower); + + if (_mm_movemask_epi8(valid) != 0xffff) { + chunk_valid = false; + break; + } + + __m128i is_letter = _mm_cmpeq_epi8(_mm_and_si128(c, letter_bit_mask), letter_bit_mask); + __m128i nib = _mm_add_epi8(_mm_and_si128(c, low_nibble_mask), _mm_and_si128(is_letter, letter_nib_offset)); + + __m128i hi_src = _mm_and_si128(nib, word_lo_mask); + __m128i lo_src = _mm_srli_epi16(nib, 8); + __m128i hi_packed = _mm_packus_epi16(hi_src, zero); + __m128i lo_packed = _mm_packus_epi16(lo_src, zero); + __m128i hi_shifted = _mm_and_si128(_mm_slli_epi16(hi_packed, 4), nibble_hi_mask); + + packed[half] = _mm_or_si128(hi_shifted, lo_packed); + } + + if (!chunk_valid) { + return false; + } + + memcpy(out16, &packed[0], 8); + memcpy(out16 + 8, &packed[1], 8); + return true; +} +#endif + /* {{{ php_hex2bin */ static zend_string *php_hex2bin(const unsigned char *old, const size_t oldlen) { @@ -58,7 +113,16 @@ static zend_string *php_hex2bin(const unsigned char *old, const size_t oldlen) unsigned char *ret = (unsigned char *)ZSTR_VAL(str); size_t i, j; - for (i = j = 0; i < target_length; i++) { + i = j = 0; +#ifdef XSSE2 + for (; j + 2 * sizeof(__m128i) <= oldlen; j += 2 * sizeof(__m128i), i += sizeof(__m128i)) { + if (!php_hex2bin_simd_chunk(ret + i, old + j)) { + break; + } + } +#endif + + for (; i < target_length; i++) { unsigned char c = old[j++]; unsigned char l = c & ~0x20; int is_letter = ((unsigned int) ((l - 'A') ^ (l - 'F' - 1))) >> (8 * sizeof(unsigned int) - 1); diff --git a/ext/standard/tests/strings/bin2hex_hex2bin_simd_fuzz.phpt b/ext/standard/tests/strings/bin2hex_hex2bin_simd_fuzz.phpt new file mode 100644 index 000000000000..aaf971aa2e15 --- /dev/null +++ b/ext/standard/tests/strings/bin2hex_hex2bin_simd_fuzz.phpt @@ -0,0 +1,27 @@ +--TEST-- +bin2hex()/hex2bin() SIMD fast path: round-trip fuzz +--FILE-- + +--EXPECT-- +done diff --git a/ext/standard/tests/strings/bin2hex_simd_boundary.phpt b/ext/standard/tests/strings/bin2hex_simd_boundary.phpt new file mode 100644 index 000000000000..2a8df76a1d09 --- /dev/null +++ b/ext/standard/tests/strings/bin2hex_simd_boundary.phpt @@ -0,0 +1,47 @@ +--TEST-- +bin2hex() SIMD fast path: boundary lengths +--FILE-- += 16 bytes) and still leaves a scalar remainder tail (4 bytes), +// encoding the known byte sequence 0..19. +$bin = ''; +for ($i = 0; $i < 20; $i++) { + $bin .= chr($i); +} +echo bin2hex($bin), "\n"; + +$all = ''; +for ($i = 0; $i < 256; $i++) { + $all .= chr($i); +} +$source = str_repeat($all, 5); // 1280 bytes, enough for len 1000 + offsets + +$lengths = [0, 1, 15, 16, 17, 20, 31, 32, 33, 1000]; + +foreach ($lengths as $len) { + $expected = ''; + for ($i = 0; $i < $len; $i++) { + $expected .= sprintf('%02x', ord($source[$i])); + } + $actual = bin2hex(substr($source, 0, $len)); + if ($actual !== $expected) { + echo "FAIL len=$len\n"; + } +} + +// A long string spanning many SIMD chunk iterations. +$big = str_repeat($all, 300); // 76800 bytes +$expectedBig = ''; +for ($i = 0; $i < strlen($big); $i++) { + $expectedBig .= sprintf('%02x', ord($big[$i])); +} +if (bin2hex($big) !== $expectedBig) { + echo "FAIL long string\n"; +} + +echo "done\n"; +?> +--EXPECT-- +000102030405060708090a0b0c0d0e0f10111213 +done diff --git a/ext/standard/tests/strings/hex2bin_simd_boundary.phpt b/ext/standard/tests/strings/hex2bin_simd_boundary.phpt new file mode 100644 index 000000000000..b62b998b7d13 --- /dev/null +++ b/ext/standard/tests/strings/hex2bin_simd_boundary.phpt @@ -0,0 +1,83 @@ +--TEST-- +hex2bin() SIMD fast path: boundary lengths and invalid-byte-at-every-offset +--FILE-- += 32 hex chars) and still leaves a scalar remainder +// tail (4 bytes), decoding to the known byte sequence 0..19. +$hex = "000102030405060708090a0b0c0d0e0f10111213"; +$bin = hex2bin($hex); +$bytes = []; +for ($i = 0; $i < strlen($bin); $i++) { + $bytes[] = ord($bin[$i]); +} +echo implode(' ', $bytes), "\n"; + +$hex = str_repeat('0123456789abcdefABCDEF', 10); +$hex = substr($hex, 0, 2000); +if (strlen($hex) % 2 !== 0) { + $hex = substr($hex, 0, -1); +} + +$lengths = [0, 2, 30, 32, 34, 40, 62, 64, 66, 2000]; +foreach ($lengths as $len) { + $chunk = substr($hex, 0, $len); + $bin = hex2bin($chunk); + $roundtrip = bin2hex($bin); + if (strtolower($chunk) !== $roundtrip) { + echo "FAIL len=$len\n"; + } +} + +// Dedicated regression cases for the early-exit in php_hex2bin_simd_chunk's +// per-half loop +$prefix = str_repeat('0123456789abcdef', 4); // 64 valid hex chars, unrelated leading chunk +$validChunk32 = str_repeat('0123456789abcdef', 2); // exactly one 32-byte SIMD chunk +$suffix = str_repeat('abcdef0123456789', 4); // 64 more valid hex chars after + +// Invalid byte at position 5 -> falls in the FIRST half (offsets 0-15) of the chunk. +$chunk = $validChunk32; +$chunk[5] = 'z'; +$s = $prefix . $chunk . $suffix; +if (@hex2bin($s) !== false) { + echo "FAIL: invalid byte in first half (pos 5) was not rejected\n"; +} + +// Invalid byte at position 20 -> falls in the SECOND half (offsets 16-31) of the chunk. +$chunk = $validChunk32; +$chunk[20] = 'z'; +$s = $prefix . $chunk . $suffix; +if (@hex2bin($s) !== false) { + echo "FAIL: invalid byte in second half (pos 20) was not rejected\n"; +} + +// Invalid byte at every offset 0..63 relative to the 32-byte SIMD chunk +// boundary (two full SIMD chunks), across a few different bad-byte classes. +$valid64 = str_repeat('0123456789abcdef', 4); +$badChars = ['g', 'G', 'z', '!', ' ', "\xFF", "\x00", '-']; +foreach (range(0, 63) as $pos) { + foreach ($badChars as $bad) { + $s = $valid64; + $s[$pos] = $bad; + $result = @hex2bin($s); + if ($result !== false) { + echo "FAIL: pos=$pos char=" . bin2hex($bad) . " was not rejected\n"; + } + } +} + +// A long hex string spanning many SIMD chunk iterations. +$bigHex = str_repeat('0123456789abcdefABCDEF', 3000); +if (strlen($bigHex) % 2 !== 0) { + $bigHex = substr($bigHex, 0, -1); +} +$bigBin = hex2bin($bigHex); +if (strtolower($bigHex) !== bin2hex($bigBin)) { + echo "FAIL long hex string\n"; +} + +echo "done\n"; +?> +--EXPECT-- +0 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 +done