ext/standard: Add SSE2/NEON fast path for bin2hex()/hex2bin()

Zend/zend_string.c

@@ -18,6 +18,7 @@
 #include "zend.h"
 #include "zend_globals.h"
 #include "zend_multiply.h"
+#include "zend_simd.h"
 
 #ifdef HAVE_VALGRIND
 # include "valgrind/callgrind.h"
@@ -63,6 +64,35 @@ static zend_always_inline void zend_bin2hex_impl(char *out, const unsigned char
 	}
 }
 
+#ifdef XSSE2
+static zend_never_inline void zend_bin2hex_simd(char *out, const unsigned char *in, size_t in_len)
+{
+	const __m128i nibble_mask = _mm_set1_epi8(0x0f);
+	const __m128i nine = _mm_set1_epi8(9);
+	const __m128i zero_digit = _mm_set1_epi8('0');
+	const __m128i alpha_offset = _mm_set1_epi8('a' - '0' - 10);
+	size_t i = 0;
+
+	for (; i + sizeof(__m128i) <= in_len; i += sizeof(__m128i)) {
+		__m128i v = _mm_loadu_si128((const __m128i *) (in + i));
+		__m128i hi_nib = _mm_and_si128(_mm_srli_epi16(v, 4), nibble_mask);
+		__m128i lo_nib = _mm_and_si128(v, nibble_mask);
+
+		/* Signed cmpgt is safe here: every threshold is < 0x80, so bytes >= 0x80
+		   compare as negative and correctly fail every range test. */
+		__m128i hi_hex = _mm_add_epi8(_mm_add_epi8(hi_nib, zero_digit),
+			_mm_and_si128(_mm_cmpgt_epi8(hi_nib, nine), alpha_offset));
+		__m128i lo_hex = _mm_add_epi8(_mm_add_epi8(lo_nib, zero_digit),
+			_mm_and_si128(_mm_cmpgt_epi8(lo_nib, nine), alpha_offset));
+
+		_mm_storeu_si128((__m128i *) (out + i * 2), _mm_unpacklo_epi8(hi_hex, lo_hex));
+		_mm_storeu_si128((__m128i *) (out + i * 2 + sizeof(__m128i)), _mm_unpackhi_epi8(hi_hex, lo_hex));
+	}
+
+	zend_bin2hex_impl(out + i * 2, in + i, in_len - i, zend_hexconvtab);
+}
+#endif
+
 ZEND_API zend_ulong ZEND_FASTCALL zend_string_hash_func(zend_string *str)
 {
 	return ZSTR_H(str) = zend_hash_func(ZSTR_VAL(str), ZSTR_LEN(str));
@@ -75,6 +105,12 @@ ZEND_API zend_ulong ZEND_FASTCALL zend_hash_func(const char *str, size_t len)
 
 ZEND_API void ZEND_FASTCALL zend_bin2hex(char *out, const unsigned char *in, size_t in_len)
 {
+#ifdef XSSE2
+	if (in_len >= sizeof(__m128i)) {
+		zend_bin2hex_simd(out, in, in_len);
+		return;
+	}
+#endif
 	zend_bin2hex_impl(out, in, in_len, zend_hexconvtab);
 }
 

ext/standard/string.c

@@ -50,6 +50,61 @@
 static MUTEX_T locale_mutex = NULL;
 #endif
 
+#ifdef XSSE2
+static zend_never_inline bool php_hex2bin_simd_chunk(unsigned char *out16, const unsigned char *in32)
+{
+	const __m128i lo_bound_digit = _mm_set1_epi8(0x2f); /* '0' - 1 */
+	const __m128i hi_bound_digit = _mm_set1_epi8(0x3a); /* '9' + 1 */
+	const __m128i lo_bound_upper = _mm_set1_epi8(0x40); /* 'A' - 1 */
+	const __m128i hi_bound_upper = _mm_set1_epi8(0x47); /* 'F' + 1 */
+	const __m128i lo_bound_lower = _mm_set1_epi8(0x60); /* 'a' - 1 */
+	const __m128i hi_bound_lower = _mm_set1_epi8(0x67); /* 'f' + 1 */
+	const __m128i letter_bit_mask = _mm_set1_epi8(0x40); /* set in 'A'-'F' and 'a'-'f', clear in '0'-'9' */
+	const __m128i low_nibble_mask = _mm_set1_epi8(0x0f);
+	const __m128i letter_nib_offset = _mm_set1_epi8(9);
+	const __m128i word_lo_mask = _mm_set1_epi16(0x00ff);
+	const __m128i nibble_hi_mask = _mm_set1_epi8((char) 0xf0);
+	const __m128i zero = _mm_setzero_si128();
+	__m128i packed[2];
+	bool chunk_valid = true;
+
+	for (int half = 0; half < 2; half++) {
+		__m128i c = _mm_loadu_si128((const __m128i *) (in32 + half * sizeof(__m128i)));
+
+		/* Signed cmpgt is safe here: every threshold is < 0x80, so bytes >= 0x80
+		   compare as negative and correctly fail every range test. */
+		__m128i is_digit = _mm_and_si128(_mm_cmpgt_epi8(c, lo_bound_digit), _mm_cmpgt_epi8(hi_bound_digit, c));
+		__m128i is_upper = _mm_and_si128(_mm_cmpgt_epi8(c, lo_bound_upper), _mm_cmpgt_epi8(hi_bound_upper, c));
+		__m128i is_lower = _mm_and_si128(_mm_cmpgt_epi8(c, lo_bound_lower), _mm_cmpgt_epi8(hi_bound_lower, c));
+		__m128i valid = _mm_or_si128(_mm_or_si128(is_digit, is_upper), is_lower);
+
+		if (_mm_movemask_epi8(valid) != 0xffff) {
+			chunk_valid = false;
+			break;
+		}
+
+		__m128i is_letter = _mm_cmpeq_epi8(_mm_and_si128(c, letter_bit_mask), letter_bit_mask);
+		__m128i nib = _mm_add_epi8(_mm_and_si128(c, low_nibble_mask), _mm_and_si128(is_letter, letter_nib_offset));
+
+		__m128i hi_src = _mm_and_si128(nib, word_lo_mask);
+		__m128i lo_src = _mm_srli_epi16(nib, 8);
+		__m128i hi_packed = _mm_packus_epi16(hi_src, zero);
+		__m128i lo_packed = _mm_packus_epi16(lo_src, zero);
+		__m128i hi_shifted = _mm_and_si128(_mm_slli_epi16(hi_packed, 4), nibble_hi_mask);
+
+		packed[half] = _mm_or_si128(hi_shifted, lo_packed);
+	}
+
+	if (!chunk_valid) {
+		return false;
+	}
+
+	memcpy(out16, &packed[0], 8);
+	memcpy(out16 + 8, &packed[1], 8);
+	return true;
+}
+#endif
+
 /* {{{ php_hex2bin */
 static zend_string *php_hex2bin(const unsigned char *old, const size_t oldlen)
 {
@@ -58,7 +113,16 @@ static zend_string *php_hex2bin(const unsigned char *old, const size_t oldlen)
 	unsigned char *ret = (unsigned char *)ZSTR_VAL(str);
 	size_t i, j;
 
-	for (i = j = 0; i < target_length; i++) {
+	i = j = 0;
+#ifdef XSSE2
+	for (; j + 2 * sizeof(__m128i) <= oldlen; j += 2 * sizeof(__m128i), i += sizeof(__m128i)) {
+		if (!php_hex2bin_simd_chunk(ret + i, old + j)) {
+			break;
+		}
+	}
+#endif
+
+	for (; i < target_length; i++) {
 		unsigned char c = old[j++];
 		unsigned char l = c & ~0x20;
 		int is_letter = ((unsigned int) ((l - 'A') ^ (l - 'F' - 1))) >> (8 * sizeof(unsigned int) - 1);

ext/standard/tests/strings/bin2hex_hex2bin_simd_fuzz.phpt

@@ -0,0 +1,27 @@
+--TEST--
+bin2hex()/hex2bin() SIMD fast path: round-trip fuzz
+--FILE--
+<?php
+mt_srand(1234567);
+
+$fail = 0;
+for ($i = 0; $i < 500; $i++) {
+    $len = mt_rand(0, 200);
+    $bytes = '';
+    for ($j = 0; $j < $len; $j++) {
+        $bytes .= chr(mt_rand(0, 255));
+    }
+
+    $hex = bin2hex($bytes);
+    $roundtrip = bin2hex(hex2bin($hex));
+
+    if ($roundtrip !== $hex) {
+        echo "FAIL at i=$i len=$len\n";
+        $fail++;
+    }
+}
+
+echo $fail === 0 ? "done\n" : "$fail failures\n";
+?>
+--EXPECT--
+done

ext/standard/tests/strings/bin2hex_simd_boundary.phpt

@@ -0,0 +1,47 @@
+--TEST--
+bin2hex() SIMD fast path: boundary lengths
+--FILE--
+<?php
+// Hardcoded golden vector: 20 bytes is long enough to engage the SIMD path
+// (needs >= 16 bytes) and still leaves a scalar remainder tail (4 bytes),
+// encoding the known byte sequence 0..19.
+$bin = '';
+for ($i = 0; $i < 20; $i++) {
+    $bin .= chr($i);
+}
+echo bin2hex($bin), "\n";
+
+$all = '';
+for ($i = 0; $i < 256; $i++) {
+    $all .= chr($i);
+}
+$source = str_repeat($all, 5); // 1280 bytes, enough for len 1000 + offsets
+
+$lengths = [0, 1, 15, 16, 17, 20, 31, 32, 33, 1000];
+
+foreach ($lengths as $len) {
+    $expected = '';
+    for ($i = 0; $i < $len; $i++) {
+        $expected .= sprintf('%02x', ord($source[$i]));
+    }
+    $actual = bin2hex(substr($source, 0, $len));
+    if ($actual !== $expected) {
+        echo "FAIL len=$len\n";
+    }
+}
+
+// A long string spanning many SIMD chunk iterations.
+$big = str_repeat($all, 300); // 76800 bytes
+$expectedBig = '';
+for ($i = 0; $i < strlen($big); $i++) {
+    $expectedBig .= sprintf('%02x', ord($big[$i]));
+}
+if (bin2hex($big) !== $expectedBig) {
+    echo "FAIL long string\n";
+}
+
+echo "done\n";
+?>
+--EXPECT--
+000102030405060708090a0b0c0d0e0f10111213
+done

ext/standard/tests/strings/hex2bin_simd_boundary.phpt

@@ -0,0 +1,83 @@
+--TEST--
+hex2bin() SIMD fast path: boundary lengths and invalid-byte-at-every-offset
+--FILE--
+<?php
+// Hardcoded golden vector: 40 hex chars (20 bytes) is long enough to engage
+// the SIMD path (needs >= 32 hex chars) and still leaves a scalar remainder
+// tail (4 bytes), decoding to the known byte sequence 0..19.
+$hex = "000102030405060708090a0b0c0d0e0f10111213";
+$bin = hex2bin($hex);
+$bytes = [];
+for ($i = 0; $i < strlen($bin); $i++) {
+    $bytes[] = ord($bin[$i]);
+}
+echo implode(' ', $bytes), "\n";
+
+$hex = str_repeat('0123456789abcdefABCDEF', 10);
+$hex = substr($hex, 0, 2000);
+if (strlen($hex) % 2 !== 0) {
+    $hex = substr($hex, 0, -1);
+}
+
+$lengths = [0, 2, 30, 32, 34, 40, 62, 64, 66, 2000];
+foreach ($lengths as $len) {
+    $chunk = substr($hex, 0, $len);
+    $bin = hex2bin($chunk);
+    $roundtrip = bin2hex($bin);
+    if (strtolower($chunk) !== $roundtrip) {
+        echo "FAIL len=$len\n";
+    }
+}
+
+// Dedicated regression cases for the early-exit in php_hex2bin_simd_chunk's
+// per-half loop
+$prefix = str_repeat('0123456789abcdef', 4); // 64 valid hex chars, unrelated leading chunk
+$validChunk32 = str_repeat('0123456789abcdef', 2); // exactly one 32-byte SIMD chunk
+$suffix = str_repeat('abcdef0123456789', 4); // 64 more valid hex chars after
+
+// Invalid byte at position 5 -> falls in the FIRST half (offsets 0-15) of the chunk.
+$chunk = $validChunk32;
+$chunk[5] = 'z';
+$s = $prefix . $chunk . $suffix;
+if (@hex2bin($s) !== false) {
+    echo "FAIL: invalid byte in first half (pos 5) was not rejected\n";
+}
+
+// Invalid byte at position 20 -> falls in the SECOND half (offsets 16-31) of the chunk.
+$chunk = $validChunk32;
+$chunk[20] = 'z';
+$s = $prefix . $chunk . $suffix;
+if (@hex2bin($s) !== false) {
+    echo "FAIL: invalid byte in second half (pos 20) was not rejected\n";
+}
+
+// Invalid byte at every offset 0..63 relative to the 32-byte SIMD chunk
+// boundary (two full SIMD chunks), across a few different bad-byte classes.
+$valid64 = str_repeat('0123456789abcdef', 4);
+$badChars = ['g', 'G', 'z', '!', ' ', "\xFF", "\x00", '-'];
+foreach (range(0, 63) as $pos) {
+    foreach ($badChars as $bad) {
+        $s = $valid64;
+        $s[$pos] = $bad;
+        $result = @hex2bin($s);
+        if ($result !== false) {
+            echo "FAIL: pos=$pos char=" . bin2hex($bad) . " was not rejected\n";
+        }
+    }
+}
+
+// A long hex string spanning many SIMD chunk iterations.
+$bigHex = str_repeat('0123456789abcdefABCDEF', 3000);
+if (strlen($bigHex) % 2 !== 0) {
+    $bigHex = substr($bigHex, 0, -1);
+}
+$bigBin = hex2bin($bigHex);
+if (strtolower($bigHex) !== bin2hex($bigBin)) {
+    echo "FAIL long hex string\n";
+}
+
+echo "done\n";
+?>
+--EXPECT--
+0 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19
+done