// Convert up to 16 bytes from utf8 to utf16 using a mask indicating the // end of the code points. Only the least significant 12 bits of the mask // are accessed. // It returns how many bytes were consumed (up to 16, usually 12). template <endianness big_endian> size_t convert_masked_utf8_to_utf16(const char *input, uint64_t utf8_end_of_code_point_mask, char16_t *&utf16_output) { // we use an approach where we try to process up to 12 input bytes. // Why 12 input bytes and not 16? Because we are concerned with the size of // the lookup tables. Also 12 is nicely divisible by two and three. // uint8x16_t in = vld1q_u8(reinterpret_cast<const uint8_t *>(input)); const uint16_t input_utf8_end_of_code_point_mask = utf8_end_of_code_point_mask & 0xfff; // // Optimization note: our main path below is load-latency dependent. Thus it // is maybe beneficial to have fast paths that depend on branch prediction but // have less latency. This results in more instructions but, potentially, also // higher speeds. // We first try a few fast paths. // The obvious first test is ASCII, which actually consumes the full 16. if ((utf8_end_of_code_point_mask & 0xFFFF) == 0xffff) { // We process in chunks of 16 bytes // The routine in simd.h is reused. simd8<int8_t> temp{vreinterpretq_s8_u8(in)}; temp.store_ascii_as_utf16<big_endian>(utf16_output); utf16_output += 16; // We wrote 16 16-bit characters. return 16; // We consumed 16 bytes. } // 3 byte sequences are the next most common, as seen in CJK, which has long // sequences of these. if (input_utf8_end_of_code_point_mask == 0x924) { // We want to take 4 3-byte UTF-8 code units and turn them into 4 2-byte // UTF-16 code units. uint16x4_t composed = convert_utf8_3_byte_to_utf16(in); // Byte swap if necessary if (!match_system(big_endian)) { composed = vreinterpret_u16_u8(vrev16_u8(vreinterpret_u8_u16(composed))); } vst1_u16(reinterpret_cast<uint16_t *>(utf16_output), composed); utf16_output += 4; // We wrote 4 16-bit characters. return 12; // We consumed 12 bytes. } // 2 byte sequences occur in short bursts in languages like Greek and Russian. if ((utf8_end_of_code_point_mask & 0xFFF) == 0xaaa) { // We want to take 6 2-byte UTF-8 code units and turn them into 6 2-byte // UTF-16 code units. uint16x8_t composed = convert_utf8_2_byte_to_utf16(in); // Byte swap if necessary if (!match_system(big_endian)) { composed = vreinterpretq_u16_u8(vrev16q_u8(vreinterpretq_u8_u16(composed))); } vst1q_u16(reinterpret_cast<uint16_t *>(utf16_output), composed); utf16_output += 6; // We wrote 6 16-bit characters. return 12; // We consumed 12 bytes. } /// We do not have a fast path available, or the fast path is unimportant, so /// we fallback. const uint8_t idx = simdutf::tables::utf8_to_utf16::utf8bigindex [input_utf8_end_of_code_point_mask][0]; const uint8_t consumed = simdutf::tables::utf8_to_utf16::utf8bigindex [input_utf8_end_of_code_point_mask][1]; if (idx < 64) { // SIX (6) input code-code units // Convert to UTF-16 uint16x8_t composed = convert_utf8_1_to_2_byte_to_utf16(in, idx); // Byte swap if necessary if (!match_system(big_endian)) { composed = vreinterpretq_u16_u8(vrev16q_u8(vreinterpretq_u8_u16(composed))); } // Store vst1q_u16(reinterpret_cast<uint16_t *>(utf16_output), composed); utf16_output += 6; // We wrote 6 16-bit characters. return consumed; } else if (idx < 145) { // FOUR (4) input code-code units // UTF-16 and UTF-32 use similar algorithms, but UTF-32 skips the narrowing. uint8x16_t sh = vld1q_u8(reinterpret_cast<const uint8_t *>( simdutf::tables::utf8_to_utf16::shufutf8[idx])); // XXX: depending on the system scalar instructions might be faster. // 1 byte: 00000000 00000000 0ccccccc // 2 byte: 00000000 110bbbbb 10cccccc // 3 byte: 1110aaaa 10bbbbbb 10cccccc uint32x4_t perm = vreinterpretq_u32_u8(vqtbl1q_u8(in, sh)); // 1 byte: 00000000 0ccccccc // 2 byte: xx0bbbbb x0cccccc // 3 byte: xxbbbbbb x0cccccc uint16x4_t lowperm = vmovn_u32(perm); // Partially mask with bic (doesn't require a temporary register unlike and) // The shift left insert below will clear the top bits. // 1 byte: 00000000 00000000 // 2 byte: xx0bbbbb 00000000 // 3 byte: xxbbbbbb 00000000 uint16x4_t middlebyte = vbic_u16(lowperm, vmov_n_u16(uint16_t(~0xFF00))); // ASCII // 1 byte: 00000000 0ccccccc // 2+byte: 00000000 00cccccc uint16x4_t ascii = vand_u16(lowperm, vmov_n_u16(0x7F)); // Split into narrow vectors. // 2 byte: 00000000 00000000 // 3 byte: 00000000 xxxxaaaa uint16x4_t highperm = vshrn_n_u32(perm, 16); // Shift right accumulate the middle byte // 1 byte: 00000000 0ccccccc // 2 byte: 00xx0bbb bbcccccc // 3 byte: 00xxbbbb bbcccccc uint16x4_t middlelow = vsra_n_u16(ascii, middlebyte, 2); // Shift left and insert the top 4 bits, overwriting the garbage // 1 byte: 00000000 0ccccccc // 2 byte: 00000bbb bbcccccc // 3 byte: aaaabbbb bbcccccc uint16x4_t composed = vsli_n_u16(middlelow, highperm, 12); // Byte swap if necessary if (!match_system(big_endian)) { composed = vreinterpret_u16_u8(vrev16_u8(vreinterpret_u8_u16(composed))); } vst1_u16(reinterpret_cast<uint16_t *>(utf16_output), composed); utf16_output += 4; // We wrote 4 16-bit codepoints return consumed; } else if (idx < 209) { // THREE (3) input code-code units if (input_utf8_end_of_code_point_mask == 0x888) { // We want to take 3 4-byte UTF-8 code units and turn them into 3 4-byte // UTF-16 pairs. Generating surrogate pairs is a little tricky though, but // it is easier when we can assume they are all pairs. This version does // not use the LUT, but 4 byte sequences are less common and the overhead // of the extra memory access is less important than the early branch // overhead in shorter sequences. // Swap byte pairs // 10dddddd 10cccccc|10bbbbbb 11110aaa // 10cccccc 10dddddd|11110aaa 10bbbbbb uint8x16_t swap = vrev16q_u8(in); // Shift left 2 bits // cccccc00 dddddd00 xxxxxxxx bbbbbb00 uint32x4_t shift = vreinterpretq_u32_u8(vshlq_n_u8(swap, 2)); // Create a magic number containing the low 2 bits of the trail surrogate // and all the corrections needed to create the pair. UTF-8 4b prefix = // -0x0000|0xF000 surrogate offset = -0x0000|0x0040 (0x10000 << 6) // surrogate high = +0x0000|0xD800 // surrogate low = +0xDC00|0x0000 // ------------------------------- // = +0xDC00|0xE7C0 uint32x4_t magic = vmovq_n_u32(0xDC00E7C0); // Generate unadjusted trail surrogate minus lowest 2 bits // xxxxxxxx xxxxxxxx|11110aaa bbbbbb00 uint32x4_t trail = vbslq_u32(vmovq_n_u32(0x0000FF00), vreinterpretq_u32_u8(swap), shift); // Insert low 2 bits of trail surrogate to magic number for later // 11011100 00000000 11100111 110000cc uint16x8_t magic_with_low_2 = vreinterpretq_u16_u32(vsraq_n_u32(magic, shift, 30)); // Generate lead surrogate // xxxxcccc ccdddddd|xxxxxxxx xxxxxxxx uint32x4_t lead = vreinterpretq_u32_u16( vsliq_n_u16(vreinterpretq_u16_u8(swap), vreinterpretq_u16_u8(in), 6)); // Mask out lead // 000000cc ccdddddd|xxxxxxxx xxxxxxxx lead = vbicq_u32(lead, vmovq_n_u32(uint32_t(~0x03FFFFFF))); // Blend pairs // 000000cc ccdddddd|11110aaa bbbbbb00 uint16x8_t blend = vreinterpretq_u16_u32( vbslq_u32(vmovq_n_u32(0x0000FFFF), trail, lead)); // Add magic number to finish the result // 110111CC CCDDDDDD|110110AA BBBBBBCC uint16x8_t composed = vaddq_u16(blend, magic_with_low_2); // Byte swap if necessary if (!match_system(big_endian)) { composed = vreinterpretq_u16_u8(vrev16q_u8(vreinterpretq_u8_u16(composed))); } uint16_t buffer[8]; vst1q_u16(reinterpret_cast<uint16_t *>(buffer), composed); for (int k = 0; k < 6; k++) { utf16_output[k] = buffer[k]; } // the loop might compiler to a couple of instructions. utf16_output += 6; // We wrote 3 32-bit surrogate pairs. return 12; // We consumed 12 bytes. } // 3 1-4 byte sequences uint8x16_t sh = vld1q_u8(reinterpret_cast<const uint8_t *>( simdutf::tables::utf8_to_utf16::shufutf8[idx])); // 1 byte: 00000000 00000000 00000000 0ddddddd // 3 byte: 00000000 00000000 110ccccc 10dddddd // 3 byte: 00000000 1110bbbb 10cccccc 10dddddd // 4 byte: 11110aaa 10bbbbbb 10cccccc 10dddddd uint32x4_t perm = vreinterpretq_u32_u8(vqtbl1q_u8(in, sh)); // added to fix issue https://github.com/simdutf/simdutf/issues/514 // We only want to write 2 * 16-bit code units when that is actually what we // have. Unfortunately, we cannot trust the input. So it is possible to get // 0xff as an input byte and it should not result in a surrogate pair. We // need to check for that. uint32_t permbuffer[4]; vst1q_u32(permbuffer, perm); // Mask the low and middle bytes // 00000000 00000000 00000000 0ddddddd uint32x4_t ascii = vandq_u32(perm, vmovq_n_u32(0x7f)); // Because the surrogates need more work, the high surrogate is computed // first. uint32x4_t middlehigh = vshlq_n_u32(perm, 2); // 00000000 00000000 00cccccc 00000000 uint32x4_t middlebyte = vandq_u32(perm, vmovq_n_u32(0x3F00)); // Start assembling the sequence. Since the 4th byte is in the same position // as it would be in a surrogate and there is no dependency, shift left // instead of right. 3 byte: 00000000 10bbbbxx xxxxxxxx xxxxxxxx 4 byte: // 11110aaa bbbbbbxx xxxxxxxx xxxxxxxx uint32x4_t ab = vbslq_u32(vmovq_n_u32(0xFF000000), perm, middlehigh); // Top 16 bits contains the high ten bits of the surrogate pair before // correction 3 byte: 00000000 10bbbbcc|cccc0000 00000000 4 byte: 11110aaa // bbbbbbcc|cccc0000 00000000 - high 10 bits correct w/o correction uint32x4_t abc = vbslq_u32(vmovq_n_u32(0xFFFC0000), ab, vshlq_n_u32(middlebyte, 4)); // Combine the low 6 or 7 bits by a shift right accumulate // 3 byte: 00000000 00000010|bbbbcccc ccdddddd - low 16 bits correct // 4 byte: 00000011 110aaabb|bbbbcccc ccdddddd - low 10 bits correct w/o // correction uint32x4_t composed = vsraq_n_u32(ascii, abc, 6); // After this is for surrogates // Blend the low and high surrogates // 4 byte: 11110aaa bbbbbbcc|bbbbcccc ccdddddd uint32x4_t mixed = vbslq_u32(vmovq_n_u32(0xFFFF0000), abc, composed); // Clear the upper 6 bits of the low surrogate. Don't clear the upper bits // yet as 0x10000 was not subtracted from the codepoint yet. 4 byte: // 11110aaa bbbbbbcc|000000cc ccdddddd uint16x8_t masked_pair = vreinterpretq_u16_u32( vbicq_u32(mixed, vmovq_n_u32(uint32_t(~0xFFFF03FF)))); // Correct the remaining UTF-8 prefix, surrogate offset, and add the // surrogate prefixes in one magic 16-bit addition. similar magic number but // without the continue byte adjust and halfword swapped UTF-8 4b prefix = // -0xF000|0x0000 surrogate offset = -0x0040|0x0000 (0x10000 << 6) // surrogate high = +0xD800|0x0000 // surrogate low = +0x0000|0xDC00 // ----------------------------------- // = +0xE7C0|0xDC00 uint16x8_t magic = vreinterpretq_u16_u32(vmovq_n_u32(0xE7C0DC00)); // 4 byte: 110110AA BBBBBBCC|110111CC CCDDDDDD - surrogate pair complete uint32x4_t surrogates = vreinterpretq_u32_u16(vaddq_u16(masked_pair, magic)); // If the high bit is 1 (s32 less than zero), this needs a surrogate pair uint32x4_t is_pair = vcltzq_s32(vreinterpretq_s32_u32(perm)); // Select either the 4 byte surrogate pair or the 2 byte solo codepoint // 3 byte: 0xxxxxxx xxxxxxxx|bbbbcccc ccdddddd // 4 byte: 110110AA BBBBBBCC|110111CC CCDDDDDD uint32x4_t selected = vbslq_u32(is_pair, surrogates, composed); // Byte swap if necessary if (!match_system(big_endian)) { selected = vreinterpretq_u32_u8(vrev16q_u8(vreinterpretq_u8_u32(selected))); } // Attempting to shuffle and store would be complex, just scalarize. uint32_t buffer[4]; vst1q_u32(buffer, selected); // Test for the top bit of the surrogate mask. Remove due to issue 514 // const uint32_t SURROGATE_MASK = match_system(big_endian) ? 0x80000000 : // 0x00800000; for (size_t i = 0; i < 3; i++) { // Surrogate // Used to be if (buffer[i] & SURROGATE_MASK) { // See discussion above. // patch for issue https://github.com/simdutf/simdutf/issues/514 if ((permbuffer[i] & 0xf8000000) == 0xf0000000) { utf16_output[0] = uint16_t(buffer[i] >> 16); utf16_output[1] = uint16_t(buffer[i] & 0xFFFF); utf16_output += 2; } else { utf16_output[0] = uint16_t(buffer[i] & 0xFFFF); utf16_output++; } } return consumed; } else { // here we know that there is an error but we do not handle errors return 12; } }