1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
|
namespace simdutf {
namespace SIMDUTF_IMPLEMENTATION {
namespace {
namespace utf8_validation {
using namespace simd;
simdutf_really_inline simd8<uint8_t>
check_special_cases(const simd8<uint8_t> input, const simd8<uint8_t> prev1) {
// Bit 0 = Too Short (lead byte/ASCII followed by lead byte/ASCII)
// Bit 1 = Too Long (ASCII followed by continuation)
// Bit 2 = Overlong 3-byte
// Bit 4 = Surrogate
// Bit 5 = Overlong 2-byte
// Bit 7 = Two Continuations
constexpr const uint8_t TOO_SHORT = 1 << 0; // 11______ 0_______
// 11______ 11______
constexpr const uint8_t TOO_LONG = 1 << 1; // 0_______ 10______
constexpr const uint8_t OVERLONG_3 = 1 << 2; // 11100000 100_____
constexpr const uint8_t SURROGATE = 1 << 4; // 11101101 101_____
constexpr const uint8_t OVERLONG_2 = 1 << 5; // 1100000_ 10______
constexpr const uint8_t TWO_CONTS = 1 << 7; // 10______ 10______
constexpr const uint8_t TOO_LARGE = 1 << 3; // 11110100 1001____
// 11110100 101_____
// 11110101 1001____
// 11110101 101_____
// 1111011_ 1001____
// 1111011_ 101_____
// 11111___ 1001____
// 11111___ 101_____
constexpr const uint8_t TOO_LARGE_1000 = 1 << 6;
// 11110101 1000____
// 1111011_ 1000____
// 11111___ 1000____
constexpr const uint8_t OVERLONG_4 = 1 << 6; // 11110000 1000____
const simd8<uint8_t> byte_1_high = prev1.shr<4>().lookup_16<uint8_t>(
// 0_______ ________ <ASCII in byte 1>
TOO_LONG, TOO_LONG, TOO_LONG, TOO_LONG, TOO_LONG, TOO_LONG, TOO_LONG,
TOO_LONG,
// 10______ ________ <continuation in byte 1>
TWO_CONTS, TWO_CONTS, TWO_CONTS, TWO_CONTS,
// 1100____ ________ <two byte lead in byte 1>
TOO_SHORT | OVERLONG_2,
// 1101____ ________ <two byte lead in byte 1>
TOO_SHORT,
// 1110____ ________ <three byte lead in byte 1>
TOO_SHORT | OVERLONG_3 | SURROGATE,
// 1111____ ________ <four+ byte lead in byte 1>
TOO_SHORT | TOO_LARGE | TOO_LARGE_1000 | OVERLONG_4);
constexpr const uint8_t CARRY =
TOO_SHORT | TOO_LONG | TWO_CONTS; // These all have ____ in byte 1 .
const simd8<uint8_t> byte_1_low =
(prev1 & 0x0F)
.lookup_16<uint8_t>(
// ____0000 ________
CARRY | OVERLONG_3 | OVERLONG_2 | OVERLONG_4,
// ____0001 ________
CARRY | OVERLONG_2,
// ____001_ ________
CARRY, CARRY,
// ____0100 ________
CARRY | TOO_LARGE,
// ____0101 ________
CARRY | TOO_LARGE | TOO_LARGE_1000,
// ____011_ ________
CARRY | TOO_LARGE | TOO_LARGE_1000,
CARRY | TOO_LARGE | TOO_LARGE_1000,
// ____1___ ________
CARRY | TOO_LARGE | TOO_LARGE_1000,
CARRY | TOO_LARGE | TOO_LARGE_1000,
CARRY | TOO_LARGE | TOO_LARGE_1000,
CARRY | TOO_LARGE | TOO_LARGE_1000,
CARRY | TOO_LARGE | TOO_LARGE_1000,
// ____1101 ________
CARRY | TOO_LARGE | TOO_LARGE_1000 | SURROGATE,
CARRY | TOO_LARGE | TOO_LARGE_1000,
CARRY | TOO_LARGE | TOO_LARGE_1000);
const simd8<uint8_t> byte_2_high = input.shr<4>().lookup_16<uint8_t>(
// ________ 0_______ <ASCII in byte 2>
TOO_SHORT, TOO_SHORT, TOO_SHORT, TOO_SHORT, TOO_SHORT, TOO_SHORT,
TOO_SHORT, TOO_SHORT,
// ________ 1000____
TOO_LONG | OVERLONG_2 | TWO_CONTS | OVERLONG_3 | TOO_LARGE_1000 |
OVERLONG_4,
// ________ 1001____
TOO_LONG | OVERLONG_2 | TWO_CONTS | OVERLONG_3 | TOO_LARGE,
// ________ 101_____
TOO_LONG | OVERLONG_2 | TWO_CONTS | SURROGATE | TOO_LARGE,
TOO_LONG | OVERLONG_2 | TWO_CONTS | SURROGATE | TOO_LARGE,
// ________ 11______
TOO_SHORT, TOO_SHORT, TOO_SHORT, TOO_SHORT);
return (byte_1_high & byte_1_low & byte_2_high);
}
simdutf_really_inline simd8<uint8_t>
check_multibyte_lengths(const simd8<uint8_t> input,
const simd8<uint8_t> prev_input,
const simd8<uint8_t> sc) {
simd8<uint8_t> prev2 = input.prev<2>(prev_input);
simd8<uint8_t> prev3 = input.prev<3>(prev_input);
simd8<uint8_t> must23 =
simd8<uint8_t>(must_be_2_3_continuation(prev2, prev3));
simd8<uint8_t> must23_80 = must23 & uint8_t(0x80);
return must23_80 ^ sc;
}
//
// Return nonzero if there are incomplete multibyte characters at the end of the
// block: e.g. if there is a 4-byte character, but it is 3 bytes from the end.
//
simdutf_really_inline simd8<uint8_t> is_incomplete(const simd8<uint8_t> input) {
// If the previous input's last 3 bytes match this, they're too short (they
// ended at EOF):
// ... 1111____ 111_____ 11______
static const uint8_t max_array[32] = {255,
255,
255,
255,
255,
255,
255,
255,
255,
255,
255,
255,
255,
255,
255,
255,
255,
255,
255,
255,
255,
255,
255,
255,
255,
255,
255,
255,
255,
0b11110000u - 1,
0b11100000u - 1,
0b11000000u - 1};
const simd8<uint8_t> max_value(
&max_array[sizeof(max_array) - sizeof(simd8<uint8_t>)]);
return input.gt_bits(max_value);
}
struct utf8_checker {
// If this is nonzero, there has been a UTF-8 error.
simd8<uint8_t> error;
// The last input we received
simd8<uint8_t> prev_input_block;
// Whether the last input we received was incomplete (used for ASCII fast
// path)
simd8<uint8_t> prev_incomplete;
//
// Check whether the current bytes are valid UTF-8.
//
simdutf_really_inline void check_utf8_bytes(const simd8<uint8_t> input,
const simd8<uint8_t> prev_input) {
// Flip prev1...prev3 so we can easily determine if they are 2+, 3+ or 4+
// lead bytes (2, 3, 4-byte leads become large positive numbers instead of
// small negative numbers)
simd8<uint8_t> prev1 = input.prev<1>(prev_input);
simd8<uint8_t> sc = check_special_cases(input, prev1);
this->error |= check_multibyte_lengths(input, prev_input, sc);
}
// The only problem that can happen at EOF is that a multibyte character is
// too short or a byte value too large in the last bytes: check_special_cases
// only checks for bytes too large in the first of two bytes.
simdutf_really_inline void check_eof() {
// If the previous block had incomplete UTF-8 characters at the end, an
// ASCII block can't possibly finish them.
this->error |= this->prev_incomplete;
}
simdutf_really_inline void check_next_input(const simd8x64<uint8_t> &input) {
if (simdutf_likely(is_ascii(input))) {
this->error |= this->prev_incomplete;
} else {
// you might think that a for-loop would work, but under Visual Studio, it
// is not good enough.
static_assert((simd8x64<uint8_t>::NUM_CHUNKS == 2) ||
(simd8x64<uint8_t>::NUM_CHUNKS == 4),
"We support either two or four chunks per 64-byte block.");
if (simd8x64<uint8_t>::NUM_CHUNKS == 2) {
this->check_utf8_bytes(input.chunks[0], this->prev_input_block);
this->check_utf8_bytes(input.chunks[1], input.chunks[0]);
} else if (simd8x64<uint8_t>::NUM_CHUNKS == 4) {
this->check_utf8_bytes(input.chunks[0], this->prev_input_block);
this->check_utf8_bytes(input.chunks[1], input.chunks[0]);
this->check_utf8_bytes(input.chunks[2], input.chunks[1]);
this->check_utf8_bytes(input.chunks[3], input.chunks[2]);
}
this->prev_incomplete =
is_incomplete(input.chunks[simd8x64<uint8_t>::NUM_CHUNKS - 1]);
this->prev_input_block = input.chunks[simd8x64<uint8_t>::NUM_CHUNKS - 1];
}
}
// do not forget to call check_eof!
simdutf_really_inline bool errors() const {
return this->error.any_bits_set_anywhere();
}
}; // struct utf8_checker
} // namespace utf8_validation
using utf8_validation::utf8_checker;
} // unnamed namespace
} // namespace SIMDUTF_IMPLEMENTATION
} // namespace simdutf
|
