Ada 4.1.1
Fast spec-compliant URL parser
Loading...
Searching...
No Matches
unicode_transcoding.cpp
Go to the documentation of this file.
2
3#include <algorithm>
4#include <cstdint>
5#include <cstring>
6
7namespace ada::idna {
8
9size_t utf8_to_utf32(const char* buf, size_t len, char32_t* utf32_output) {
10 const uint8_t* data = reinterpret_cast<const uint8_t*>(buf);
11 size_t pos = 0;
12 const char32_t* start{utf32_output};
13 while (pos < len) {
14 // try to convert the next block of 16 ASCII bytes
15 if (pos + 16 <= len) { // if it is safe to read 16 more
16 // bytes, check that they are ascii
17 uint64_t v1;
18 std::memcpy(&v1, data + pos, sizeof(uint64_t));
19 uint64_t v2;
20 std::memcpy(&v2, data + pos + sizeof(uint64_t), sizeof(uint64_t));
21 uint64_t v{v1 | v2};
22 if ((v & 0x8080808080808080) == 0) {
23 size_t final_pos = pos + 16;
24 while (pos < final_pos) {
25 *utf32_output++ = char32_t(buf[pos]);
26 pos++;
27 }
28 continue;
29 }
30 }
31 uint8_t leading_byte = data[pos]; // leading byte
32 if (leading_byte < 0b10000000) {
33 // converting one ASCII byte !!!
34 *utf32_output++ = char32_t(leading_byte);
35 pos++;
36 } else if ((leading_byte & 0b11100000) == 0b11000000) {
37 // We have a two-byte UTF-8
38 if (pos + 1 >= len) {
39 return 0;
40 } // minimal bound checking
41 if ((data[pos + 1] & 0b11000000) != 0b10000000) {
42 return 0;
43 }
44 // range check
45 uint32_t code_point =
46 (leading_byte & 0b00011111) << 6 | (data[pos + 1] & 0b00111111);
47 if (code_point < 0x80 || 0x7ff < code_point) {
48 return 0;
49 }
50 *utf32_output++ = char32_t(code_point);
51 pos += 2;
52 } else if ((leading_byte & 0b11110000) == 0b11100000) {
53 // We have a three-byte UTF-8
54 if (pos + 2 >= len) {
55 return 0;
56 } // minimal bound checking
57
58 if ((data[pos + 1] & 0b11000000) != 0b10000000) {
59 return 0;
60 }
61 if ((data[pos + 2] & 0b11000000) != 0b10000000) {
62 return 0;
63 }
64 // range check
65 uint32_t code_point = (leading_byte & 0b00001111) << 12 |
66 (data[pos + 1] & 0b00111111) << 6 |
67 (data[pos + 2] & 0b00111111);
68 if (code_point < 0x800 || 0xffff < code_point ||
69 (0xd7ff < code_point && code_point < 0xe000)) {
70 return 0;
71 }
72 *utf32_output++ = char32_t(code_point);
73 pos += 3;
74 } else if ((leading_byte & 0b11111000) == 0b11110000) { // 0b11110000
75 // we have a 4-byte UTF-8 word.
76 if (pos + 3 >= len) {
77 return 0;
78 } // minimal bound checking
79 if ((data[pos + 1] & 0b11000000) != 0b10000000) {
80 return 0;
81 }
82 if ((data[pos + 2] & 0b11000000) != 0b10000000) {
83 return 0;
84 }
85 if ((data[pos + 3] & 0b11000000) != 0b10000000) {
86 return 0;
87 }
88
89 // range check
90 uint32_t code_point = (leading_byte & 0b00000111) << 18 |
91 (data[pos + 1] & 0b00111111) << 12 |
92 (data[pos + 2] & 0b00111111) << 6 |
93 (data[pos + 3] & 0b00111111);
94 if (code_point <= 0xffff || 0x10ffff < code_point) {
95 return 0;
96 }
97 *utf32_output++ = char32_t(code_point);
98 pos += 4;
99 } else {
100 return 0;
101 }
102 }
103 return utf32_output - start;
104}
105
106size_t utf8_length_from_utf32(const char32_t* buf, size_t len) {
107 // We are not BOM aware.
108 const uint32_t* p = reinterpret_cast<const uint32_t*>(buf);
109 size_t counter{0};
110 for (size_t i = 0; i != len; ++i) {
111 ++counter; // ASCII
112 counter += static_cast<size_t>(p[i] > 0x7F); // two-byte
113 counter += static_cast<size_t>(p[i] > 0x7FF); // three-byte
114 counter += static_cast<size_t>(p[i] > 0xFFFF); // four-bytes
115 }
116 return counter;
117}
118
119size_t utf32_length_from_utf8(const char* buf, size_t len) {
120 const int8_t* p = reinterpret_cast<const int8_t*>(buf);
121 return std::count_if(p, std::next(p, len), [](int8_t c) {
122 // -65 is 0b10111111, anything larger in two-complement's
123 // should start a new code point.
124 return c > -65;
125 });
126}
127
128size_t utf32_to_utf8(const char32_t* buf, size_t len, char* utf8_output) {
129 const uint32_t* data = reinterpret_cast<const uint32_t*>(buf);
130 size_t pos = 0;
131 const char* start{utf8_output};
132 while (pos < len) {
133 // try to convert the next block of 2 ASCII characters
134 if (pos + 2 <= len) { // if it is safe to read 8 more
135 // bytes, check that they are ascii
136 uint64_t v;
137 std::memcpy(&v, data + pos, sizeof(uint64_t));
138 if ((v & 0xFFFFFF80FFFFFF80) == 0) {
139 *utf8_output++ = char(buf[pos]);
140 *utf8_output++ = char(buf[pos + 1]);
141 pos += 2;
142 continue;
143 }
144 }
145 uint32_t word = data[pos];
146 if ((word & 0xFFFFFF80) == 0) {
147 // will generate one UTF-8 bytes
148 *utf8_output++ = char(word);
149 pos++;
150 } else if ((word & 0xFFFFF800) == 0) {
151 // will generate two UTF-8 bytes
152 // we have 0b110XXXXX 0b10XXXXXX
153 *utf8_output++ = char((word >> 6) | 0b11000000);
154 *utf8_output++ = char((word & 0b111111) | 0b10000000);
155 pos++;
156 } else if ((word & 0xFFFF0000) == 0) {
157 // will generate three UTF-8 bytes
158 // we have 0b1110XXXX 0b10XXXXXX 0b10XXXXXX
159 if (word >= 0xD800 && word <= 0xDFFF) {
160 return 0;
161 }
162 *utf8_output++ = char((word >> 12) | 0b11100000);
163 *utf8_output++ = char(((word >> 6) & 0b111111) | 0b10000000);
164 *utf8_output++ = char((word & 0b111111) | 0b10000000);
165 pos++;
166 } else {
167 // will generate four UTF-8 bytes
168 // we have 0b11110XXX 0b10XXXXXX 0b10XXXXXX
169 // 0b10XXXXXX
170 if (word > 0x10FFFF) {
171 return 0;
172 }
173 *utf8_output++ = char((word >> 18) | 0b11110000);
174 *utf8_output++ = char(((word >> 12) & 0b111111) | 0b10000000);
175 *utf8_output++ = char(((word >> 6) & 0b111111) | 0b10000000);
176 *utf8_output++ = char((word & 0b111111) | 0b10000000);
177 pos++;
178 }
179 }
180 return utf8_output - start;
181}
182} // namespace ada::idna
size_t utf32_length_from_utf8(const char *buf, size_t len)
Definition ada_idna.cpp:142
size_t utf32_to_utf8(const char32_t *buf, size_t len, char *utf8_output)
Definition ada_idna.cpp:151
size_t utf8_length_from_utf32(const char32_t *buf, size_t len)
Definition ada_idna.cpp:129
size_t utf8_to_utf32(const char *buf, size_t len, char32_t *utf32_output)
Definition ada_idna.cpp:32