Ada
4.1.1
Fast spec-compliant URL parser
Toggle main menu visibility
Loading...
Searching...
No Matches
to_ascii_tests.cpp
Go to the documentation of this file.
1
#include <filesystem>
2
#include <fstream>
3
#include <sstream>
4
#include <string>
5
#include <vector>
6
7
#include "
ada/idna/punycode.h
"
8
#include "
ada/idna/to_ascii.h
"
9
#include "
ada/idna/unicode_transcoding.h
"
10
#include "gtest/gtest.h"
11
12
class
IdnaFileTest
:
public
::testing::Test {
13
protected
:
14
// Helper function to read a file
15
static
std::string
ReadFile
(
const
std::string& filename) {
16
constexpr
auto
read_size = std::size_t(4096);
17
auto
stream = std::ifstream(filename.c_str());
18
stream.exceptions(std::ios_base::badbit);
19
auto
out = std::string();
20
auto
buf = std::string(read_size,
'\0'
);
21
while
(stream.read(&buf[0], read_size)) {
22
out.append(buf, 0,
size_t
(stream.gcount()));
23
}
24
out.append(buf, 0,
size_t
(stream.gcount()));
25
return
out;
26
}
27
28
// Helper function to split a string by newlines
29
static
std::vector<std::string>
SplitString
(
const
std::string& str) {
30
auto
result = std::vector<std::string>{};
31
auto
ss = std::stringstream{str};
32
for
(std::string line; std::getline(ss, line,
'\n'
);) {
33
result.push_back(line);
34
}
35
return
result;
36
}
37
38
// Set default filenames that can be overridden in tests
39
std::string
alternating_filename
=
"fixtures/to_ascii_alternating.txt"
;
40
std::string
invalid_filename
=
"fixtures/to_ascii_invalid.txt"
;
41
};
42
43
// Test for valid ASCII conversions from file
44
TEST_F
(
IdnaFileTest
, AlternatingFileConversions) {
45
// Skip test if file doesn't exist
46
if
(!std::filesystem::exists(alternating_filename)) {
47
GTEST_SKIP() <<
"Test file not found: "
<< alternating_filename;
48
}
49
50
std::string buffer = ReadFile(alternating_filename);
51
std::vector<std::string> lines = SplitString(buffer);
52
53
ASSERT_GT(lines.size(), 1) <<
"File must contain at least 2 lines"
;
54
55
for
(
size_t
i = 0; i + 1 < lines.size(); i += 2) {
56
const
std::string& utf8_string = lines[i];
57
const
std::string& puny_string = lines[i + 1];
58
59
SCOPED_TRACE(
"Line "
+ std::to_string(i) +
": "
+ utf8_string);
60
61
std::string processed =
ada::idna::to_ascii
(utf8_string);
62
EXPECT_EQ(processed, puny_string)
63
<<
"Conversion failed for '"
<< utf8_string <<
"'"
64
<<
"\nExpected: "
<< puny_string <<
"\nGot: "
<< processed;
65
}
66
}
67
68
// Test for invalid conversions from file
69
TEST_F
(
IdnaFileTest
, InvalidFileConversions) {
70
// Skip test if file doesn't exist
71
if
(!std::filesystem::exists(invalid_filename)) {
72
GTEST_SKIP() <<
"Test file not found: "
<< invalid_filename;
73
}
74
75
std::string buffer = ReadFile(invalid_filename);
76
std::vector<std::string> lines = SplitString(buffer);
77
78
for
(
size_t
i = 0; i < lines.size(); i++) {
79
const
std::string& utf8_string = lines[i];
80
81
SCOPED_TRACE(
"Line "
+ std::to_string(i) +
": "
+ utf8_string);
82
83
EXPECT_TRUE(
ada::idna::to_ascii
(utf8_string).empty())
84
<<
"Should have failed for invalid input: "
<< utf8_string;
85
}
86
}
87
#define UNICODE16 0
88
TEST
(to_ascii_tests, special_cases) {
89
// Test case for Tangut ideograph U+17E68 (𗹨)
90
#if UNICODE16
91
// This test needs to be reviewed for Unicode 16.
92
ASSERT_TRUE(
ada::idna::to_ascii
(
"\xf0\xaf\xa1\xa8"
).empty())
93
<<
"Tangut ideograph should result in empty string"
;
94
#endif
95
// Test case for German capital sharp S (ẞ)
96
// We would prefer "\u1E9E" but Visual Studio complains.
97
ASSERT_EQ(
ada::idna::to_ascii
(
"\xe1\xba\x9e"
),
"xn--zca"
)
98
<<
"German capital sharp S should convert to expected Punycode"
;
99
100
// Test case for soft hyphen (U+00AD)
101
ASSERT_TRUE(
ada::idna::to_ascii
(
"\u00AD"
).empty())
102
<<
"Soft hyphen should result in empty string"
;
103
104
// Test case for replacement character (U+FFFD)
105
ASSERT_TRUE(
ada::idna::to_ascii
(
"\xef\xbf\xbd.com"
).empty())
106
<<
"Replacement character in domain should result in empty string"
;
107
}
108
109
TEST
(to_ascii_tests, nfc_reorders_precomposed_starter) {
110
// A precomposed starter followed by a combining mark of lower combining class
111
// is not in NFC: normalization decomposes the starter and reorders the marks.
112
// U+00E1 (a + acute, class 230) then U+0323 (dot below, class 220) normalizes
113
// to U+1EA1 (a + dot below) with the acute floating -> xn--lsa752l.
114
ASSERT_EQ(
ada::idna::to_ascii
(
"\xc3\xa1\xcc\xa3"
),
"xn--lsa752l"
)
115
<<
"acute+dot-below should reorder to NFC before punycode"
;
116
// U+015A (S + acute, class 230) then U+0327 (cedilla, class 202).
117
ASSERT_EQ(
ada::idna::to_ascii
(
"\xc5\x9a\xcc\xa7"
),
"xn--nga05f"
)
118
<<
"acute+cedilla should reorder to NFC before punycode"
;
119
}
120
121
TEST
(to_ascii_tests, comma_test) {
122
ASSERT_FALSE(
ada::idna::to_ascii
(
"128.0,0.1"
).empty());
123
}
124
125
// Unicode 17 test cases - CJK Extension J code points (U+323B0..U+33479)
126
TEST
(to_ascii_tests, unicode17_cjk_extension_j) {
127
// Test case: U+32931 () - CJK Extension J character
128
// Input: "20.音.ꡦ1." (U+32931, ASCII, U+97F3, U+A866)
129
ASSERT_EQ(
ada::idna::to_ascii
(
"\xf0\xb2\xa4\xb1"
130
"20.\xe9\x9f\xb3.\xea\xa1\xa6"
131
"1."
),
132
"xn--20-9802c.xn--0w5a.xn--1-eg4e."
)
133
<<
"CJK Extension J character should be valid in Unicode 17"
;
134
135
// Test case: Already punycode-encoded version should pass through
136
ASSERT_EQ(
ada::idna::to_ascii
(
"xn--20-9802c.xn--0w5a.xn--1-eg4e."
),
137
"xn--20-9802c.xn--0w5a.xn--1-eg4e."
)
138
<<
"Punycode version should remain unchanged"
;
139
140
// Test case: U+32B9A () - CJK Extension J with combining marks
141
// Input: "9ꍩ៓.ss" (U+32B9A, ASCII 9, U+A369, U+17D3)
142
ASSERT_EQ(
ada::idna::to_ascii
(
"\xf0\xb2\xae\x9a"
143
"9\xea\x8d\xa9\xe1\x9f\x93.ss"
),
144
"xn--9-i0j5967eg3qz.ss"
)
145
<<
"CJK Extension J with combining marks should convert correctly"
;
146
147
// Test case: Same as above but with uppercase SS - should normalize to
148
// lowercase
149
ASSERT_EQ(
ada::idna::to_ascii
(
"\xf0\xb2\xae\x9a"
150
"9\xea\x8d\xa9\xe1\x9f\x93.SS"
),
151
"xn--9-i0j5967eg3qz.ss"
)
152
<<
"Uppercase SS should normalize to lowercase ss"
;
153
}
154
155
// Unicode 18 test cases - Small Seal characters (U+3D000..U+3FC3F)
156
TEST
(to_ascii_tests, unicode18_small_seal) {
157
EXPECT_EQ(
ada::idna::to_ascii
(
"\U0003D000"
),
"xn--d47r"
);
158
EXPECT_EQ(
ada::idna::to_ascii
(
"\U0003FC3F"
),
"xn--zd7s"
);
159
// U+3FC40 is unassigned.
160
EXPECT_TRUE(
ada::idna::to_ascii
(
"\U0003FC40"
).empty());
161
// IdnaTestV2.txt 18.0.0: a Small Seal character in a multi-label domain.
162
EXPECT_EQ(
ada::idna::to_ascii
(
"\U0003E8AC.\u0729.\uCBD95"
),
163
"xn--t92s.xn--znb.xn--5-y88f"
);
164
}
165
166
// Regression test for https://github.com/whatwg/url/issues/803
167
// A label like U+33FF U+33FD followed by ASCII "xn--" encodes to a punycode
168
// payload starting with "xn--", but the decoded form begins with non-ASCII
169
// characters -- it is NOT a double-encoded ACE label and must be accepted.
170
TEST
(to_ascii_tests, mixed_label_xn_prefix_regression) {
171
// U+33FF = \xe3\x8f\xbf, U+33FD = \xe3\x8f\xbd in UTF-8
172
const
std::string input =
"\xe3\x8f\xbf\xe3\x8f\xbdxn--.example"
;
173
std::string first =
ada::idna::to_ascii
(input);
174
ASSERT_FALSE(first.empty()) <<
"Mixed IDNA label whose punycode starts with "
175
"'xn--' must not be rejected"
;
176
// Idempotency: to_ascii applied twice must give the same result.
177
std::string second =
ada::idna::to_ascii
(first);
178
ASSERT_EQ(first, second) <<
"to_ascii must be idempotent"
;
179
}
180
181
TEST
(to_ascii_tests, bidi_regression) {
182
// LTR label ending in RTL/AL/AN should fail (Rule 6)
183
EXPECT_TRUE(
ada::idna::to_ascii
(
"a\u05D0"
).empty())
184
<<
"LTR + RTL should fail"
;
185
EXPECT_TRUE(
ada::idna::to_ascii
(
"a\u0627"
).empty()) <<
"LTR + AL should fail"
;
186
EXPECT_TRUE(
ada::idna::to_ascii
(
"a\u0661"
).empty()) <<
"LTR + AN should fail"
;
187
188
// RTL + trailing NSM
189
EXPECT_TRUE(
ada::idna::to_ascii
(
"a\u05D0\u0301"
).empty())
190
<<
"RTL + trailing NSM should fail"
;
191
192
// NSM before RTL
193
EXPECT_TRUE(
ada::idna::to_ascii
(
"a\u0301\u05D0"
).empty())
194
<<
"NSM before RTL should fail"
;
195
196
// multiple RTL
197
EXPECT_TRUE(
ada::idna::to_ascii
(
"a\u05D0\u05D1"
).empty())
198
<<
"multiple RTL should fail"
;
199
200
// multi-label
201
EXPECT_TRUE(
ada::idna::to_ascii
(
"a\u05D0.b"
).empty())
202
<<
"multi-label should fail"
;
203
}
204
205
TEST
(to_ascii_tests, contextj_joiner_does_not_bypass_bidi) {
206
// A ZWNJ preceded by a virama satisfies the ContextJ rule, but that must not
207
// short-circuit the Bidi validity check for the rest of the label. Here the
208
// label starts with L (U+0915) so it is an LTR label, and the trailing AL
209
// (U+0627) violates Rule 5. It must be rejected.
210
EXPECT_TRUE(
ada::idna::to_ascii
(
"\u0915\u094D\u200C\u0627"
).empty())
211
<<
"valid ZWNJ must not bypass Bidi rule 5"
;
212
// Same with ZWJ (U+200D).
213
EXPECT_TRUE(
ada::idna::to_ascii
(
"\u0915\u094D\u200D\u0627"
).empty())
214
<<
"valid ZWJ must not bypass Bidi rule 5"
;
215
// A contextually valid joiner in an otherwise valid label still converts.
216
EXPECT_EQ(
ada::idna::to_ascii
(
"\u0915\u094D\u200C\u0916"
),
"xn--11bc8nw90g"
)
217
<<
"valid ZWNJ label must still convert"
;
218
}
219
220
TEST
(to_ascii_tests, contextj_checks_every_joiner_and_bidi_rule_1) {
221
// Ported from ada-url/ada#1276 and ada-url/ada#1277. A virama before a joiner
222
// settles only that joiner: later joiners in the label still need their own
223
// ContextJ check, and the RFC 5893 Bidi rule must still run on the label.
224
// 'a' (L) + virama + ZWNJ + ZWJ: the ZWJ has no virama before it.
225
EXPECT_TRUE(
ada::idna::to_ascii
(
"a\u094D\u200C\u200D.example"
).empty())
226
<<
"a joiner after a valid virama+ZWNJ must still be checked"
;
227
// '1' (EN) + U+05D0 (R) + virama + ZWNJ: the ZWNJ is valid, but Bidi rule 1
228
// requires the first character to be L, R or AL.
229
EXPECT_TRUE(
ada::idna::to_ascii
(
"1\u05D0\u094D\u200C.example"
).empty())
230
<<
"valid ZWNJ must not bypass Bidi rule 1"
;
231
// Devanagari KA + virama + ZWNJ + KA is valid.
232
EXPECT_EQ(
ada::idna::to_ascii
(
"\u0915\u094D\u200C\u0915.example"
),
233
"xn--11ba1ow90g.example"
)
234
<<
"valid ZWNJ label must still convert"
;
235
236
// Regression tests from ada-url/ada#1277:
237
// 1) U+0644 U+200C U+0073 U+06EF ("\u0644\u200Cs\u06EF"):
238
// ZWNJ is between U+0644 (LAM, D) and U+0073 ('s', U).
239
// Before is D, but after is U (not R or D), violating ContextJ.
240
EXPECT_TRUE(
ada::idna::to_ascii
(
"\u0644\u200Cs\u06EF"
).empty())
241
<<
"ZWNJ followed by latin 's' violates ContextJ"
;
242
// 2) U+0061 U+0644 U+200C U+06EF ("a\u0644\u200C\u06EF"):
243
// Label starts with 'a' (L), but contains U+0644 (AL) and U+06EF (AL),
244
// violating Bidi rule 5.
245
EXPECT_TRUE(
ada::idna::to_ascii
(
"a\u0644\u200C\u06EF"
).empty())
246
<<
"LTR label containing Arabic letters violates Bidi rule 5"
;
247
// 3) U+0627 U+0BCD U+200D ("\u0627\u0BCD\u200D"):
248
// U+0627 (AL) + U+0BCD (TAMIL SIGN VIRAMA) + U+200D (ZWJ).
249
// ZWJ preceded by virama satisfies ContextJ, but Bidi rule 3 requires
250
// the RTL label to end with R, AL, EN, or AN followed by zero or more NSM.
251
// ZWJ (U+200D) has Bidi property BN, not NSM/R/AL/EN/AN, so it violates
252
// Bidi rule 3.
253
EXPECT_TRUE(
ada::idna::to_ascii
(
"\u0627\u0BCD\u200D"
).empty())
254
<<
"RTL label ending with ZWJ violates Bidi rule 3"
;
255
256
// Hangul LV + T composition regression test from ada-url/idna#104:
257
// U+AC00 (가, LV) + U+11A8 (ᆨ, T) -> U+AC01 (각) under NFC.
258
// Must produce xn--p39a, not xn--rud7310f.
259
EXPECT_EQ(
ada::idna::to_ascii
(
"\uAC00\u11A8.example"
),
"xn--p39a.example"
);
260
}
261
262
TEST
(to_ascii_tests, bidi_domain_checks_every_label) {
263
// RFC 5893 applies to every label of a domain that contains an RTL label,
264
// including labels that have no R, AL, or AN character of their own.
265
// ada-url/ada#1277.
266
EXPECT_TRUE(
ada::idna::to_ascii
(
"1.\u0627"
).empty())
267
<<
"EN-leading label in a Bidi domain violates rule 1"
;
268
EXPECT_TRUE(
ada::idna::to_ascii
(
"\u0627.1a"
).empty())
269
<<
"EN-leading label in a Bidi domain violates rule 1"
;
270
EXPECT_TRUE(
ada::idna::to_ascii
(
"1a.\u0627"
).empty());
271
// An LTR label that starts with L and ends with L or EN is allowed.
272
EXPECT_EQ(
ada::idna::to_ascii
(
"example.\u0627"
),
"example.xn--mgb"
);
273
EXPECT_EQ(
ada::idna::to_ascii
(
"a1.\u0627"
),
"a1.xn--mgb"
);
274
EXPECT_EQ(
ada::idna::to_ascii
(
"\u0627.example"
),
"xn--mgb.example"
);
275
// Not a Bidi domain, so a leading digit is fine.
276
EXPECT_EQ(
ada::idna::to_ascii
(
"1.example"
),
"1.example"
);
277
// All-ASCII input, including punycode, stays on the WHATWG fast path.
278
EXPECT_EQ(
ada::idna::to_ascii
(
"1.xn--mgb"
),
"1.xn--mgb"
);
279
}
280
281
TEST
(to_ascii_tests, contextj_viramas_from_newer_unicode) {
282
// Any Canonical_Combining_Class=Virama (9) character allows a following
283
// ZWNJ or ZWJ (RFC 5892 Appendix A). Each label is letter + virama + joiner
284
// + letter.
285
struct
virama_case {
286
const
char
* zwnj_input;
287
const
char
* zwnj_output;
288
const
char
* zwj_input;
289
const
char
* zwj_output;
290
};
291
const
virama_case cases[] = {
292
// U+1715 TAGALOG SIGN PAMUDPOD
293
{
"\u1703\u1715\u200C\u1703"
,
"xn--xyea7c825c"
,
"\u1703\u1715\u200D\u1703"
,
294
"xn--xyea7cv35c"
},
295
// U+11070 BRAHMI SIGN OLD TAMIL VIRAMA
296
{
"\U00011013\U00011070\u200C\U00011013"
,
"xn--0ug4565gba48a"
,
297
"\U00011013\U00011070\u200D\U00011013"
,
"xn--1ug2565gba48a"
},
298
// U+113CE TULU-TIGALARI SIGN VIRAMA
299
{
"\U00011392\U000113CE\u200C\U00011392"
,
"xn--0ug0307gba2v"
,
300
"\U00011392\U000113CE\u200D\U00011392"
,
"xn--1ugy307gba2v"
},
301
// U+113CF TULU-TIGALARI SIGN LOOPED VIRAMA
302
{
"\U00011392\U000113CF\u200C\U00011392"
,
"xn--0ug0307gba6v"
,
303
"\U00011392\U000113CF\u200D\U00011392"
,
"xn--1ugy307gba6v"
},
304
// U+113D0 TULU-TIGALARI CONJOINER
305
{
"\U00011392\U000113D0\u200C\U00011392"
,
"xn--0ug0307gba0w"
,
306
"\U00011392\U000113D0\u200D\U00011392"
,
"xn--1ugy307gba0w"
},
307
// U+11F41 KAWI SIGN KILLER
308
{
"\U00011F12\U00011F41\u200C\U00011F12"
,
"xn--0ugz651hba0q"
,
309
"\U00011F12\U00011F41\u200D\U00011F12"
,
"xn--1ugx651hba0q"
},
310
// U+11F42 KAWI CONJOINER
311
{
"\U00011F12\U00011F42\u200C\U00011F12"
,
"xn--0ugz651hba4q"
,
312
"\U00011F12\U00011F42\u200D\U00011F12"
,
"xn--1ugx651hba4q"
},
313
// U+1612F GURUNG KHEMA SIGN THOLHOMA
314
{
"\U00016101\U0001612F\u200C\U00016101"
,
"xn--0ug9257jba6p"
,
315
"\U00016101\U0001612F\u200D\U00016101"
,
"xn--1ug7257jba6p"
},
316
};
317
for
(
const
virama_case& c : cases) {
318
EXPECT_EQ(
ada::idna::to_ascii
(c.zwnj_input), c.zwnj_output);
319
EXPECT_EQ(
ada::idna::to_ascii
(c.zwj_input), c.zwj_output);
320
}
321
}
322
323
TEST
(to_ascii_tests, contextj_joining_types) {
324
// ZWNJ between a Joining_Type L or D character and an R or D character.
325
EXPECT_EQ(
ada::idna::to_ascii
(
"\u07CA\u200C\u07CB"
),
"xn--lsbc047p"
)
326
<<
"N'Ko"
;
327
EXPECT_EQ(
ada::idna::to_ascii
(
"\U00010AC0\u200C\U00010AC1"
),
"xn--0ug8553gea"
)
328
<<
"Manichaean"
;
329
EXPECT_EQ(
ada::idna::to_ascii
(
"\U00010D01\u200C\U00010D02"
),
"xn--0ug5444gea"
)
330
<<
"Hanifi Rohingya"
;
331
EXPECT_EQ(
ada::idna::to_ascii
(
"\U00010F30\u200C\U00010F31"
),
"xn--0ug5035gea"
)
332
<<
"Sogdian"
;
333
EXPECT_EQ(
ada::idna::to_ascii
(
"\U00010F70\u200C\U00010F71"
),
"xn--0ug3045gea"
)
334
<<
"Old Uyghur"
;
335
EXPECT_EQ(
ada::idna::to_ascii
(
"\U00010FB0\u200C\U00010FB0"
),
"xn--0ug1055gba"
)
336
<<
"Chorasmian"
;
337
EXPECT_EQ(
ada::idna::to_ascii
(
"\U0001E922\u200C\U0001E923"
),
"xn--0ug1411pea"
)
338
<<
"Adlam"
;
339
EXPECT_EQ(
ada::idna::to_ascii
(
"\u0886\u200C\u0886"
),
"xn--yxba971p"
)
340
<<
"Arabic Extended-B"
;
341
// U+10ED9 ARABIC CROWN LETTER BEH (Unicode 18) is left-joining: it may come
342
// before a ZWNJ but not after one.
343
EXPECT_EQ(
ada::idna::to_ascii
(
"\U00010ED9\u200C\u0628"
),
"xn--ngb963k752o"
);
344
EXPECT_TRUE(
ada::idna::to_ascii
(
"\u0628\u200C\U00010ED9"
).empty());
345
}
346
347
TEST
(to_ascii_tests, contextj_transparent_characters) {
348
// Only Joining_Type T characters may come between the ZWNJ and the joining
349
// characters around it: (L|D) T* ZWNJ T* (R|D).
350
EXPECT_EQ(
ada::idna::to_ascii
(
"\u0628\u064E\u200C\u0628"
),
"xn--ngba7iz95i"
);
351
EXPECT_EQ(
ada::idna::to_ascii
(
"\u0628\u200C\u064E\u0628"
),
"xn--ngba7iy95i"
);
352
EXPECT_EQ(
ada::idna::to_ascii
(
"\u0628\u064E\u0651\u200C\u0628"
),
353
"xn--ngba7im9404a"
);
354
// U+0627 ALEF (R) directly before the ZWNJ.
355
EXPECT_TRUE(
ada::idna::to_ascii
(
"\u0628\u0627\u200C\u0628"
).empty());
356
// U+0621 HAMZA (U) directly after the ZWNJ.
357
EXPECT_TRUE(
ada::idna::to_ascii
(
"\u0628\u200C\u0621\u0628"
).empty());
358
}
359
360
// Helper: domain-to-ASCII as URL Standard callers use it (to_ascii + forbidden
361
// domain code point filter). Empty string means failure.
362
static
std::string
domain_to_ascii
(std::string_view input) {
363
std::string out =
ada::idna::to_ascii
(input);
364
if
(
ada::idna::contains_forbidden_domain_code_point
(out)) {
365
return
""
;
366
}
367
return
out;
368
}
369
370
// Regression tests for the WHATWG URL "domain to ASCII" web-compatibility
371
// carve-out: when the input domain is an ASCII string (beStrict = false), the
372
// result is the input lowercased, regardless of Unicode ToASCII's outcome. An
373
// ACE ("xn--") label may decode yet still fail IDNA validity criteria and must
374
// nevertheless be accepted as-is.
375
// See https://url.spec.whatwg.org/#concept-domain-to-ascii
376
//
377
// Cases below are drawn from:
378
// - WPT toascii.json (vendored as fixtures/toascii.json)
379
// - WPT IdnaTestV2.json carve-out flips (null -> pass-through)
380
// - WPT urltestdata.json hostnames that flipped failure -> success in Ada
381
// - WPT setters_tests.json host/hostname "xn--" acceptance
382
// - Ada URLPattern hostname unit tests (xn--a / XN--A)
383
TEST
(to_ascii_tests, ascii_xn_carveout) {
384
// --- Bare / empty ACE payload ---
385
// Bare ACE prefix: the Punycode payload is empty. Previously rejected.
386
// WPT setters_tests / urltestdata: host "xn--" is accepted.
387
EXPECT_EQ(
ada::idna::to_ascii
(
"xn--"
),
"xn--"
);
388
EXPECT_EQ(
domain_to_ascii
(
"xn--"
),
"xn--"
);
389
EXPECT_EQ(
ada::idna::to_ascii
(
"XN--"
),
"xn--"
);
390
// Trailing hyphen only in the ACE payload.
391
EXPECT_EQ(
ada::idna::to_ascii
(
"xn---"
),
"xn---"
);
392
393
// --- Invalid Punycode (decode failure) ---
394
// WPT toascii.json "Invalid Punycode": the ACE segment does not decode.
395
// Ada URLPattern hostname "xn--a" must be accepted as "xn--a".
396
EXPECT_EQ(
ada::idna::to_ascii
(
"xn--a"
),
"xn--a"
);
397
EXPECT_EQ(
domain_to_ascii
(
"xn--a"
),
"xn--a"
);
398
EXPECT_EQ(
ada::idna::to_ascii
(
"XN--A"
),
"xn--a"
);
399
EXPECT_EQ(
ada::idna::to_ascii
(
"Xn--a"
),
"xn--a"
);
400
// Same invalid ACE as a label of a larger domain (IdnaTestV2 V7 / toascii).
401
EXPECT_EQ(
ada::idna::to_ascii
(
"xn--a.pt"
),
"xn--a.pt"
);
402
EXPECT_EQ(
ada::idna::to_ascii
(
"xn--a.xn--zca"
),
"xn--a.xn--zca"
);
403
// Invalid ACE with trailing junk that remains ASCII (toascii.json).
404
EXPECT_EQ(
ada::idna::to_ascii
(
"xn--ls8h="
),
"xn--ls8h="
);
405
// Invalid ACE that is pure digits after prefix (IdnaTestV2 / toascii).
406
EXPECT_EQ(
ada::idna::to_ascii
(
"xn--0"
),
"xn--0"
);
407
EXPECT_EQ(
ada::idna::to_ascii
(
"xn--0.com"
),
"xn--0.com"
);
408
EXPECT_EQ(
ada::idna::to_ascii
(
"xn--0.pt"
),
"xn--0.pt"
);
409
410
// --- Decode succeeds but IDNA validity fails (ContextJ / mapped / etc.) ---
411
// ContextJ (C1): xn--ab-j1t decodes to "a\u200Cb" (ZWNJ not preceded by a
412
// virama). Previously rejected; now accepted as-is.
413
EXPECT_EQ(
ada::idna::to_ascii
(
"xn--ab-j1t"
),
"xn--ab-j1t"
);
414
// ContextJ (C2): ZWJ variant.
415
EXPECT_EQ(
ada::idna::to_ascii
(
"xn--ab-m1t"
),
"xn--ab-m1t"
);
416
// V1 / other validity failures that remain pure-ASCII ACE forms.
417
EXPECT_EQ(
ada::idna::to_ascii
(
"xn--u-ccb"
),
"xn--u-ccb"
);
418
EXPECT_EQ(
ada::idna::to_ascii
(
"xn--acom-0w1b"
),
"xn--acom-0w1b"
);
419
EXPECT_EQ(
ada::idna::to_ascii
(
"xn--a-ecp.ru"
),
"xn--a-ecp.ru"
);
420
EXPECT_EQ(
ada::idna::to_ascii
(
"xn--xn--a--gua.pt"
),
"xn--xn--a--gua.pt"
);
421
// Joiners ACE still accepted when input is pure ASCII (toascii CheckJoiners).
422
EXPECT_EQ(
ada::idna::to_ascii
(
"xn--1ug.example"
),
"xn--1ug.example"
);
423
EXPECT_EQ(
ada::idna::to_ascii
(
"xn--1ug"
),
"xn--1ug"
);
424
// Bidi ACE form accepted as ASCII (toascii CheckBidi counterpart).
425
EXPECT_EQ(
ada::idna::to_ascii
(
"xn--a-yoc"
),
"xn--a-yoc"
);
426
// U+FFFD encoded in Punycode is pure ASCII input -> accepted (toascii).
427
EXPECT_EQ(
ada::idna::to_ascii
(
"xn--zn7c.com"
),
"xn--zn7c.com"
);
428
429
// Decoded label has code points with "mapped" status (enclosed CJK), so the
430
// ACE form is non-canonical. Previously rejected; now accepted as-is.
431
// From Ada WPT urltestdata.json (failure -> success under carve-out).
432
EXPECT_EQ(
ada::idna::to_ascii
(
"a.b.c.xn--pokxncvks"
),
"a.b.c.xn--pokxncvks"
);
433
EXPECT_EQ(
ada::idna::to_ascii
(
"10.0.0.xn--pokxncvks"
),
434
"10.0.0.xn--pokxncvks"
);
435
436
// The URL spec's own example: xn--8i7caa decodes to fullwidth "www", whose
437
// code points have "mapped" status.
438
EXPECT_EQ(
ada::idna::to_ascii
(
"xn--8i7caa"
),
"xn--8i7caa"
);
439
440
// Double-encoded ACE prefix forms that remain pure ASCII (IdnaTestV2 flips).
441
EXPECT_EQ(
ada::idna::to_ascii
(
"xn--xn--bss-7z6ccid"
),
"xn--xn--bss-7z6ccid"
);
442
EXPECT_EQ(
ada::idna::to_ascii
(
"xn--xn---epa"
),
"xn--xn---epa"
);
443
EXPECT_EQ(
ada::idna::to_ascii
(
"xn--ASCII-"
),
"xn--ascii-"
);
444
EXPECT_EQ(
ada::idna::to_ascii
(
"xn--unicode-.org"
),
"xn--unicode-.org"
);
445
446
// Long ACE labels that previously failed validity (IdnaTestV2 C1/C2/A4).
447
EXPECT_EQ(
448
ada::idna::to_ascii
(
449
"1.xn--"
450
"assbcssssssssdssssssssssssssssessssssssssssssssssssxssssssssssss"
451
"ssssssssysssssssssssssssssz-pxq1419aa69989dba9gc"
),
452
"1.xn--"
453
"assbcssssssssdssssssssssssssssessssssssssssssssssssxssssssssssssssss"
454
"ssssysssssssssssssssssz-pxq1419aa69989dba9gc"
);
455
EXPECT_EQ(
456
ada::idna::to_ascii
(
457
"1.xn--abcdexyz-qyacaaabaaaaaaabaaaaaaaaabaaaaaaaaabaaaaaaaa010ze"
458
"2isb1140zba8cc"
),
459
"1.xn--abcdexyz-qyacaaabaaaaaaabaaaaaaaaabaaaaaaaaabaaaaaaaa010ze2isb"
460
"1140zba8cc"
);
461
462
// --- Case folding of ACE prefixes (urltestdata mixed-case hostnames) ---
463
EXPECT_EQ(
ada::idna::to_ascii
(
"a.b.c.XN--pokxncvks"
),
"a.b.c.xn--pokxncvks"
);
464
EXPECT_EQ(
ada::idna::to_ascii
(
"a.b.c.Xn--pokxncvks"
),
"a.b.c.xn--pokxncvks"
);
465
EXPECT_EQ(
ada::idna::to_ascii
(
"10.0.0.XN--pokxncvks"
),
466
"10.0.0.xn--pokxncvks"
);
467
EXPECT_EQ(
ada::idna::to_ascii
(
"10.0.0.xN--pokxncvks"
),
468
"10.0.0.xn--pokxncvks"
);
469
470
// --- Ordinary pure-ASCII domains ---
471
EXPECT_EQ(
ada::idna::to_ascii
(
"EXAMPLE.COM"
),
"example.com"
);
472
EXPECT_EQ(
ada::idna::to_ascii
(
"gOoGle.com"
),
"google.com"
);
473
// Labels with hyphens in 3rd/4th position (toascii.json, UseSTD3=false).
474
EXPECT_EQ(
ada::idna::to_ascii
(
"aa--"
),
"aa--"
);
475
EXPECT_EQ(
ada::idna::to_ascii
(
"ab--c"
),
"ab--c"
);
476
EXPECT_EQ(
ada::idna::to_ascii
(
"ab--cd.com"
),
"ab--cd.com"
);
477
// Leading / trailing hyphen labels (toascii.json).
478
EXPECT_EQ(
ada::idna::to_ascii
(
"-x"
),
"-x"
);
479
EXPECT_EQ(
ada::idna::to_ascii
(
"-foo.bar.com"
),
"-foo.bar.com"
);
480
EXPECT_EQ(
ada::idna::to_ascii
(
"foo-.bar.com"
),
"foo-.bar.com"
);
481
// Empty labels between dots (toascii.json).
482
EXPECT_EQ(
ada::idna::to_ascii
(
"x..xn--zca"
),
"x..xn--zca"
);
483
EXPECT_EQ(
ada::idna::to_ascii
(
"-x.xn--zca"
),
"-x.xn--zca"
);
484
EXPECT_EQ(
ada::idna::to_ascii
(
"x-.xn--zca"
),
"x-.xn--zca"
);
485
EXPECT_EQ(
ada::idna::to_ascii
(
"ab--c.xn--zca"
),
"ab--c.xn--zca"
);
486
EXPECT_EQ(
ada::idna::to_ascii
(
"xn--zca.xn--zca"
),
"xn--zca.xn--zca"
);
487
488
// Overlong labels / domains (toascii.json): still pure ASCII -> accepted.
489
EXPECT_EQ(
490
ada::idna::to_ascii
(
491
"x01234567890123456789012345678901234567890123456789012345678901"
492
"x"
),
493
"x01234567890123456789012345678901234567890123456789012345678901x"
);
494
EXPECT_EQ(
495
ada::idna::to_ascii
(
496
"x01234567890123456789012345678901234567890123456789012345678901x."
497
"xn--zca"
),
498
"x01234567890123456789012345678901234567890123456789012345678901x.xn--"
499
"zca"
);
500
501
// --- Idempotency ---
502
const
std::string once =
ada::idna::to_ascii
(
"xn--ab-j1t"
);
503
EXPECT_EQ(
ada::idna::to_ascii
(once), once);
504
EXPECT_EQ(
ada::idna::to_ascii
(
"xn--a"
),
"xn--a"
);
505
EXPECT_EQ(
ada::idna::to_ascii
(
ada::idna::to_ascii
(
"a.b.c.XN--pokxncvks"
)),
506
"a.b.c.xn--pokxncvks"
);
507
EXPECT_EQ(
ada::idna::to_ascii
(
ada::idna::to_ascii
(
"XN--"
)),
"xn--"
);
508
}
509
510
// Table-driven mirror of the Ada urltestdata / setters carve-out hostnames:
511
// every entry was previously failure (or rejected by set_host) and must now
512
// round-trip through to_ascii as a lowercased ASCII domain.
513
TEST
(to_ascii_tests, ada_wpt_urltestdata_carveout_hosts) {
514
static
constexpr
std::pair<const char*, const char*> kCases[] = {
515
// urltestdata.json: mapped-status ACE labels
516
{
"a.b.c.xn--pokxncvks"
,
"a.b.c.xn--pokxncvks"
},
517
{
"10.0.0.xn--pokxncvks"
,
"10.0.0.xn--pokxncvks"
},
518
{
"a.b.c.XN--pokxncvks"
,
"a.b.c.xn--pokxncvks"
},
519
{
"a.b.c.Xn--pokxncvks"
,
"a.b.c.xn--pokxncvks"
},
520
{
"10.0.0.XN--pokxncvks"
,
"10.0.0.xn--pokxncvks"
},
521
{
"10.0.0.xN--pokxncvks"
,
"10.0.0.xn--pokxncvks"
},
522
// urltestdata.json / setters_tests.json: bare xn--
523
{
"xn--"
,
"xn--"
},
524
{
"XN--"
,
"xn--"
},
525
// file:// and https:// hosts that flipped failure -> success
526
{
"xn--"
,
"xn--"
},
527
};
528
for
(
const
auto
& [input, expected] : kCases) {
529
SCOPED_TRACE(input);
530
EXPECT_EQ(
domain_to_ascii
(input), expected);
531
// Second application is a no-op (already lowercased ASCII).
532
EXPECT_EQ(
domain_to_ascii
(
domain_to_ascii
(input)), expected);
533
}
534
}
535
536
// The ASCII carve-out must NOT relax validation for non-ASCII inputs: those go
537
// through full UTS#46 validation. Contrast xn--ab-j1t (ASCII, accepted above)
538
// with its decoded form "a\u200Cb" supplied directly (non-ASCII, rejected).
539
// Also covers toascii.json cases that remain failures for non-ASCII input.
540
TEST
(to_ascii_tests, non_ascii_inputs_still_validated) {
541
// ZWNJ (U+200C) without a preceding virama: ContextJ (C1) violation.
542
EXPECT_TRUE(
ada::idna::to_ascii
(
"a\u200Cb"
).empty())
543
<<
"non-ASCII ContextJ violation should still fail"
;
544
545
// LTR label containing an RTL code point: Bidi rule violation.
546
EXPECT_TRUE(
ada::idna::to_ascii
(
"a\u05D0"
).empty())
547
<<
"non-ASCII Bidi violation should still fail"
;
548
549
// toascii.json: CheckJoiners is true — bare ZWJ label fails.
550
EXPECT_TRUE(
ada::idna::to_ascii
(
"\u200D.example"
).empty())
551
<<
"non-ASCII ZWJ joiner should still fail"
;
552
553
// toascii.json: CheckBidi is true — Arabic + Latin fails.
554
EXPECT_TRUE(
ada::idna::to_ascii
(
"يa"
).empty())
555
<<
"non-ASCII Bidi (Arabic+Latin) should still fail"
;
556
557
// toascii.json: U+FFFD literal fails; contrast ACE form xn--zn7c.com above.
558
EXPECT_TRUE(
ada::idna::to_ascii
(
"\uFFFD.com"
).empty())
559
<<
"U+FFFD in domain should still fail"
;
560
561
// toascii.json: Invalid Punycode containing non-ASCII (xn--tešla) fails.
562
// UTF-8 for "xn--tešla" (š = U+0161 = C5 A1).
563
EXPECT_TRUE(
ada::idna::to_ascii
(
"xn--te\xc5\xa1la"
).empty())
564
<<
"invalid punycode with non-ASCII must not use the ASCII carve-out"
;
565
566
// toascii.json: xn--a.ß mixes invalid ACE with non-ASCII sharp-s -> fail.
567
EXPECT_TRUE(
ada::idna::to_ascii
(
"xn--a.\xc3\x9f"
).empty())
568
<<
"mixed invalid ACE + non-ASCII label should still fail"
;
569
}
570
571
// The combining mark, Bidi and normalization tables must cover the same
572
// Unicode version as the mapping table (ada-url/idna#97).
573
TEST
(to_ascii_tests, leading_combining_mark_from_newer_unicode) {
574
// A label must not begin with a combining mark (General_Category M).
575
EXPECT_TRUE(
ada::idna::to_ascii
(
"\u0301a"
).empty()) <<
"Unicode 1.1 mark"
;
576
EXPECT_TRUE(
ada::idna::to_ascii
(
"\u08CBa"
).empty()) <<
"Unicode 14 mark"
;
577
EXPECT_TRUE(
ada::idna::to_ascii
(
"\u0897a"
).empty()) <<
"Unicode 16 mark"
;
578
EXPECT_TRUE(
ada::idna::to_ascii
(
"\u1AE0a"
).empty()) <<
"Unicode 17 mark"
;
579
EXPECT_TRUE(
ada::idna::to_ascii
(
"\u1ADEa"
).empty()) <<
"Unicode 18 mark"
;
580
// The same mark after a base character is valid.
581
EXPECT_EQ(
ada::idna::to_ascii
(
"a\u08CB"
),
"xn--a-mqd"
);
582
EXPECT_EQ(
ada::idna::to_ascii
(
"a\u1ADE"
),
"xn--a-09k"
);
583
}
584
585
TEST
(to_ascii_tests, rtl_characters_from_newer_unicode) {
586
// An RTL label must not contain an L character such as 'a'.
587
EXPECT_TRUE(
ada::idna::to_ascii
(
"\u0870a"
).empty()) <<
"Unicode 14 AL"
;
588
EXPECT_TRUE(
ada::idna::to_ascii
(
"\U00010D4Aa"
).empty()) <<
"Unicode 16 R"
;
589
EXPECT_TRUE(
ada::idna::to_ascii
(
"\U00010940a"
).empty()) <<
"Unicode 17 R"
;
590
EXPECT_TRUE(
ada::idna::to_ascii
(
"\U00010ED9a"
).empty()) <<
"Unicode 18 AL"
;
591
// An LTR label must not contain an R or AL character.
592
EXPECT_TRUE(
ada::idna::to_ascii
(
"a\U00010940"
).empty()) <<
"Unicode 17 R"
;
593
EXPECT_TRUE(
ada::idna::to_ascii
(
"a\U00010ED9"
).empty()) <<
"Unicode 18 AL"
;
594
// RTL labels made of these characters are valid.
595
EXPECT_EQ(
ada::idna::to_ascii
(
"\u0870\u0871"
),
"xn--cxbc"
);
596
EXPECT_EQ(
ada::idna::to_ascii
(
"\U00010D4A\U00010D4B"
),
"xn--9f0dc"
);
597
EXPECT_EQ(
ada::idna::to_ascii
(
"\U00010940\U00010941"
),
"xn--ql9cc"
);
598
EXPECT_EQ(
ada::idna::to_ascii
(
"\U00010ED9\U00010EDA"
),
"xn--or0dc"
);
599
}
600
601
TEST
(to_ascii_tests, bidi_class_changes_in_unicode18) {
602
// Unicode 18 changes U+1B3A BALINESE VOWEL SIGN RA REPA from NSM to L, so it
603
// may no longer follow an R character in an RTL label.
604
EXPECT_TRUE(
ada::idna::to_ascii
(
"\u05D0\u1B3A"
).empty());
605
EXPECT_EQ(
ada::idna::to_ascii
(
"\u1B13\u1B3A"
),
"xn--ntf8e"
);
606
}
607
608
TEST
(to_ascii_tests, unicode18_canonical_ordering) {
609
// U+05C8 (ccc 10) and U+05C9 (ccc 21) are new Hebrew points; NFC sorts them
610
// by combining class.
611
EXPECT_EQ(
ada::idna::to_ascii
(
"\u05D0\u05C8\u05C9"
),
"xn--wdbct"
);
612
EXPECT_EQ(
ada::idna::to_ascii
(
"\u05D0\u05C9\u05C8"
),
"xn--wdbct"
);
613
}
614
615
TEST
(to_ascii_tests, unicode16_canonical_composition) {
616
// U+1611E U+1611E composes to U+16121 GURUNG KHEMA VOWEL SIGN U, so both
617
// spellings are the same host.
618
EXPECT_EQ(
ada::idna::to_ascii
(
"a\U0001611E\U0001611E"
),
"xn--a-rn1m"
);
619
EXPECT_EQ(
ada::idna::to_ascii
(
"a\U00016121"
),
"xn--a-rn1m"
);
620
// U+1611E U+16121 is U+1611E U+1611E U+1611E, which normalizes to
621
// U+16121 U+1611E.
622
EXPECT_EQ(
ada::idna::to_ascii
(
"a\U0001611E\U00016121"
),
"xn--a-ln1mia"
);
623
EXPECT_EQ(
ada::idna::to_ascii
(
"a\U00016121\U0001611E"
),
"xn--a-ln1mia"
);
624
// U+16D68 is U+16D67 U+16D67: U+16D63 U+16D68 normalizes to U+16D6A.
625
EXPECT_EQ(
ada::idna::to_ascii
(
"a\U00016D63\U00016D68"
),
"xn--a-hs6m"
);
626
EXPECT_EQ(
ada::idna::to_ascii
(
"a\U00016D6A"
),
"xn--a-hs6m"
);
627
}
IdnaFileTest
Definition
to_ascii_tests.cpp:12
IdnaFileTest::invalid_filename
std::string invalid_filename
Definition
to_ascii_tests.cpp:40
IdnaFileTest::ReadFile
static std::string ReadFile(const std::string &filename)
Definition
to_ascii_tests.cpp:15
IdnaFileTest::alternating_filename
std::string alternating_filename
Definition
to_ascii_tests.cpp:39
IdnaFileTest::SplitString
static std::vector< std::string > SplitString(const std::string &str)
Definition
to_ascii_tests.cpp:29
ada::idna::to_ascii
std::string to_ascii(std::string_view ut8_string)
Definition
ada_idna.cpp:6695
ada::idna::contains_forbidden_domain_code_point
bool contains_forbidden_domain_code_point(std::string_view ascii_string)
Definition
ada_idna.cpp:6481
punycode.h
to_ascii.h
TEST
TEST(to_ascii_tests, special_cases)
Definition
to_ascii_tests.cpp:88
TEST_F
TEST_F(IdnaFileTest, AlternatingFileConversions)
Definition
to_ascii_tests.cpp:44
domain_to_ascii
static std::string domain_to_ascii(std::string_view input)
Definition
to_ascii_tests.cpp:362
unicode_transcoding.h