Ada 4.1.1
Fast spec-compliant URL parser
Loading...
Searching...
No Matches
to_ascii_tests.cpp
Go to the documentation of this file.
1#include <filesystem>
2#include <fstream>
3#include <sstream>
4#include <string>
5#include <vector>
6
7#include "ada/idna/punycode.h"
8#include "ada/idna/to_ascii.h"
10#include "gtest/gtest.h"
11
12class IdnaFileTest : public ::testing::Test {
13 protected:
14 // Helper function to read a file
15 static std::string ReadFile(const std::string& filename) {
16 constexpr auto read_size = std::size_t(4096);
17 auto stream = std::ifstream(filename.c_str());
18 stream.exceptions(std::ios_base::badbit);
19 auto out = std::string();
20 auto buf = std::string(read_size, '\0');
21 while (stream.read(&buf[0], read_size)) {
22 out.append(buf, 0, size_t(stream.gcount()));
23 }
24 out.append(buf, 0, size_t(stream.gcount()));
25 return out;
26 }
27
28 // Helper function to split a string by newlines
29 static std::vector<std::string> SplitString(const std::string& str) {
30 auto result = std::vector<std::string>{};
31 auto ss = std::stringstream{str};
32 for (std::string line; std::getline(ss, line, '\n');) {
33 result.push_back(line);
34 }
35 return result;
36 }
37
38 // Set default filenames that can be overridden in tests
39 std::string alternating_filename = "fixtures/to_ascii_alternating.txt";
40 std::string invalid_filename = "fixtures/to_ascii_invalid.txt";
41};
42
43// Test for valid ASCII conversions from file
44TEST_F(IdnaFileTest, AlternatingFileConversions) {
45 // Skip test if file doesn't exist
46 if (!std::filesystem::exists(alternating_filename)) {
47 GTEST_SKIP() << "Test file not found: " << alternating_filename;
48 }
49
50 std::string buffer = ReadFile(alternating_filename);
51 std::vector<std::string> lines = SplitString(buffer);
52
53 ASSERT_GT(lines.size(), 1) << "File must contain at least 2 lines";
54
55 for (size_t i = 0; i + 1 < lines.size(); i += 2) {
56 const std::string& utf8_string = lines[i];
57 const std::string& puny_string = lines[i + 1];
58
59 SCOPED_TRACE("Line " + std::to_string(i) + ": " + utf8_string);
60
61 std::string processed = ada::idna::to_ascii(utf8_string);
62 EXPECT_EQ(processed, puny_string)
63 << "Conversion failed for '" << utf8_string << "'"
64 << "\nExpected: " << puny_string << "\nGot: " << processed;
65 }
66}
67
68// Test for invalid conversions from file
69TEST_F(IdnaFileTest, InvalidFileConversions) {
70 // Skip test if file doesn't exist
71 if (!std::filesystem::exists(invalid_filename)) {
72 GTEST_SKIP() << "Test file not found: " << invalid_filename;
73 }
74
75 std::string buffer = ReadFile(invalid_filename);
76 std::vector<std::string> lines = SplitString(buffer);
77
78 for (size_t i = 0; i < lines.size(); i++) {
79 const std::string& utf8_string = lines[i];
80
81 SCOPED_TRACE("Line " + std::to_string(i) + ": " + utf8_string);
82
83 EXPECT_TRUE(ada::idna::to_ascii(utf8_string).empty())
84 << "Should have failed for invalid input: " << utf8_string;
85 }
86}
87#define UNICODE16 0
88TEST(to_ascii_tests, special_cases) {
89 // Test case for Tangut ideograph U+17E68 (𗹨)
90#if UNICODE16
91 // This test needs to be reviewed for Unicode 16.
92 ASSERT_TRUE(ada::idna::to_ascii("\xf0\xaf\xa1\xa8").empty())
93 << "Tangut ideograph should result in empty string";
94#endif
95 // Test case for German capital sharp S (ẞ)
96 // We would prefer "\u1E9E" but Visual Studio complains.
97 ASSERT_EQ(ada::idna::to_ascii("\xe1\xba\x9e"), "xn--zca")
98 << "German capital sharp S should convert to expected Punycode";
99
100 // Test case for soft hyphen (U+00AD)
101 ASSERT_TRUE(ada::idna::to_ascii("\u00AD").empty())
102 << "Soft hyphen should result in empty string";
103
104 // Test case for replacement character (U+FFFD)
105 ASSERT_TRUE(ada::idna::to_ascii("\xef\xbf\xbd.com").empty())
106 << "Replacement character in domain should result in empty string";
107}
108
109TEST(to_ascii_tests, nfc_reorders_precomposed_starter) {
110 // A precomposed starter followed by a combining mark of lower combining class
111 // is not in NFC: normalization decomposes the starter and reorders the marks.
112 // U+00E1 (a + acute, class 230) then U+0323 (dot below, class 220) normalizes
113 // to U+1EA1 (a + dot below) with the acute floating -> xn--lsa752l.
114 ASSERT_EQ(ada::idna::to_ascii("\xc3\xa1\xcc\xa3"), "xn--lsa752l")
115 << "acute+dot-below should reorder to NFC before punycode";
116 // U+015A (S + acute, class 230) then U+0327 (cedilla, class 202).
117 ASSERT_EQ(ada::idna::to_ascii("\xc5\x9a\xcc\xa7"), "xn--nga05f")
118 << "acute+cedilla should reorder to NFC before punycode";
119}
120
121TEST(to_ascii_tests, comma_test) {
122 ASSERT_FALSE(ada::idna::to_ascii("128.0,0.1").empty());
123}
124
125// Unicode 17 test cases - CJK Extension J code points (U+323B0..U+33479)
126TEST(to_ascii_tests, unicode17_cjk_extension_j) {
127 // Test case: U+32931 (𲤱) - CJK Extension J character
128 // Input: "𲤱20.音.ꡦ1." (U+32931, ASCII, U+97F3, U+A866)
129 ASSERT_EQ(ada::idna::to_ascii("\xf0\xb2\xa4\xb1"
130 "20.\xe9\x9f\xb3.\xea\xa1\xa6"
131 "1."),
132 "xn--20-9802c.xn--0w5a.xn--1-eg4e.")
133 << "CJK Extension J character should be valid in Unicode 17";
134
135 // Test case: Already punycode-encoded version should pass through
136 ASSERT_EQ(ada::idna::to_ascii("xn--20-9802c.xn--0w5a.xn--1-eg4e."),
137 "xn--20-9802c.xn--0w5a.xn--1-eg4e.")
138 << "Punycode version should remain unchanged";
139
140 // Test case: U+32B9A (𲮚) - CJK Extension J with combining marks
141 // Input: "𲮚9ꍩ៓.ss" (U+32B9A, ASCII 9, U+A369, U+17D3)
142 ASSERT_EQ(ada::idna::to_ascii("\xf0\xb2\xae\x9a"
143 "9\xea\x8d\xa9\xe1\x9f\x93.ss"),
144 "xn--9-i0j5967eg3qz.ss")
145 << "CJK Extension J with combining marks should convert correctly";
146
147 // Test case: Same as above but with uppercase SS - should normalize to
148 // lowercase
149 ASSERT_EQ(ada::idna::to_ascii("\xf0\xb2\xae\x9a"
150 "9\xea\x8d\xa9\xe1\x9f\x93.SS"),
151 "xn--9-i0j5967eg3qz.ss")
152 << "Uppercase SS should normalize to lowercase ss";
153}
154
155// Unicode 18 test cases - Small Seal characters (U+3D000..U+3FC3F)
156TEST(to_ascii_tests, unicode18_small_seal) {
157 EXPECT_EQ(ada::idna::to_ascii("\U0003D000"), "xn--d47r");
158 EXPECT_EQ(ada::idna::to_ascii("\U0003FC3F"), "xn--zd7s");
159 // U+3FC40 is unassigned.
160 EXPECT_TRUE(ada::idna::to_ascii("\U0003FC40").empty());
161 // IdnaTestV2.txt 18.0.0: a Small Seal character in a multi-label domain.
162 EXPECT_EQ(ada::idna::to_ascii("\U0003E8AC.\u0729.\uCBD95"),
163 "xn--t92s.xn--znb.xn--5-y88f");
164}
165
166// Regression test for https://github.com/whatwg/url/issues/803
167// A label like U+33FF U+33FD followed by ASCII "xn--" encodes to a punycode
168// payload starting with "xn--", but the decoded form begins with non-ASCII
169// characters -- it is NOT a double-encoded ACE label and must be accepted.
170TEST(to_ascii_tests, mixed_label_xn_prefix_regression) {
171 // U+33FF = \xe3\x8f\xbf, U+33FD = \xe3\x8f\xbd in UTF-8
172 const std::string input = "\xe3\x8f\xbf\xe3\x8f\xbdxn--.example";
173 std::string first = ada::idna::to_ascii(input);
174 ASSERT_FALSE(first.empty()) << "Mixed IDNA label whose punycode starts with "
175 "'xn--' must not be rejected";
176 // Idempotency: to_ascii applied twice must give the same result.
177 std::string second = ada::idna::to_ascii(first);
178 ASSERT_EQ(first, second) << "to_ascii must be idempotent";
179}
180
181TEST(to_ascii_tests, bidi_regression) {
182 // LTR label ending in RTL/AL/AN should fail (Rule 6)
183 EXPECT_TRUE(ada::idna::to_ascii("a\u05D0").empty())
184 << "LTR + RTL should fail";
185 EXPECT_TRUE(ada::idna::to_ascii("a\u0627").empty()) << "LTR + AL should fail";
186 EXPECT_TRUE(ada::idna::to_ascii("a\u0661").empty()) << "LTR + AN should fail";
187
188 // RTL + trailing NSM
189 EXPECT_TRUE(ada::idna::to_ascii("a\u05D0\u0301").empty())
190 << "RTL + trailing NSM should fail";
191
192 // NSM before RTL
193 EXPECT_TRUE(ada::idna::to_ascii("a\u0301\u05D0").empty())
194 << "NSM before RTL should fail";
195
196 // multiple RTL
197 EXPECT_TRUE(ada::idna::to_ascii("a\u05D0\u05D1").empty())
198 << "multiple RTL should fail";
199
200 // multi-label
201 EXPECT_TRUE(ada::idna::to_ascii("a\u05D0.b").empty())
202 << "multi-label should fail";
203}
204
205TEST(to_ascii_tests, contextj_joiner_does_not_bypass_bidi) {
206 // A ZWNJ preceded by a virama satisfies the ContextJ rule, but that must not
207 // short-circuit the Bidi validity check for the rest of the label. Here the
208 // label starts with L (U+0915) so it is an LTR label, and the trailing AL
209 // (U+0627) violates Rule 5. It must be rejected.
210 EXPECT_TRUE(ada::idna::to_ascii("\u0915\u094D\u200C\u0627").empty())
211 << "valid ZWNJ must not bypass Bidi rule 5";
212 // Same with ZWJ (U+200D).
213 EXPECT_TRUE(ada::idna::to_ascii("\u0915\u094D\u200D\u0627").empty())
214 << "valid ZWJ must not bypass Bidi rule 5";
215 // A contextually valid joiner in an otherwise valid label still converts.
216 EXPECT_EQ(ada::idna::to_ascii("\u0915\u094D\u200C\u0916"), "xn--11bc8nw90g")
217 << "valid ZWNJ label must still convert";
218}
219
220TEST(to_ascii_tests, contextj_checks_every_joiner_and_bidi_rule_1) {
221 // Ported from ada-url/ada#1276 and ada-url/ada#1277. A virama before a joiner
222 // settles only that joiner: later joiners in the label still need their own
223 // ContextJ check, and the RFC 5893 Bidi rule must still run on the label.
224 // 'a' (L) + virama + ZWNJ + ZWJ: the ZWJ has no virama before it.
225 EXPECT_TRUE(ada::idna::to_ascii("a\u094D\u200C\u200D.example").empty())
226 << "a joiner after a valid virama+ZWNJ must still be checked";
227 // '1' (EN) + U+05D0 (R) + virama + ZWNJ: the ZWNJ is valid, but Bidi rule 1
228 // requires the first character to be L, R or AL.
229 EXPECT_TRUE(ada::idna::to_ascii("1\u05D0\u094D\u200C.example").empty())
230 << "valid ZWNJ must not bypass Bidi rule 1";
231 // Devanagari KA + virama + ZWNJ + KA is valid.
232 EXPECT_EQ(ada::idna::to_ascii("\u0915\u094D\u200C\u0915.example"),
233 "xn--11ba1ow90g.example")
234 << "valid ZWNJ label must still convert";
235
236 // Regression tests from ada-url/ada#1277:
237 // 1) U+0644 U+200C U+0073 U+06EF ("\u0644\u200Cs\u06EF"):
238 // ZWNJ is between U+0644 (LAM, D) and U+0073 ('s', U).
239 // Before is D, but after is U (not R or D), violating ContextJ.
240 EXPECT_TRUE(ada::idna::to_ascii("\u0644\u200Cs\u06EF").empty())
241 << "ZWNJ followed by latin 's' violates ContextJ";
242 // 2) U+0061 U+0644 U+200C U+06EF ("a\u0644\u200C\u06EF"):
243 // Label starts with 'a' (L), but contains U+0644 (AL) and U+06EF (AL),
244 // violating Bidi rule 5.
245 EXPECT_TRUE(ada::idna::to_ascii("a\u0644\u200C\u06EF").empty())
246 << "LTR label containing Arabic letters violates Bidi rule 5";
247 // 3) U+0627 U+0BCD U+200D ("\u0627\u0BCD\u200D"):
248 // U+0627 (AL) + U+0BCD (TAMIL SIGN VIRAMA) + U+200D (ZWJ).
249 // ZWJ preceded by virama satisfies ContextJ, but Bidi rule 3 requires
250 // the RTL label to end with R, AL, EN, or AN followed by zero or more NSM.
251 // ZWJ (U+200D) has Bidi property BN, not NSM/R/AL/EN/AN, so it violates
252 // Bidi rule 3.
253 EXPECT_TRUE(ada::idna::to_ascii("\u0627\u0BCD\u200D").empty())
254 << "RTL label ending with ZWJ violates Bidi rule 3";
255
256 // Hangul LV + T composition regression test from ada-url/idna#104:
257 // U+AC00 (가, LV) + U+11A8 (ᆨ, T) -> U+AC01 (각) under NFC.
258 // Must produce xn--p39a, not xn--rud7310f.
259 EXPECT_EQ(ada::idna::to_ascii("\uAC00\u11A8.example"), "xn--p39a.example");
260}
261
262TEST(to_ascii_tests, bidi_domain_checks_every_label) {
263 // RFC 5893 applies to every label of a domain that contains an RTL label,
264 // including labels that have no R, AL, or AN character of their own.
265 // ada-url/ada#1277.
266 EXPECT_TRUE(ada::idna::to_ascii("1.\u0627").empty())
267 << "EN-leading label in a Bidi domain violates rule 1";
268 EXPECT_TRUE(ada::idna::to_ascii("\u0627.1a").empty())
269 << "EN-leading label in a Bidi domain violates rule 1";
270 EXPECT_TRUE(ada::idna::to_ascii("1a.\u0627").empty());
271 // An LTR label that starts with L and ends with L or EN is allowed.
272 EXPECT_EQ(ada::idna::to_ascii("example.\u0627"), "example.xn--mgb");
273 EXPECT_EQ(ada::idna::to_ascii("a1.\u0627"), "a1.xn--mgb");
274 EXPECT_EQ(ada::idna::to_ascii("\u0627.example"), "xn--mgb.example");
275 // Not a Bidi domain, so a leading digit is fine.
276 EXPECT_EQ(ada::idna::to_ascii("1.example"), "1.example");
277 // All-ASCII input, including punycode, stays on the WHATWG fast path.
278 EXPECT_EQ(ada::idna::to_ascii("1.xn--mgb"), "1.xn--mgb");
279}
280
281TEST(to_ascii_tests, contextj_viramas_from_newer_unicode) {
282 // Any Canonical_Combining_Class=Virama (9) character allows a following
283 // ZWNJ or ZWJ (RFC 5892 Appendix A). Each label is letter + virama + joiner
284 // + letter.
285 struct virama_case {
286 const char* zwnj_input;
287 const char* zwnj_output;
288 const char* zwj_input;
289 const char* zwj_output;
290 };
291 const virama_case cases[] = {
292 // U+1715 TAGALOG SIGN PAMUDPOD
293 {"\u1703\u1715\u200C\u1703", "xn--xyea7c825c", "\u1703\u1715\u200D\u1703",
294 "xn--xyea7cv35c"},
295 // U+11070 BRAHMI SIGN OLD TAMIL VIRAMA
296 {"\U00011013\U00011070\u200C\U00011013", "xn--0ug4565gba48a",
297 "\U00011013\U00011070\u200D\U00011013", "xn--1ug2565gba48a"},
298 // U+113CE TULU-TIGALARI SIGN VIRAMA
299 {"\U00011392\U000113CE\u200C\U00011392", "xn--0ug0307gba2v",
300 "\U00011392\U000113CE\u200D\U00011392", "xn--1ugy307gba2v"},
301 // U+113CF TULU-TIGALARI SIGN LOOPED VIRAMA
302 {"\U00011392\U000113CF\u200C\U00011392", "xn--0ug0307gba6v",
303 "\U00011392\U000113CF\u200D\U00011392", "xn--1ugy307gba6v"},
304 // U+113D0 TULU-TIGALARI CONJOINER
305 {"\U00011392\U000113D0\u200C\U00011392", "xn--0ug0307gba0w",
306 "\U00011392\U000113D0\u200D\U00011392", "xn--1ugy307gba0w"},
307 // U+11F41 KAWI SIGN KILLER
308 {"\U00011F12\U00011F41\u200C\U00011F12", "xn--0ugz651hba0q",
309 "\U00011F12\U00011F41\u200D\U00011F12", "xn--1ugx651hba0q"},
310 // U+11F42 KAWI CONJOINER
311 {"\U00011F12\U00011F42\u200C\U00011F12", "xn--0ugz651hba4q",
312 "\U00011F12\U00011F42\u200D\U00011F12", "xn--1ugx651hba4q"},
313 // U+1612F GURUNG KHEMA SIGN THOLHOMA
314 {"\U00016101\U0001612F\u200C\U00016101", "xn--0ug9257jba6p",
315 "\U00016101\U0001612F\u200D\U00016101", "xn--1ug7257jba6p"},
316 };
317 for (const virama_case& c : cases) {
318 EXPECT_EQ(ada::idna::to_ascii(c.zwnj_input), c.zwnj_output);
319 EXPECT_EQ(ada::idna::to_ascii(c.zwj_input), c.zwj_output);
320 }
321}
322
323TEST(to_ascii_tests, contextj_joining_types) {
324 // ZWNJ between a Joining_Type L or D character and an R or D character.
325 EXPECT_EQ(ada::idna::to_ascii("\u07CA\u200C\u07CB"), "xn--lsbc047p")
326 << "N'Ko";
327 EXPECT_EQ(ada::idna::to_ascii("\U00010AC0\u200C\U00010AC1"), "xn--0ug8553gea")
328 << "Manichaean";
329 EXPECT_EQ(ada::idna::to_ascii("\U00010D01\u200C\U00010D02"), "xn--0ug5444gea")
330 << "Hanifi Rohingya";
331 EXPECT_EQ(ada::idna::to_ascii("\U00010F30\u200C\U00010F31"), "xn--0ug5035gea")
332 << "Sogdian";
333 EXPECT_EQ(ada::idna::to_ascii("\U00010F70\u200C\U00010F71"), "xn--0ug3045gea")
334 << "Old Uyghur";
335 EXPECT_EQ(ada::idna::to_ascii("\U00010FB0\u200C\U00010FB0"), "xn--0ug1055gba")
336 << "Chorasmian";
337 EXPECT_EQ(ada::idna::to_ascii("\U0001E922\u200C\U0001E923"), "xn--0ug1411pea")
338 << "Adlam";
339 EXPECT_EQ(ada::idna::to_ascii("\u0886\u200C\u0886"), "xn--yxba971p")
340 << "Arabic Extended-B";
341 // U+10ED9 ARABIC CROWN LETTER BEH (Unicode 18) is left-joining: it may come
342 // before a ZWNJ but not after one.
343 EXPECT_EQ(ada::idna::to_ascii("\U00010ED9\u200C\u0628"), "xn--ngb963k752o");
344 EXPECT_TRUE(ada::idna::to_ascii("\u0628\u200C\U00010ED9").empty());
345}
346
347TEST(to_ascii_tests, contextj_transparent_characters) {
348 // Only Joining_Type T characters may come between the ZWNJ and the joining
349 // characters around it: (L|D) T* ZWNJ T* (R|D).
350 EXPECT_EQ(ada::idna::to_ascii("\u0628\u064E\u200C\u0628"), "xn--ngba7iz95i");
351 EXPECT_EQ(ada::idna::to_ascii("\u0628\u200C\u064E\u0628"), "xn--ngba7iy95i");
352 EXPECT_EQ(ada::idna::to_ascii("\u0628\u064E\u0651\u200C\u0628"),
353 "xn--ngba7im9404a");
354 // U+0627 ALEF (R) directly before the ZWNJ.
355 EXPECT_TRUE(ada::idna::to_ascii("\u0628\u0627\u200C\u0628").empty());
356 // U+0621 HAMZA (U) directly after the ZWNJ.
357 EXPECT_TRUE(ada::idna::to_ascii("\u0628\u200C\u0621\u0628").empty());
358}
359
360// Helper: domain-to-ASCII as URL Standard callers use it (to_ascii + forbidden
361// domain code point filter). Empty string means failure.
362static std::string domain_to_ascii(std::string_view input) {
363 std::string out = ada::idna::to_ascii(input);
365 return "";
366 }
367 return out;
368}
369
370// Regression tests for the WHATWG URL "domain to ASCII" web-compatibility
371// carve-out: when the input domain is an ASCII string (beStrict = false), the
372// result is the input lowercased, regardless of Unicode ToASCII's outcome. An
373// ACE ("xn--") label may decode yet still fail IDNA validity criteria and must
374// nevertheless be accepted as-is.
375// See https://url.spec.whatwg.org/#concept-domain-to-ascii
376//
377// Cases below are drawn from:
378// - WPT toascii.json (vendored as fixtures/toascii.json)
379// - WPT IdnaTestV2.json carve-out flips (null -> pass-through)
380// - WPT urltestdata.json hostnames that flipped failure -> success in Ada
381// - WPT setters_tests.json host/hostname "xn--" acceptance
382// - Ada URLPattern hostname unit tests (xn--a / XN--A)
383TEST(to_ascii_tests, ascii_xn_carveout) {
384 // --- Bare / empty ACE payload ---
385 // Bare ACE prefix: the Punycode payload is empty. Previously rejected.
386 // WPT setters_tests / urltestdata: host "xn--" is accepted.
387 EXPECT_EQ(ada::idna::to_ascii("xn--"), "xn--");
388 EXPECT_EQ(domain_to_ascii("xn--"), "xn--");
389 EXPECT_EQ(ada::idna::to_ascii("XN--"), "xn--");
390 // Trailing hyphen only in the ACE payload.
391 EXPECT_EQ(ada::idna::to_ascii("xn---"), "xn---");
392
393 // --- Invalid Punycode (decode failure) ---
394 // WPT toascii.json "Invalid Punycode": the ACE segment does not decode.
395 // Ada URLPattern hostname "xn--a" must be accepted as "xn--a".
396 EXPECT_EQ(ada::idna::to_ascii("xn--a"), "xn--a");
397 EXPECT_EQ(domain_to_ascii("xn--a"), "xn--a");
398 EXPECT_EQ(ada::idna::to_ascii("XN--A"), "xn--a");
399 EXPECT_EQ(ada::idna::to_ascii("Xn--a"), "xn--a");
400 // Same invalid ACE as a label of a larger domain (IdnaTestV2 V7 / toascii).
401 EXPECT_EQ(ada::idna::to_ascii("xn--a.pt"), "xn--a.pt");
402 EXPECT_EQ(ada::idna::to_ascii("xn--a.xn--zca"), "xn--a.xn--zca");
403 // Invalid ACE with trailing junk that remains ASCII (toascii.json).
404 EXPECT_EQ(ada::idna::to_ascii("xn--ls8h="), "xn--ls8h=");
405 // Invalid ACE that is pure digits after prefix (IdnaTestV2 / toascii).
406 EXPECT_EQ(ada::idna::to_ascii("xn--0"), "xn--0");
407 EXPECT_EQ(ada::idna::to_ascii("xn--0.com"), "xn--0.com");
408 EXPECT_EQ(ada::idna::to_ascii("xn--0.pt"), "xn--0.pt");
409
410 // --- Decode succeeds but IDNA validity fails (ContextJ / mapped / etc.) ---
411 // ContextJ (C1): xn--ab-j1t decodes to "a\u200Cb" (ZWNJ not preceded by a
412 // virama). Previously rejected; now accepted as-is.
413 EXPECT_EQ(ada::idna::to_ascii("xn--ab-j1t"), "xn--ab-j1t");
414 // ContextJ (C2): ZWJ variant.
415 EXPECT_EQ(ada::idna::to_ascii("xn--ab-m1t"), "xn--ab-m1t");
416 // V1 / other validity failures that remain pure-ASCII ACE forms.
417 EXPECT_EQ(ada::idna::to_ascii("xn--u-ccb"), "xn--u-ccb");
418 EXPECT_EQ(ada::idna::to_ascii("xn--acom-0w1b"), "xn--acom-0w1b");
419 EXPECT_EQ(ada::idna::to_ascii("xn--a-ecp.ru"), "xn--a-ecp.ru");
420 EXPECT_EQ(ada::idna::to_ascii("xn--xn--a--gua.pt"), "xn--xn--a--gua.pt");
421 // Joiners ACE still accepted when input is pure ASCII (toascii CheckJoiners).
422 EXPECT_EQ(ada::idna::to_ascii("xn--1ug.example"), "xn--1ug.example");
423 EXPECT_EQ(ada::idna::to_ascii("xn--1ug"), "xn--1ug");
424 // Bidi ACE form accepted as ASCII (toascii CheckBidi counterpart).
425 EXPECT_EQ(ada::idna::to_ascii("xn--a-yoc"), "xn--a-yoc");
426 // U+FFFD encoded in Punycode is pure ASCII input -> accepted (toascii).
427 EXPECT_EQ(ada::idna::to_ascii("xn--zn7c.com"), "xn--zn7c.com");
428
429 // Decoded label has code points with "mapped" status (enclosed CJK), so the
430 // ACE form is non-canonical. Previously rejected; now accepted as-is.
431 // From Ada WPT urltestdata.json (failure -> success under carve-out).
432 EXPECT_EQ(ada::idna::to_ascii("a.b.c.xn--pokxncvks"), "a.b.c.xn--pokxncvks");
433 EXPECT_EQ(ada::idna::to_ascii("10.0.0.xn--pokxncvks"),
434 "10.0.0.xn--pokxncvks");
435
436 // The URL spec's own example: xn--8i7caa decodes to fullwidth "www", whose
437 // code points have "mapped" status.
438 EXPECT_EQ(ada::idna::to_ascii("xn--8i7caa"), "xn--8i7caa");
439
440 // Double-encoded ACE prefix forms that remain pure ASCII (IdnaTestV2 flips).
441 EXPECT_EQ(ada::idna::to_ascii("xn--xn--bss-7z6ccid"), "xn--xn--bss-7z6ccid");
442 EXPECT_EQ(ada::idna::to_ascii("xn--xn---epa"), "xn--xn---epa");
443 EXPECT_EQ(ada::idna::to_ascii("xn--ASCII-"), "xn--ascii-");
444 EXPECT_EQ(ada::idna::to_ascii("xn--unicode-.org"), "xn--unicode-.org");
445
446 // Long ACE labels that previously failed validity (IdnaTestV2 C1/C2/A4).
447 EXPECT_EQ(
449 "1.xn--"
450 "assbcssssssssdssssssssssssssssessssssssssssssssssssxssssssssssss"
451 "ssssssssysssssssssssssssssz-pxq1419aa69989dba9gc"),
452 "1.xn--"
453 "assbcssssssssdssssssssssssssssessssssssssssssssssssxssssssssssssssss"
454 "ssssysssssssssssssssssz-pxq1419aa69989dba9gc");
455 EXPECT_EQ(
457 "1.xn--abcdexyz-qyacaaabaaaaaaabaaaaaaaaabaaaaaaaaabaaaaaaaa010ze"
458 "2isb1140zba8cc"),
459 "1.xn--abcdexyz-qyacaaabaaaaaaabaaaaaaaaabaaaaaaaaabaaaaaaaa010ze2isb"
460 "1140zba8cc");
461
462 // --- Case folding of ACE prefixes (urltestdata mixed-case hostnames) ---
463 EXPECT_EQ(ada::idna::to_ascii("a.b.c.XN--pokxncvks"), "a.b.c.xn--pokxncvks");
464 EXPECT_EQ(ada::idna::to_ascii("a.b.c.Xn--pokxncvks"), "a.b.c.xn--pokxncvks");
465 EXPECT_EQ(ada::idna::to_ascii("10.0.0.XN--pokxncvks"),
466 "10.0.0.xn--pokxncvks");
467 EXPECT_EQ(ada::idna::to_ascii("10.0.0.xN--pokxncvks"),
468 "10.0.0.xn--pokxncvks");
469
470 // --- Ordinary pure-ASCII domains ---
471 EXPECT_EQ(ada::idna::to_ascii("EXAMPLE.COM"), "example.com");
472 EXPECT_EQ(ada::idna::to_ascii("gOoGle.com"), "google.com");
473 // Labels with hyphens in 3rd/4th position (toascii.json, UseSTD3=false).
474 EXPECT_EQ(ada::idna::to_ascii("aa--"), "aa--");
475 EXPECT_EQ(ada::idna::to_ascii("ab--c"), "ab--c");
476 EXPECT_EQ(ada::idna::to_ascii("ab--cd.com"), "ab--cd.com");
477 // Leading / trailing hyphen labels (toascii.json).
478 EXPECT_EQ(ada::idna::to_ascii("-x"), "-x");
479 EXPECT_EQ(ada::idna::to_ascii("-foo.bar.com"), "-foo.bar.com");
480 EXPECT_EQ(ada::idna::to_ascii("foo-.bar.com"), "foo-.bar.com");
481 // Empty labels between dots (toascii.json).
482 EXPECT_EQ(ada::idna::to_ascii("x..xn--zca"), "x..xn--zca");
483 EXPECT_EQ(ada::idna::to_ascii("-x.xn--zca"), "-x.xn--zca");
484 EXPECT_EQ(ada::idna::to_ascii("x-.xn--zca"), "x-.xn--zca");
485 EXPECT_EQ(ada::idna::to_ascii("ab--c.xn--zca"), "ab--c.xn--zca");
486 EXPECT_EQ(ada::idna::to_ascii("xn--zca.xn--zca"), "xn--zca.xn--zca");
487
488 // Overlong labels / domains (toascii.json): still pure ASCII -> accepted.
489 EXPECT_EQ(
491 "x01234567890123456789012345678901234567890123456789012345678901"
492 "x"),
493 "x01234567890123456789012345678901234567890123456789012345678901x");
494 EXPECT_EQ(
496 "x01234567890123456789012345678901234567890123456789012345678901x."
497 "xn--zca"),
498 "x01234567890123456789012345678901234567890123456789012345678901x.xn--"
499 "zca");
500
501 // --- Idempotency ---
502 const std::string once = ada::idna::to_ascii("xn--ab-j1t");
503 EXPECT_EQ(ada::idna::to_ascii(once), once);
504 EXPECT_EQ(ada::idna::to_ascii("xn--a"), "xn--a");
505 EXPECT_EQ(ada::idna::to_ascii(ada::idna::to_ascii("a.b.c.XN--pokxncvks")),
506 "a.b.c.xn--pokxncvks");
507 EXPECT_EQ(ada::idna::to_ascii(ada::idna::to_ascii("XN--")), "xn--");
508}
509
510// Table-driven mirror of the Ada urltestdata / setters carve-out hostnames:
511// every entry was previously failure (or rejected by set_host) and must now
512// round-trip through to_ascii as a lowercased ASCII domain.
513TEST(to_ascii_tests, ada_wpt_urltestdata_carveout_hosts) {
514 static constexpr std::pair<const char*, const char*> kCases[] = {
515 // urltestdata.json: mapped-status ACE labels
516 {"a.b.c.xn--pokxncvks", "a.b.c.xn--pokxncvks"},
517 {"10.0.0.xn--pokxncvks", "10.0.0.xn--pokxncvks"},
518 {"a.b.c.XN--pokxncvks", "a.b.c.xn--pokxncvks"},
519 {"a.b.c.Xn--pokxncvks", "a.b.c.xn--pokxncvks"},
520 {"10.0.0.XN--pokxncvks", "10.0.0.xn--pokxncvks"},
521 {"10.0.0.xN--pokxncvks", "10.0.0.xn--pokxncvks"},
522 // urltestdata.json / setters_tests.json: bare xn--
523 {"xn--", "xn--"},
524 {"XN--", "xn--"},
525 // file:// and https:// hosts that flipped failure -> success
526 {"xn--", "xn--"},
527 };
528 for (const auto& [input, expected] : kCases) {
529 SCOPED_TRACE(input);
530 EXPECT_EQ(domain_to_ascii(input), expected);
531 // Second application is a no-op (already lowercased ASCII).
532 EXPECT_EQ(domain_to_ascii(domain_to_ascii(input)), expected);
533 }
534}
535
536// The ASCII carve-out must NOT relax validation for non-ASCII inputs: those go
537// through full UTS#46 validation. Contrast xn--ab-j1t (ASCII, accepted above)
538// with its decoded form "a\u200Cb" supplied directly (non-ASCII, rejected).
539// Also covers toascii.json cases that remain failures for non-ASCII input.
540TEST(to_ascii_tests, non_ascii_inputs_still_validated) {
541 // ZWNJ (U+200C) without a preceding virama: ContextJ (C1) violation.
542 EXPECT_TRUE(ada::idna::to_ascii("a\u200Cb").empty())
543 << "non-ASCII ContextJ violation should still fail";
544
545 // LTR label containing an RTL code point: Bidi rule violation.
546 EXPECT_TRUE(ada::idna::to_ascii("a\u05D0").empty())
547 << "non-ASCII Bidi violation should still fail";
548
549 // toascii.json: CheckJoiners is true — bare ZWJ label fails.
550 EXPECT_TRUE(ada::idna::to_ascii("\u200D.example").empty())
551 << "non-ASCII ZWJ joiner should still fail";
552
553 // toascii.json: CheckBidi is true — Arabic + Latin fails.
554 EXPECT_TRUE(ada::idna::to_ascii("يa").empty())
555 << "non-ASCII Bidi (Arabic+Latin) should still fail";
556
557 // toascii.json: U+FFFD literal fails; contrast ACE form xn--zn7c.com above.
558 EXPECT_TRUE(ada::idna::to_ascii("\uFFFD.com").empty())
559 << "U+FFFD in domain should still fail";
560
561 // toascii.json: Invalid Punycode containing non-ASCII (xn--tešla) fails.
562 // UTF-8 for "xn--tešla" (š = U+0161 = C5 A1).
563 EXPECT_TRUE(ada::idna::to_ascii("xn--te\xc5\xa1la").empty())
564 << "invalid punycode with non-ASCII must not use the ASCII carve-out";
565
566 // toascii.json: xn--a.ß mixes invalid ACE with non-ASCII sharp-s -> fail.
567 EXPECT_TRUE(ada::idna::to_ascii("xn--a.\xc3\x9f").empty())
568 << "mixed invalid ACE + non-ASCII label should still fail";
569}
570
571// The combining mark, Bidi and normalization tables must cover the same
572// Unicode version as the mapping table (ada-url/idna#97).
573TEST(to_ascii_tests, leading_combining_mark_from_newer_unicode) {
574 // A label must not begin with a combining mark (General_Category M).
575 EXPECT_TRUE(ada::idna::to_ascii("\u0301a").empty()) << "Unicode 1.1 mark";
576 EXPECT_TRUE(ada::idna::to_ascii("\u08CBa").empty()) << "Unicode 14 mark";
577 EXPECT_TRUE(ada::idna::to_ascii("\u0897a").empty()) << "Unicode 16 mark";
578 EXPECT_TRUE(ada::idna::to_ascii("\u1AE0a").empty()) << "Unicode 17 mark";
579 EXPECT_TRUE(ada::idna::to_ascii("\u1ADEa").empty()) << "Unicode 18 mark";
580 // The same mark after a base character is valid.
581 EXPECT_EQ(ada::idna::to_ascii("a\u08CB"), "xn--a-mqd");
582 EXPECT_EQ(ada::idna::to_ascii("a\u1ADE"), "xn--a-09k");
583}
584
585TEST(to_ascii_tests, rtl_characters_from_newer_unicode) {
586 // An RTL label must not contain an L character such as 'a'.
587 EXPECT_TRUE(ada::idna::to_ascii("\u0870a").empty()) << "Unicode 14 AL";
588 EXPECT_TRUE(ada::idna::to_ascii("\U00010D4Aa").empty()) << "Unicode 16 R";
589 EXPECT_TRUE(ada::idna::to_ascii("\U00010940a").empty()) << "Unicode 17 R";
590 EXPECT_TRUE(ada::idna::to_ascii("\U00010ED9a").empty()) << "Unicode 18 AL";
591 // An LTR label must not contain an R or AL character.
592 EXPECT_TRUE(ada::idna::to_ascii("a\U00010940").empty()) << "Unicode 17 R";
593 EXPECT_TRUE(ada::idna::to_ascii("a\U00010ED9").empty()) << "Unicode 18 AL";
594 // RTL labels made of these characters are valid.
595 EXPECT_EQ(ada::idna::to_ascii("\u0870\u0871"), "xn--cxbc");
596 EXPECT_EQ(ada::idna::to_ascii("\U00010D4A\U00010D4B"), "xn--9f0dc");
597 EXPECT_EQ(ada::idna::to_ascii("\U00010940\U00010941"), "xn--ql9cc");
598 EXPECT_EQ(ada::idna::to_ascii("\U00010ED9\U00010EDA"), "xn--or0dc");
599}
600
601TEST(to_ascii_tests, bidi_class_changes_in_unicode18) {
602 // Unicode 18 changes U+1B3A BALINESE VOWEL SIGN RA REPA from NSM to L, so it
603 // may no longer follow an R character in an RTL label.
604 EXPECT_TRUE(ada::idna::to_ascii("\u05D0\u1B3A").empty());
605 EXPECT_EQ(ada::idna::to_ascii("\u1B13\u1B3A"), "xn--ntf8e");
606}
607
608TEST(to_ascii_tests, unicode18_canonical_ordering) {
609 // U+05C8 (ccc 10) and U+05C9 (ccc 21) are new Hebrew points; NFC sorts them
610 // by combining class.
611 EXPECT_EQ(ada::idna::to_ascii("\u05D0\u05C8\u05C9"), "xn--wdbct");
612 EXPECT_EQ(ada::idna::to_ascii("\u05D0\u05C9\u05C8"), "xn--wdbct");
613}
614
615TEST(to_ascii_tests, unicode16_canonical_composition) {
616 // U+1611E U+1611E composes to U+16121 GURUNG KHEMA VOWEL SIGN U, so both
617 // spellings are the same host.
618 EXPECT_EQ(ada::idna::to_ascii("a\U0001611E\U0001611E"), "xn--a-rn1m");
619 EXPECT_EQ(ada::idna::to_ascii("a\U00016121"), "xn--a-rn1m");
620 // U+1611E U+16121 is U+1611E U+1611E U+1611E, which normalizes to
621 // U+16121 U+1611E.
622 EXPECT_EQ(ada::idna::to_ascii("a\U0001611E\U00016121"), "xn--a-ln1mia");
623 EXPECT_EQ(ada::idna::to_ascii("a\U00016121\U0001611E"), "xn--a-ln1mia");
624 // U+16D68 is U+16D67 U+16D67: U+16D63 U+16D68 normalizes to U+16D6A.
625 EXPECT_EQ(ada::idna::to_ascii("a\U00016D63\U00016D68"), "xn--a-hs6m");
626 EXPECT_EQ(ada::idna::to_ascii("a\U00016D6A"), "xn--a-hs6m");
627}
std::string invalid_filename
static std::string ReadFile(const std::string &filename)
std::string alternating_filename
static std::vector< std::string > SplitString(const std::string &str)
std::string to_ascii(std::string_view ut8_string)
bool contains_forbidden_domain_code_point(std::string_view ascii_string)
TEST(to_ascii_tests, special_cases)
TEST_F(IdnaFileTest, AlternatingFileConversions)
static std::string domain_to_ascii(std::string_view input)