idna/idna2008_test.mbt1///|2fn expect_idna2008_rejection(3 domain : String,4 expected_cp : Int32,5 check_contexto~ : Bool = false,6) -> Unit raise {7 for ascii in [true, false] {8 try {9 if ascii {10 @idna.to_ascii(11 domain,12 use_std3_ascii_rules=false,13 check_idna2008=true,14 check_contexto~,15 )16 } else {17 @idna.to_unicode(18 domain,19 use_std3_ascii_rules=false,20 check_idna2008=true,21 check_contexto~,22 )23 }24 } catch {25 @idna.DisallowedCodePoint(cp~) => @test.assert_eq(cp, expected_cp)26 error => raise error27 } noraise {28 _ => fail("Expected IDNA2008 repertoire rejection for \{domain}")29 }30 }31}3233///|34fn expect_contexto_rejection(35 domain : String,36 expected_cp? : Int32,37) -> Unit raise {38 for ascii in [true, false] {39 try {40 if ascii {41 @idna.to_ascii(domain, check_contexto=true, check_bidi=false)42 } else {43 @idna.to_unicode(domain, check_contexto=true, check_bidi=false)44 }45 } catch {46 @idna.InvalidContextO(cp~) =>47 if expected_cp is Some(expected) {48 @test.assert_eq(cp, expected)49 }50 error => raise error51 } noraise {52 _ => fail("Expected CONTEXTO rejection for \{domain}")53 }54 }55}5657///|58test "IDNA2008 repertoire: default preserves UTS46 symbols" {59 inspect(@idna.to_ascii("☕.example"), content="xn--53h.example")60 inspect(@idna.to_unicode("xn--53h.example"), content="☕.example")61 inspect(@idna.to_ascii("♥.example"), content="xn--g6h.example")62 inspect(@idna.to_unicode("xn--g6h.example"), content="♥.example")63 for64 (input, cp) in [65 ("☕.example", 0x2615),66 ("xn--53h.example", 0x2615),67 ("♥.example", 0x2665),68 ("xn--g6h.example", 0x2665),69 ] {70 expect_idna2008_rejection(input, cp)71 }72}7374///|75test "IDNA2008 repertoire: mapping and NFC still precede validation" {76 for input in ["BÜCHER.DE", "bücher.de", "bu\u{0308}cher.de"] {77 inspect(78 @idna.to_ascii(input, check_idna2008=true, check_contexto=true),79 content="xn--bcher-kva.de",80 )81 inspect(82 @idna.to_unicode(input, check_idna2008=true, check_contexto=true),83 content="bücher.de",84 )85 }86 inspect(87 @idna.to_ascii("EXAM\u{00AD}PLE.COM", check_idna2008=true),88 content="example.com",89 )90 inspect(91 @idna.to_unicode("EXAM\u{00AD}PLE.COM", check_idna2008=true),92 content="example.com",93 )94}9596///|97test "IDNA2008 repertoire: unchanged labels and decoded ACE are checked" {98 for99 input in [100 "☕.example", "☕.EXAMPLE", "xn--53h.example", "XN--53H.EXAMPLE",101 ] {102 expect_idna2008_rejection(input, 0x2615)103 }104 inspect(105 @idna.to_ascii("faß.de", check_idna2008=true),106 content="xn--fa-hia.de",107 )108 inspect(109 @idna.to_unicode("xn--fa-hia.de", check_idna2008=true),110 content="faß.de",111 )112}113114///|115test "IDNA2008 repertoire: ASCII fast path respects independent STD3 flag" {116 inspect(117 @idna.to_ascii("under_score.example", use_std3_ascii_rules=false),118 content="under_score.example",119 )120 inspect(121 @idna.to_unicode("under_score.example", use_std3_ascii_rules=false),122 content="under_score.example",123 )124 expect_idna2008_rejection("under_score.example", 0x005F)125 expect_idna2008_rejection("UNDER_SCORE.EXAMPLE", 0x005F)126 for input in ["example.com", "EXAMPLE.COM"] {127 inspect(@idna.to_ascii(input, check_idna2008=true), content="example.com")128 inspect(@idna.to_unicode(input, check_idna2008=true), content="example.com")129 }130}131132///|133test "CONTEXTO: valid contexts survive Unicode and ACE conversion" {134 for135 label in [136 "l·l", "\u{0375}α", "\u{0375}\u{1F00}", "א\u{05F3}", "א\u{05F4}", "・カ",137 "カ・", "・あ", "・中", "・\u{20000}", "\u{0660}\u{0669}", "\u{06F0}\u{06F9}",138 "l·l·l", "\u{0375}\u{0375}α", "א\u{05F3}\u{05F4}", "\u{20000}・l·l",139 ] {140 // Pure digit labels test CONTEXTO independently of RFC 5893.141 let ace = @idna.to_ascii(label, check_bidi=false)142 for input in [label, ace] {143 @test.assert_eq(144 @idna.to_ascii(145 input,146 check_idna2008=true,147 check_contexto=true,148 check_bidi=false,149 ),150 ace,151 )152 @test.assert_eq(153 @idna.to_unicode(154 input,155 check_idna2008=true,156 check_contexto=true,157 check_bidi=false,158 ),159 label,160 )161 }162 }163}164165///|166test "CONTEXTO: middle dot requires an immediate l on both sides" {167 for168 label in [169 "·", "·l", "l·", "a·l", "l·a", "l·\u{0301}l", "l··l", "l·\u{20000}",170 "\u{20000}·l",171 ] {172 expect_contexto_rejection(label, expected_cp=0x00B7)173 let ace = @idna.to_ascii(label, check_bidi=false)174 expect_contexto_rejection(ace, expected_cp=0x00B7)175 }176 inspect(177 @idna.to_unicode("L·L", check_idna2008=true, check_contexto=true),178 content="l·l",179 )180}181182///|183test "CONTEXTO: Greek numeral sign checks following Script not block" {184 // U+03E3 is Coptic despite belonging to the Greek and Coptic block.185 for186 label in [187 "\u{0375}", "\u{0375}a", "α\u{0375}", "\u{0375}\u{03E3}", "\u{0375}\u{0375}",188 ] {189 expect_contexto_rejection(label, expected_cp=0x0375)190 let ace = @idna.to_ascii(label, check_bidi=false)191 expect_contexto_rejection(ace, expected_cp=0x0375)192 }193 // A supplementary Greek Script character satisfies the neighbor rule,194 // independently of its exclusion from the IDNA2008 allowed character set.195 let label = "\u{0375}\u{1D200}"196 let ace = @idna.to_ascii(label, check_bidi=false)197 for input in [label, ace] {198 @test.assert_eq(199 @idna.to_ascii(input, check_contexto=true, check_bidi=false),200 ace,201 )202 @test.assert_eq(203 @idna.to_unicode(input, check_contexto=true, check_bidi=false),204 label,205 )206 }207}208209///|210test "CONTEXTO: Hebrew punctuation requires preceding Hebrew Script" {211 for212 (label, cp) in [213 ("\u{05F3}", 0x05F3),214 ("\u{05F4}", 0x05F4),215 ("\u{05F3}א", 0x05F3),216 ("a\u{05F3}", 0x05F3),217 ("a\u{05F4}", 0x05F4),218 ("\u{20000}\u{05F3}", 0x05F3),219 ("א\u{20000}\u{05F4}", 0x05F4),220 ] {221 expect_contexto_rejection(label, expected_cp=cp)222 let ace = @idna.to_ascii(label, check_bidi=false)223 expect_contexto_rejection(ace, expected_cp=cp)224 }225}226227///|228test "CONTEXTO: Katakana middle dot needs a qualifying Script in its label" {229 // U+30FC has Script=Common, even though its Script_Extensions are Japanese.230 for label in ["・", "a・", "・ー", "カ.a・", "a・.カ"] {231 expect_contexto_rejection(label, expected_cp=0x30FB)232 let ace = @idna.to_ascii(label, check_bidi=false)233 expect_contexto_rejection(ace, expected_cp=0x30FB)234 }235}236237///|238test "CONTEXTO: Arabic digit sets cannot mix within one label" {239 for label in ["\u{0660}\u{06F0}", "\u{06F9}\u{0669}"] {240 expect_contexto_rejection(label)241 let ace = @idna.to_ascii(label, check_bidi=false)242 expect_contexto_rejection(ace)243 }244 let separate_labels = "\u{0660}.\u{06F0}"245 let ace = @idna.to_ascii(246 separate_labels,247 check_idna2008=true,248 check_contexto=true,249 check_bidi=false,250 )251 @test.assert_eq(252 @idna.to_unicode(253 ace,254 check_idna2008=true,255 check_contexto=true,256 check_bidi=false,257 ),258 separate_labels,259 )260}261262///|263test "CONTEXTO: adjacent labels cannot supply immediate context" {264 for265 (domain, cp) in [266 ("l.·l", 0x00B7),267 ("l·.l", 0x00B7),268 ("\u{0375}.α", 0x0375),269 ("א.\u{05F3}", 0x05F3),270 ("א.\u{05F4}", 0x05F4),271 ] {272 expect_contexto_rejection(domain, expected_cp=cp)273 }274}275276///|277test "CONTEXTO: neighboring violations precede later and deferred errors" {278 for279 (label, cp) in [280 ("l·\u{05F3}", 0x00B7),281 ("\u{0375}·", 0x0375),282 ("・l·a", 0x00B7),283 ("・l·", 0x00B7),284 ("・\u{0375}", 0x0375),285 ("\u{0660}\u{0375}\u{06F0}", 0x0375),286 ("\u{0660}\u{06F0}\u{0375}", 0x06F0),287 ] {288 expect_contexto_rejection(label, expected_cp=cp)289 let ace = @idna.to_ascii(label, check_bidi=false)290 expect_contexto_rejection(ace, expected_cp=cp)291 }292}293294///|295test "IDNA2008 repertoire and CONTEXTO flags are independent" {296 inspect(@idna.to_unicode("a·b", check_idna2008=true), content="a·b")297 expect_contexto_rejection("a·b", expected_cp=0x00B7)298 inspect(299 @idna.to_ascii("☕.example", check_contexto=true),300 content="xn--53h.example",301 )302 inspect(303 @idna.to_unicode("xn--53h.example", check_contexto=true),304 content="☕.example",305 )306 // The label's character validation finishes before contextual rules run,307 // so the disallowed coffee character takes precedence over the middle dot.308 let label = "a·☕"309 let ace = @idna.to_ascii(label)310 for input in [label, ace] {311 expect_idna2008_rejection(input, 0x2615, check_contexto=true)312 }313 inspect(314 @idna.to_ascii("-example", check_idna2008=true, check_hyphens=false),315 content="-example",316 )317 inspect(318 @idna.to_unicode("-example", check_idna2008=true, check_hyphens=false),319 content="-example",320 )321}322323///|324test "IDNA2008 repertoire: CONTEXTJ still obeys check_joiners" {325 for ascii in [true, false] {326 try {327 if ascii {328 @idna.to_ascii("a\u{200C}b", check_idna2008=true)329 } else {330 @idna.to_unicode("a\u{200C}b", check_idna2008=true)331 }332 } catch {333 @idna.InvalidJoiner => ()334 error => raise error335 } noraise {336 _ => fail("Expected existing CONTEXTJ validation")337 }338 }339 let ace = @idna.to_ascii(340 "a\u{200C}b",341 check_idna2008=true,342 check_contexto=true,343 check_joiners=false,344 )345 inspect(346 @idna.to_unicode(347 ace,348 check_idna2008=true,349 check_contexto=true,350 check_joiners=false,351 ),352 content="a\u{200C}b",353 )354}355
///|
fn expect_idna2008_rejection(
domain : String,
expected_cp : Int32,
check_contexto~ : Bool = false,
) -> Unit raise {
for ascii in [true, false] {
try {
if ascii {
@idna.to_ascii(
domain,
use_std3_ascii_rules=false,
check_idna2008=true,
check_contexto~,
)
} else {
@idna.to_unicode(
domain,
use_std3_ascii_rules=false,
check_idna2008=true,
check_contexto~,
)
}
} catch {
@idna.DisallowedCodePoint(cp~) => @test.assert_eq(cp, expected_cp)
error => raise error
} noraise {
_ => fail("Expected IDNA2008 repertoire rejection for \{domain}")
}
}
}
///|
fn expect_contexto_rejection(
domain : String,
expected_cp? : Int32,
) -> Unit raise {
for ascii in [true, false] {
try {
if ascii {
@idna.to_ascii(domain, check_contexto=true, check_bidi=false)
} else {
@idna.to_unicode(domain, check_contexto=true, check_bidi=false)
}
} catch {
@idna.InvalidContextO(cp~) =>
if expected_cp is Some(expected) {
@test.assert_eq(cp, expected)
}
error => raise error
} noraise {
_ => fail("Expected CONTEXTO rejection for \{domain}")
}
}
}
///|
test "IDNA2008 repertoire: default preserves UTS46 symbols" {
inspect(@idna.to_ascii("☕.example"), content="xn--53h.example")
inspect(@idna.to_unicode("xn--53h.example"), content="☕.example")
inspect(@idna.to_ascii("♥.example"), content="xn--g6h.example")
inspect(@idna.to_unicode("xn--g6h.example"), content="♥.example")
for
(input, cp) in [
("☕.example", 0x2615),
("xn--53h.example", 0x2615),
("♥.example", 0x2665),
("xn--g6h.example", 0x2665),
] {
expect_idna2008_rejection(input, cp)
}
}
///|
test "IDNA2008 repertoire: mapping and NFC still precede validation" {
for input in ["BÜCHER.DE", "bücher.de", "bu\u{0308}cher.de"] {
inspect(
@idna.to_ascii(input, check_idna2008=true, check_contexto=true),
content="xn--bcher-kva.de",
)
inspect(
@idna.to_unicode(input, check_idna2008=true, check_contexto=true),
content="bücher.de",
)
}
inspect(
@idna.to_ascii("EXAM\u{00AD}PLE.COM", check_idna2008=true),
content="example.com",
)
inspect(
@idna.to_unicode("EXAM\u{00AD}PLE.COM", check_idna2008=true),
content="example.com",
)
}
///|
test "IDNA2008 repertoire: unchanged labels and decoded ACE are checked" {
for
input in [
"☕.example", "☕.EXAMPLE", "xn--53h.example", "XN--53H.EXAMPLE",
] {
expect_idna2008_rejection(input, 0x2615)
}
inspect(
@idna.to_ascii("faß.de", check_idna2008=true),
content="xn--fa-hia.de",
)
inspect(
@idna.to_unicode("xn--fa-hia.de", check_idna2008=true),
content="faß.de",
)
}
///|
test "IDNA2008 repertoire: ASCII fast path respects independent STD3 flag" {
inspect(
@idna.to_ascii("under_score.example", use_std3_ascii_rules=false),
content="under_score.example",
)
inspect(
@idna.to_unicode("under_score.example", use_std3_ascii_rules=false),
content="under_score.example",
)
expect_idna2008_rejection("under_score.example", 0x005F)
expect_idna2008_rejection("UNDER_SCORE.EXAMPLE", 0x005F)
for input in ["example.com", "EXAMPLE.COM"] {
inspect(@idna.to_ascii(input, check_idna2008=true), content="example.com")
inspect(@idna.to_unicode(input, check_idna2008=true), content="example.com")
}
}
///|
test "CONTEXTO: valid contexts survive Unicode and ACE conversion" {
for
label in [
"l·l", "\u{0375}α", "\u{0375}\u{1F00}", "א\u{05F3}", "א\u{05F4}", "・カ",
"カ・", "・あ", "・中", "・\u{20000}", "\u{0660}\u{0669}", "\u{06F0}\u{06F9}",
"l·l·l", "\u{0375}\u{0375}α", "א\u{05F3}\u{05F4}", "\u{20000}・l·l",
] {
// Pure digit labels test CONTEXTO independently of RFC 5893.
let ace = @idna.to_ascii(label, check_bidi=false)
for input in [label, ace] {
@test.assert_eq(
@idna.to_ascii(
input,
check_idna2008=true,
check_contexto=true,
check_bidi=false,
),
ace,
)
@test.assert_eq(
@idna.to_unicode(
input,
check_idna2008=true,
check_contexto=true,
check_bidi=false,
),
label,
)
}
}
}
///|
test "CONTEXTO: middle dot requires an immediate l on both sides" {
for
label in [
"·", "·l", "l·", "a·l", "l·a", "l·\u{0301}l", "l··l", "l·\u{20000}",
"\u{20000}·l",
] {
expect_contexto_rejection(label, expected_cp=0x00B7)
let ace = @idna.to_ascii(label, check_bidi=false)
expect_contexto_rejection(ace, expected_cp=0x00B7)
}
inspect(
@idna.to_unicode("L·L", check_idna2008=true, check_contexto=true),
content="l·l",
)
}
///|
test "CONTEXTO: Greek numeral sign checks following Script not block" {
// U+03E3 is Coptic despite belonging to the Greek and Coptic block.
for
label in [
"\u{0375}", "\u{0375}a", "α\u{0375}", "\u{0375}\u{03E3}", "\u{0375}\u{0375}",
] {
expect_contexto_rejection(label, expected_cp=0x0375)
let ace = @idna.to_ascii(label, check_bidi=false)
expect_contexto_rejection(ace, expected_cp=0x0375)
}
// A supplementary Greek Script character satisfies the neighbor rule,
// independently of its exclusion from the IDNA2008 allowed character set.
let label = "\u{0375}\u{1D200}"
let ace = @idna.to_ascii(label, check_bidi=false)
for input in [label, ace] {
@test.assert_eq(
@idna.to_ascii(input, check_contexto=true, check_bidi=false),
ace,
)
@test.assert_eq(
@idna.to_unicode(input, check_contexto=true, check_bidi=false),
label,
)
}
}
///|
test "CONTEXTO: Hebrew punctuation requires preceding Hebrew Script" {
for
(label, cp) in [
("\u{05F3}", 0x05F3),
("\u{05F4}", 0x05F4),
("\u{05F3}א", 0x05F3),
("a\u{05F3}", 0x05F3),
("a\u{05F4}", 0x05F4),
("\u{20000}\u{05F3}", 0x05F3),
("א\u{20000}\u{05F4}", 0x05F4),
] {
expect_contexto_rejection(label, expected_cp=cp)
let ace = @idna.to_ascii(label, check_bidi=false)
expect_contexto_rejection(ace, expected_cp=cp)
}
}
///|
test "CONTEXTO: Katakana middle dot needs a qualifying Script in its label" {
// U+30FC has Script=Common, even though its Script_Extensions are Japanese.
for label in ["・", "a・", "・ー", "カ.a・", "a・.カ"] {
expect_contexto_rejection(label, expected_cp=0x30FB)
let ace = @idna.to_ascii(label, check_bidi=false)
expect_contexto_rejection(ace, expected_cp=0x30FB)
}
}
///|
test "CONTEXTO: Arabic digit sets cannot mix within one label" {
for label in ["\u{0660}\u{06F0}", "\u{06F9}\u{0669}"] {
expect_contexto_rejection(label)
let ace = @idna.to_ascii(label, check_bidi=false)
expect_contexto_rejection(ace)
}
let separate_labels = "\u{0660}.\u{06F0}"
let ace = @idna.to_ascii(
separate_labels,
check_idna2008=true,
check_contexto=true,
check_bidi=false,
)
@test.assert_eq(
@idna.to_unicode(
ace,
check_idna2008=true,
check_contexto=true,
check_bidi=false,
),
separate_labels,
)
}
///|
test "CONTEXTO: adjacent labels cannot supply immediate context" {
for
(domain, cp) in [
("l.·l", 0x00B7),
("l·.l", 0x00B7),
("\u{0375}.α", 0x0375),
("א.\u{05F3}", 0x05F3),
("א.\u{05F4}", 0x05F4),
] {
expect_contexto_rejection(domain, expected_cp=cp)
}
}
///|
test "CONTEXTO: neighboring violations precede later and deferred errors" {
for
(label, cp) in [
("l·\u{05F3}", 0x00B7),
("\u{0375}·", 0x0375),
("・l·a", 0x00B7),
("・l·", 0x00B7),
("・\u{0375}", 0x0375),
("\u{0660}\u{0375}\u{06F0}", 0x0375),
("\u{0660}\u{06F0}\u{0375}", 0x06F0),
] {
expect_contexto_rejection(label, expected_cp=cp)
let ace = @idna.to_ascii(label, check_bidi=false)
expect_contexto_rejection(ace, expected_cp=cp)
}
}
///|
test "IDNA2008 repertoire and CONTEXTO flags are independent" {
inspect(@idna.to_unicode("a·b", check_idna2008=true), content="a·b")
expect_contexto_rejection("a·b", expected_cp=0x00B7)
inspect(
@idna.to_ascii("☕.example", check_contexto=true),
content="xn--53h.example",
)
inspect(
@idna.to_unicode("xn--53h.example", check_contexto=true),
content="☕.example",
)
// The label's character validation finishes before contextual rules run,
// so the disallowed coffee character takes precedence over the middle dot.
let label = "a·☕"
let ace = @idna.to_ascii(label)
for input in [label, ace] {
expect_idna2008_rejection(input, 0x2615, check_contexto=true)
}
inspect(
@idna.to_ascii("-example", check_idna2008=true, check_hyphens=false),
content="-example",
)
inspect(
@idna.to_unicode("-example", check_idna2008=true, check_hyphens=false),
content="-example",
)
}
///|
test "IDNA2008 repertoire: CONTEXTJ still obeys check_joiners" {
for ascii in [true, false] {
try {
if ascii {
@idna.to_ascii("a\u{200C}b", check_idna2008=true)
} else {
@idna.to_unicode("a\u{200C}b", check_idna2008=true)
}
} catch {
@idna.InvalidJoiner => ()
error => raise error
} noraise {
_ => fail("Expected existing CONTEXTJ validation")
}
}
let ace = @idna.to_ascii(
"a\u{200C}b",
check_idna2008=true,
check_contexto=true,
check_joiners=false,
)
inspect(
@idna.to_unicode(
ace,
check_idna2008=true,
check_contexto=true,
check_joiners=false,
),
content="a\u{200C}b",
)
}