tests/idna_tests/idna_test.mbt1///|2/// IDNA (UTS #46) conformance tests, read directly from IdnaTestV2.txt3/// (Unicode 18.0.0).4/// Source: https://www.unicode.org/Public/18.0.0/idna/IdnaTestV2.txt5///6/// Each line is `source; toUnicode; toUnicodeStatus; toAscii; toAsciiStatus`.7/// Cases whose status list only contains codes tied to a validation flag are8/// expected to fail exactly when those flags are enabled; cases with9/// flag-independent status codes fail unconditionally.1011///|12struct TestCase {13 line_number : Int3214 source : String15 to_ascii : String16 status : Array[String]17}1819///|20struct RequiredFlags {21 check_bidi : Bool22 check_joiners : Bool23 check_hyphens : Bool24 use_std3_ascii_rules : Bool25 verify_dns_length : Bool26}2728///|29fn parse_test_line(line : StringView, line_number : Int32) -> TestCase? {30 let line = line.trim()31 if line.is_empty() || line.has_prefix("#") {32 return None33 }34 guard line.split(";").collect()35 is [36 source_text,37 unicode_text,38 unicode_status_text,39 ascii_text,40 ascii_status_text,41 ..,42 ] else {43 return None44 }45 let source = @support.parse_escaped_text(source_text.trim())46 let unicode = @support.parse_escaped_text(unicode_text.trim())47 let ascii = @support.parse_escaped_text(ascii_text.trim())48 let to_unicode = if unicode.text.is_empty() {49 source.text50 } else {51 unicode.text52 }53 let to_ascii = if ascii.text.is_empty() { to_unicode } else { ascii.text }54 let to_ascii_has_surrogate = if ascii.text.is_empty() {55 if unicode.text.is_empty() {56 source.has_surrogate57 } else {58 unicode.has_surrogate59 }60 } else {61 ascii.has_surrogate62 }63 let unicode_status = @support.parse_status(unicode_status_text).unwrap_or([])64 let ascii_status = @support.parse_status(ascii_status_text).unwrap_or(65 unicode_status,66 )67 if source.has_surrogate || to_ascii_has_surrogate {68 return None69 }70 Some({ line_number, source: source.text, to_ascii, status: ascii_status, })71}7273///|74fn flag_for_status(status : StringView) -> String? {75 match status {76 "B1" | "B2" | "B3" | "B4" | "B5" | "B6" => Some("check_bidi")77 "C1" | "C2" => Some("check_joiners")78 "V2" | "V3" => Some("check_hyphens")79 "V4" | "U1" => Some("use_std3_ascii_rules")80 "A4_1" | "A4_2" => Some("verify_dns_length")81 _ => None82 }83}8485///|86fn required_flags(statuses : ArrayView[String]) -> RequiredFlags {87 let mut check_bidi = false88 let mut check_joiners = false89 let mut check_hyphens = false90 let mut use_std3_ascii_rules = false91 let mut verify_dns_length = false92 for status in statuses {93 match flag_for_status(status) {94 Some("check_bidi") => check_bidi = true95 Some("check_joiners") => check_joiners = true96 Some("check_hyphens") => check_hyphens = true97 Some("use_std3_ascii_rules") => use_std3_ascii_rules = true98 Some("verify_dns_length") => verify_dns_length = true99 _ => ()100 }101 }102 {103 check_bidi,104 check_joiners,105 check_hyphens,106 use_std3_ascii_rules,107 verify_dns_length,108 }109}110111///|112fn RequiredFlags::any(self : RequiredFlags) -> Bool {113 self.check_bidi ||114 self.check_joiners ||115 self.check_hyphens ||116 self.use_std3_ascii_rules ||117 self.verify_dns_length118}119120///|121fn has_flag_independent_error(statuses : ArrayView[String]) -> Bool {122 statuses.any(status => flag_for_status(status) is None)123}124125///|126let no_flags : RequiredFlags = {127 check_bidi: false,128 check_joiners: false,129 check_hyphens: false,130 use_std3_ascii_rules: false,131 verify_dns_length: false,132}133134///|135fn call_to_ascii(136 source : String,137 flags : RequiredFlags,138) -> String raise @idna.IdnaError {139 @idna.to_ascii(140 source,141 use_std3_ascii_rules=flags.use_std3_ascii_rules,142 check_hyphens=flags.check_hyphens,143 check_bidi=flags.check_bidi,144 check_joiners=flags.check_joiners,145 verify_dns_length=flags.verify_dns_length,146 )147}148149///|150fn assert_success(case : TestCase) -> Unit raise {151 let result = call_to_ascii(case.source, no_flags)152 @debug.assert_eq(153 result,154 case.to_ascii,155 msg="IdnaTestV2.txt line \{case.line_number}: to_ascii mismatch",156 )157}158159///|160/// Assert that `to_ascii` raises for the case with the given flags. The try161/// body can only raise `IdnaError`, so any error escaping it is the expected162/// failure; a successful call falls through to `fail` with the line number.163fn assert_failure(case : TestCase, flags : RequiredFlags) -> Unit raise {164 try {165 let _ = call_to_ascii(case.source, flags)166 } catch {167 _ => return168 }169 fail("IdnaTestV2.txt line \{case.line_number}: expected to_ascii to fail")170}171172///|173async test "IdnaTestV2.txt conformance" {174 let content = @support.read_data_file("IdnaTestV2.txt")175 for line_number, raw_line in content.split("\n") {176 guard parse_test_line(raw_line, line_number + 1) is Some(case) else {177 continue178 }179 if case.status.is_empty() {180 assert_success(case)181 continue182 }183 let flags = required_flags(case.status)184 if flags.any() {185 assert_failure(case, flags)186 } else if has_flag_independent_error(case.status) {187 assert_failure(case, no_flags)188 } else {189 assert_success(case)190 }191 }192}193
///|
/// IDNA (UTS #46) conformance tests, read directly from IdnaTestV2.txt
/// (Unicode 18.0.0).
/// Source: https://www.unicode.org/Public/18.0.0/idna/IdnaTestV2.txt
///
/// Each line is `source; toUnicode; toUnicodeStatus; toAscii; toAsciiStatus`.
/// Cases whose status list only contains codes tied to a validation flag are
/// expected to fail exactly when those flags are enabled; cases with
/// flag-independent status codes fail unconditionally.
///|
struct TestCase {
line_number : Int32
source : String
to_ascii : String
status : Array[String]
}
///|
struct RequiredFlags {
check_bidi : Bool
check_joiners : Bool
check_hyphens : Bool
use_std3_ascii_rules : Bool
verify_dns_length : Bool
}
///|
fn parse_test_line(line : StringView, line_number : Int32) -> TestCase? {
let line = line.trim()
if line.is_empty() || line.has_prefix("#") {
return None
}
guard line.split(";").collect()
is [
source_text,
unicode_text,
unicode_status_text,
ascii_text,
ascii_status_text,
..,
] else {
return None
}
let source = @support.parse_escaped_text(source_text.trim())
let unicode = @support.parse_escaped_text(unicode_text.trim())
let ascii = @support.parse_escaped_text(ascii_text.trim())
let to_unicode = if unicode.text.is_empty() {
source.text
} else {
unicode.text
}
let to_ascii = if ascii.text.is_empty() { to_unicode } else { ascii.text }
let to_ascii_has_surrogate = if ascii.text.is_empty() {
if unicode.text.is_empty() {
source.has_surrogate
} else {
unicode.has_surrogate
}
} else {
ascii.has_surrogate
}
let unicode_status = @support.parse_status(unicode_status_text).unwrap_or([])
let ascii_status = @support.parse_status(ascii_status_text).unwrap_or(
unicode_status,
)
if source.has_surrogate || to_ascii_has_surrogate {
return None
}
Some({ line_number, source: source.text, to_ascii, status: ascii_status, })
}
///|
fn flag_for_status(status : StringView) -> String? {
match status {
"B1" | "B2" | "B3" | "B4" | "B5" | "B6" => Some("check_bidi")
"C1" | "C2" => Some("check_joiners")
"V2" | "V3" => Some("check_hyphens")
"V4" | "U1" => Some("use_std3_ascii_rules")
"A4_1" | "A4_2" => Some("verify_dns_length")
_ => None
}
}
///|
fn required_flags(statuses : ArrayView[String]) -> RequiredFlags {
let mut check_bidi = false
let mut check_joiners = false
let mut check_hyphens = false
let mut use_std3_ascii_rules = false
let mut verify_dns_length = false
for status in statuses {
match flag_for_status(status) {
Some("check_bidi") => check_bidi = true
Some("check_joiners") => check_joiners = true
Some("check_hyphens") => check_hyphens = true
Some("use_std3_ascii_rules") => use_std3_ascii_rules = true
Some("verify_dns_length") => verify_dns_length = true
_ => ()
}
}
{
check_bidi,
check_joiners,
check_hyphens,
use_std3_ascii_rules,
verify_dns_length,
}
}
///|
fn RequiredFlags::any(self : RequiredFlags) -> Bool {
self.check_bidi ||
self.check_joiners ||
self.check_hyphens ||
self.use_std3_ascii_rules ||
self.verify_dns_length
}
///|
fn has_flag_independent_error(statuses : ArrayView[String]) -> Bool {
statuses.any(status => flag_for_status(status) is None)
}
///|
let no_flags : RequiredFlags = {
check_bidi: false,
check_joiners: false,
check_hyphens: false,
use_std3_ascii_rules: false,
verify_dns_length: false,
}
///|
fn call_to_ascii(
source : String,
flags : RequiredFlags,
) -> String raise @idna.IdnaError {
@idna.to_ascii(
source,
use_std3_ascii_rules=flags.use_std3_ascii_rules,
check_hyphens=flags.check_hyphens,
check_bidi=flags.check_bidi,
check_joiners=flags.check_joiners,
verify_dns_length=flags.verify_dns_length,
)
}
///|
fn assert_success(case : TestCase) -> Unit raise {
let result = call_to_ascii(case.source, no_flags)
@debug.assert_eq(
result,
case.to_ascii,
msg="IdnaTestV2.txt line \{case.line_number}: to_ascii mismatch",
)
}
///|
/// Assert that `to_ascii` raises for the case with the given flags. The try
/// body can only raise `IdnaError`, so any error escaping it is the expected
/// failure; a successful call falls through to `fail` with the line number.
fn assert_failure(case : TestCase, flags : RequiredFlags) -> Unit raise {
try {
let _ = call_to_ascii(case.source, flags)
} catch {
_ => return
}
fail("IdnaTestV2.txt line \{case.line_number}: expected to_ascii to fail")
}
///|
async test "IdnaTestV2.txt conformance" {
let content = @support.read_data_file("IdnaTestV2.txt")
for line_number, raw_line in content.split("\n") {
guard parse_test_line(raw_line, line_number + 1) is Some(case) else {
continue
}
if case.status.is_empty() {
assert_success(case)
continue
}
let flags = required_flags(case.status)
if flags.any() {
assert_failure(case, flags)
} else if has_flag_independent_error(case.status) {
assert_failure(case, no_flags)
} else {
assert_success(case)
}
}
}