unicode

Branch: main

Switch branches or tags

Branch: main
Go to file t

413 files indexed

History
Code

tests/idna_tests/idna_test.mbt
1///|2/// IDNA (UTS #46) conformance tests, read directly from IdnaTestV2.txt3/// (Unicode 18.0.0).4/// Source: https://www.unicode.org/Public/18.0.0/idna/IdnaTestV2.txt5///6/// Each line is `source; toUnicode; toUnicodeStatus; toAscii; toAsciiStatus`.7/// Cases whose status list only contains codes tied to a validation flag are8/// expected to fail exactly when those flags are enabled; cases with9/// flag-independent status codes fail unconditionally.1011///|12struct TestCase {13  line_number : Int3214  source : String15  to_ascii : String16  status : Array[String]17}1819///|20struct RequiredFlags {21  check_bidi : Bool22  check_joiners : Bool23  check_hyphens : Bool24  use_std3_ascii_rules : Bool25  verify_dns_length : Bool26}2728///|29fn parse_test_line(line : StringView, line_number : Int32) -> TestCase? {30  let line = line.trim()31  if line.is_empty() || line.has_prefix("#") {32    return None33  }34  guard line.split(";").collect()35    is [36      source_text,37      unicode_text,38      unicode_status_text,39      ascii_text,40      ascii_status_text,41      ..,42    ] else {43    return None44  }45  let source = @support.parse_escaped_text(source_text.trim())46  let unicode = @support.parse_escaped_text(unicode_text.trim())47  let ascii = @support.parse_escaped_text(ascii_text.trim())48  let to_unicode = if unicode.text.is_empty() {49    source.text50  } else {51    unicode.text52  }53  let to_ascii = if ascii.text.is_empty() { to_unicode } else { ascii.text }54  let to_ascii_has_surrogate = if ascii.text.is_empty() {55    if unicode.text.is_empty() {56      source.has_surrogate57    } else {58      unicode.has_surrogate59    }60  } else {61    ascii.has_surrogate62  }63  let unicode_status = @support.parse_status(unicode_status_text).unwrap_or([])64  let ascii_status = @support.parse_status(ascii_status_text).unwrap_or(65    unicode_status,66  )67  if source.has_surrogate || to_ascii_has_surrogate {68    return None69  }70  Some({ line_number, source: source.text, to_ascii, status: ascii_status, })71}7273///|74fn flag_for_status(status : StringView) -> String? {75  match status {76    "B1" | "B2" | "B3" | "B4" | "B5" | "B6" => Some("check_bidi")77    "C1" | "C2" => Some("check_joiners")78    "V2" | "V3" => Some("check_hyphens")79    "V4" | "U1" => Some("use_std3_ascii_rules")80    "A4_1" | "A4_2" => Some("verify_dns_length")81    _ => None82  }83}8485///|86fn required_flags(statuses : ArrayView[String]) -> RequiredFlags {87  let mut check_bidi = false88  let mut check_joiners = false89  let mut check_hyphens = false90  let mut use_std3_ascii_rules = false91  let mut verify_dns_length = false92  for status in statuses {93    match flag_for_status(status) {94      Some("check_bidi") => check_bidi = true95      Some("check_joiners") => check_joiners = true96      Some("check_hyphens") => check_hyphens = true97      Some("use_std3_ascii_rules") => use_std3_ascii_rules = true98      Some("verify_dns_length") => verify_dns_length = true99      _ => ()100    }101  }102  {103    check_bidi,104    check_joiners,105    check_hyphens,106    use_std3_ascii_rules,107    verify_dns_length,108  }109}110111///|112fn RequiredFlags::any(self : RequiredFlags) -> Bool {113  self.check_bidi ||114  self.check_joiners ||115  self.check_hyphens ||116  self.use_std3_ascii_rules ||117  self.verify_dns_length118}119120///|121fn has_flag_independent_error(statuses : ArrayView[String]) -> Bool {122  statuses.any(status => flag_for_status(status) is None)123}124125///|126let no_flags : RequiredFlags = {127  check_bidi: false,128  check_joiners: false,129  check_hyphens: false,130  use_std3_ascii_rules: false,131  verify_dns_length: false,132}133134///|135fn call_to_ascii(136  source : String,137  flags : RequiredFlags,138) -> String raise @idna.IdnaError {139  @idna.to_ascii(140    source,141    use_std3_ascii_rules=flags.use_std3_ascii_rules,142    check_hyphens=flags.check_hyphens,143    check_bidi=flags.check_bidi,144    check_joiners=flags.check_joiners,145    verify_dns_length=flags.verify_dns_length,146  )147}148149///|150fn assert_success(case : TestCase) -> Unit raise {151  let result = call_to_ascii(case.source, no_flags)152  @debug.assert_eq(153    result,154    case.to_ascii,155    msg="IdnaTestV2.txt line \{case.line_number}: to_ascii mismatch",156  )157}158159///|160/// Assert that `to_ascii` raises for the case with the given flags. The try161/// body can only raise `IdnaError`, so any error escaping it is the expected162/// failure; a successful call falls through to `fail` with the line number.163fn assert_failure(case : TestCase, flags : RequiredFlags) -> Unit raise {164  try {165    let _ = call_to_ascii(case.source, flags)166  } catch {167    _ => return168  }169  fail("IdnaTestV2.txt line \{case.line_number}: expected to_ascii to fail")170}171172///|173async test "IdnaTestV2.txt conformance" {174  let content = @support.read_data_file("IdnaTestV2.txt")175  for line_number, raw_line in content.split("\n") {176    guard parse_test_line(raw_line, line_number + 1) is Some(case) else {177      continue178    }179    if case.status.is_empty() {180      assert_success(case)181      continue182    }183    let flags = required_flags(case.status)184    if flags.any() {185      assert_failure(case, flags)186    } else if has_flag_independent_error(case.status) {187      assert_failure(case, no_flags)188    } else {189      assert_success(case)190    }191  }192}193