tests/normalization_tests/normalization_test.mbt1///|2/// Unicode normalization conformance tests, read directly from3/// NormalizationTest.txt (Unicode 18.0.0).4/// Source: https://www.unicode.org/Public/18.0.0/ucd/NormalizationTest.txt5///6/// Each line contains five strings (c1..c5) that must satisfy:7/// - NFC: c2 == NFC(c1) == NFC(c2) == NFC(c3)8/// - NFD: c3 == NFD(c1) == NFD(c2) == NFD(c3)9/// - NFKC: c4 == NFKC(c1) == NFKC(c2) == NFKC(c3) == NFKC(c4) == NFKC(c5)10/// - NFKD: c5 == NFKD(c1) == NFKD(c2) == NFKD(c3) == NFKD(c4) == NFKD(c5)1112///|13struct NormalizationCase {14 line_number : Int3215 c1 : String16 c2 : String17 c3 : String18 c4 : String19 c5 : String20}2122///|23fn parse_field(field : StringView) -> String? raise {24 @support.code_points_to_string(@support.parse_code_point_sequence(field))25}2627///|28fn parse_case(29 line : StringView,30 line_number : Int32,31) -> NormalizationCase? raise {32 let line = line.trim()33 if line.is_empty() || line.has_prefix("#") || line.has_prefix("@") {34 return None35 }36 guard line.split(";").collect() is [c1, c2, c3, c4, c5, ..] else {37 return None38 }39 guard parse_field(c1) is Some(c1) else { return None }40 guard parse_field(c2) is Some(c2) else { return None }41 guard parse_field(c3) is Some(c3) else { return None }42 guard parse_field(c4) is Some(c4) else { return None }43 guard parse_field(c5) is Some(c5) else { return None }44 Some({ line_number, c1, c2, c3, c4, c5, })45}4647///|48fn parse_cases(content : String) -> Array[NormalizationCase] raise {49 let cases : Array[NormalizationCase] = []50 for line_number, raw_line in content.split("\n") {51 if parse_case(raw_line, line_number + 1) is Some(case) {52 cases.push(case)53 }54 }55 cases56}5758///|59async test "NormalizationTest.txt - NFC" {60 for case in parse_cases(@support.read_data_file("NormalizationTest.txt")) {61 @debug.assert_eq(62 @normalization.nfc(case.c1),63 case.c2,64 msg="NormalizationTest.txt line \{case.line_number}: NFC(c1) != c2",65 )66 @debug.assert_eq(67 @normalization.nfc(case.c2),68 case.c2,69 msg="NormalizationTest.txt line \{case.line_number}: NFC(c2) != c2",70 )71 @debug.assert_eq(72 @normalization.nfc(case.c3),73 case.c2,74 msg="NormalizationTest.txt line \{case.line_number}: NFC(c3) != c2",75 )76 }77}7879///|80async test "NormalizationTest.txt - NFD" {81 for case in parse_cases(@support.read_data_file("NormalizationTest.txt")) {82 @debug.assert_eq(83 @normalization.nfd(case.c1),84 case.c3,85 msg="NormalizationTest.txt line \{case.line_number}: NFD(c1) != c3",86 )87 @debug.assert_eq(88 @normalization.nfd(case.c2),89 case.c3,90 msg="NormalizationTest.txt line \{case.line_number}: NFD(c2) != c3",91 )92 @debug.assert_eq(93 @normalization.nfd(case.c3),94 case.c3,95 msg="NormalizationTest.txt line \{case.line_number}: NFD(c3) != c3",96 )97 }98}99100///|101async test "NormalizationTest.txt - NFKC" {102 for case in parse_cases(@support.read_data_file("NormalizationTest.txt")) {103 @debug.assert_eq(104 @normalization.nfkc(case.c1),105 case.c4,106 msg="NormalizationTest.txt line \{case.line_number}: NFKC(c1) != c4",107 )108 @debug.assert_eq(109 @normalization.nfkc(case.c2),110 case.c4,111 msg="NormalizationTest.txt line \{case.line_number}: NFKC(c2) != c4",112 )113 @debug.assert_eq(114 @normalization.nfkc(case.c3),115 case.c4,116 msg="NormalizationTest.txt line \{case.line_number}: NFKC(c3) != c4",117 )118 @debug.assert_eq(119 @normalization.nfkc(case.c4),120 case.c4,121 msg="NormalizationTest.txt line \{case.line_number}: NFKC(c4) != c4",122 )123 @debug.assert_eq(124 @normalization.nfkc(case.c5),125 case.c4,126 msg="NormalizationTest.txt line \{case.line_number}: NFKC(c5) != c4",127 )128 }129}130131///|132async test "NormalizationTest.txt - NFKD" {133 for case in parse_cases(@support.read_data_file("NormalizationTest.txt")) {134 @debug.assert_eq(135 @normalization.nfkd(case.c1),136 case.c5,137 msg="NormalizationTest.txt line \{case.line_number}: NFKD(c1) != c5",138 )139 @debug.assert_eq(140 @normalization.nfkd(case.c2),141 case.c5,142 msg="NormalizationTest.txt line \{case.line_number}: NFKD(c2) != c5",143 )144 @debug.assert_eq(145 @normalization.nfkd(case.c3),146 case.c5,147 msg="NormalizationTest.txt line \{case.line_number}: NFKD(c3) != c5",148 )149 @debug.assert_eq(150 @normalization.nfkd(case.c4),151 case.c5,152 msg="NormalizationTest.txt line \{case.line_number}: NFKD(c4) != c5",153 )154 @debug.assert_eq(155 @normalization.nfkd(case.c5),156 case.c5,157 msg="NormalizationTest.txt line \{case.line_number}: NFKD(c5) != c5",158 )159 }160}161162///|163async test "NormalizationTest.txt - quick check and copied span boundaries" {164 for case in parse_cases(@support.read_data_file("NormalizationTest.txt")) {165 let inputs = [case.c1, case.c2, case.c3, case.c4, case.c5]166 for167 form in [168 @normalization.NFC,169 @normalization.NFD,170 @normalization.NFKC,171 @normalization.NFKD,172 ] {173 for i, input in inputs {174 let expected = match form {175 NFC => if i < 3 { case.c2 } else { case.c4 }176 NFD => if i < 3 { case.c3 } else { case.c5 }177 NFKC => case.c4178 NFKD => case.c5179 }180 @debug.assert_eq(181 @normalization.is_normalized(input, form),182 input == expected,183 msg="NormalizationTest.txt line \{case.line_number}: is_normalized",184 )185 @debug.assert_eq(186 @normalization.normalize("😀prefix|\{input}|suffix😀", form),187 "😀prefix|\{expected}|suffix😀",188 msg="NormalizationTest.txt line \{case.line_number}: copied boundaries",189 )190 }191 }192 }193}194
///|
/// Unicode normalization conformance tests, read directly from
/// NormalizationTest.txt (Unicode 18.0.0).
/// Source: https://www.unicode.org/Public/18.0.0/ucd/NormalizationTest.txt
///
/// Each line contains five strings (c1..c5) that must satisfy:
/// - NFC: c2 == NFC(c1) == NFC(c2) == NFC(c3)
/// - NFD: c3 == NFD(c1) == NFD(c2) == NFD(c3)
/// - NFKC: c4 == NFKC(c1) == NFKC(c2) == NFKC(c3) == NFKC(c4) == NFKC(c5)
/// - NFKD: c5 == NFKD(c1) == NFKD(c2) == NFKD(c3) == NFKD(c4) == NFKD(c5)
///|
struct NormalizationCase {
line_number : Int32
c1 : String
c2 : String
c3 : String
c4 : String
c5 : String
}
///|
fn parse_field(field : StringView) -> String? raise {
@support.code_points_to_string(@support.parse_code_point_sequence(field))
}
///|
fn parse_case(
line : StringView,
line_number : Int32,
) -> NormalizationCase? raise {
let line = line.trim()
if line.is_empty() || line.has_prefix("#") || line.has_prefix("@") {
return None
}
guard line.split(";").collect() is [c1, c2, c3, c4, c5, ..] else {
return None
}
guard parse_field(c1) is Some(c1) else { return None }
guard parse_field(c2) is Some(c2) else { return None }
guard parse_field(c3) is Some(c3) else { return None }
guard parse_field(c4) is Some(c4) else { return None }
guard parse_field(c5) is Some(c5) else { return None }
Some({ line_number, c1, c2, c3, c4, c5, })
}
///|
fn parse_cases(content : String) -> Array[NormalizationCase] raise {
let cases : Array[NormalizationCase] = []
for line_number, raw_line in content.split("\n") {
if parse_case(raw_line, line_number + 1) is Some(case) {
cases.push(case)
}
}
cases
}
///|
async test "NormalizationTest.txt - NFC" {
for case in parse_cases(@support.read_data_file("NormalizationTest.txt")) {
@debug.assert_eq(
@normalization.nfc(case.c1),
case.c2,
msg="NormalizationTest.txt line \{case.line_number}: NFC(c1) != c2",
)
@debug.assert_eq(
@normalization.nfc(case.c2),
case.c2,
msg="NormalizationTest.txt line \{case.line_number}: NFC(c2) != c2",
)
@debug.assert_eq(
@normalization.nfc(case.c3),
case.c2,
msg="NormalizationTest.txt line \{case.line_number}: NFC(c3) != c2",
)
}
}
///|
async test "NormalizationTest.txt - NFD" {
for case in parse_cases(@support.read_data_file("NormalizationTest.txt")) {
@debug.assert_eq(
@normalization.nfd(case.c1),
case.c3,
msg="NormalizationTest.txt line \{case.line_number}: NFD(c1) != c3",
)
@debug.assert_eq(
@normalization.nfd(case.c2),
case.c3,
msg="NormalizationTest.txt line \{case.line_number}: NFD(c2) != c3",
)
@debug.assert_eq(
@normalization.nfd(case.c3),
case.c3,
msg="NormalizationTest.txt line \{case.line_number}: NFD(c3) != c3",
)
}
}
///|
async test "NormalizationTest.txt - NFKC" {
for case in parse_cases(@support.read_data_file("NormalizationTest.txt")) {
@debug.assert_eq(
@normalization.nfkc(case.c1),
case.c4,
msg="NormalizationTest.txt line \{case.line_number}: NFKC(c1) != c4",
)
@debug.assert_eq(
@normalization.nfkc(case.c2),
case.c4,
msg="NormalizationTest.txt line \{case.line_number}: NFKC(c2) != c4",
)
@debug.assert_eq(
@normalization.nfkc(case.c3),
case.c4,
msg="NormalizationTest.txt line \{case.line_number}: NFKC(c3) != c4",
)
@debug.assert_eq(
@normalization.nfkc(case.c4),
case.c4,
msg="NormalizationTest.txt line \{case.line_number}: NFKC(c4) != c4",
)
@debug.assert_eq(
@normalization.nfkc(case.c5),
case.c4,
msg="NormalizationTest.txt line \{case.line_number}: NFKC(c5) != c4",
)
}
}
///|
async test "NormalizationTest.txt - NFKD" {
for case in parse_cases(@support.read_data_file("NormalizationTest.txt")) {
@debug.assert_eq(
@normalization.nfkd(case.c1),
case.c5,
msg="NormalizationTest.txt line \{case.line_number}: NFKD(c1) != c5",
)
@debug.assert_eq(
@normalization.nfkd(case.c2),
case.c5,
msg="NormalizationTest.txt line \{case.line_number}: NFKD(c2) != c5",
)
@debug.assert_eq(
@normalization.nfkd(case.c3),
case.c5,
msg="NormalizationTest.txt line \{case.line_number}: NFKD(c3) != c5",
)
@debug.assert_eq(
@normalization.nfkd(case.c4),
case.c5,
msg="NormalizationTest.txt line \{case.line_number}: NFKD(c4) != c5",
)
@debug.assert_eq(
@normalization.nfkd(case.c5),
case.c5,
msg="NormalizationTest.txt line \{case.line_number}: NFKD(c5) != c5",
)
}
}
///|
async test "NormalizationTest.txt - quick check and copied span boundaries" {
for case in parse_cases(@support.read_data_file("NormalizationTest.txt")) {
let inputs = [case.c1, case.c2, case.c3, case.c4, case.c5]
for
form in [
@normalization.NFC,
@normalization.NFD,
@normalization.NFKC,
@normalization.NFKD,
] {
for i, input in inputs {
let expected = match form {
NFC => if i < 3 { case.c2 } else { case.c4 }
NFD => if i < 3 { case.c3 } else { case.c5 }
NFKC => case.c4
NFKD => case.c5
}
@debug.assert_eq(
@normalization.is_normalized(input, form),
input == expected,
msg="NormalizationTest.txt line \{case.line_number}: is_normalized",
)
@debug.assert_eq(
@normalization.normalize("😀prefix|\{input}|suffix😀", form),
"😀prefix|\{expected}|suffix😀",
msg="NormalizationTest.txt line \{case.line_number}: copied boundaries",
)
}
}
}
}