tests/case_folding_tests/case_folding_test.mbt1///|2/// Read the official Unicode 18.0.0 files independently of the generator.3fn mapping_string(field : StringView) -> String raise {4 guard @support.code_points_to_string(5 @support.parse_code_point_sequence(field),6 )7 is Some(text) else {8 fail("Invalid scalar in folding mapping")9 }10 text11}1213///|14async test "CaseFolding.txt full default mappings and identity for every scalar" {15 let expected : Map[Int32, String] = Map([])16 let content = @support.read_data_file("CaseFolding.txt")17 assert_true(content.has_prefix("# CaseFolding-18.0.0.txt"))18 for raw_line in content.split("\n") {19 let line = raw_line.split("#").next().unwrap().trim()20 if line.is_empty() {21 continue22 }23 guard line.split(";").collect() is [cp, status, target, ..] else {24 fail("Invalid CaseFolding record")25 }26 if status.trim() == "C" || status.trim() == "F" {27 expected[@support.parse_hex_int(cp)] = mapping_string(target)28 }29 }30 assert_true(expected.length() > 1500)31 for cp in 0..<=0x10FFFF {32 if cp >= 0xD800 && cp <= 0xDFFF {33 continue34 }35 let input = cp.unsafe_to_char().to_string()36 let target = expected.get(cp).unwrap_or(input)37 let actual = @casefold.case_fold(input)38 @debug.assert_eq(actual, target, msg="CaseFolding mismatch at \{cp}")39 @debug.assert_eq(40 @casefold.case_fold(actual),41 actual,42 msg="CaseFolding not idempotent at \{cp}",43 )44 // Matching walks code points instead of folding whole strings.45 assert_true(46 @casefold.caseless_match("A\{input}b", "a\{target}B"),47 msg="caseless_match rejects its folding at \{cp}",48 )49 assert_true(50 !@casefold.caseless_match("\{input}a", "\{target}b"),51 msg="caseless_match ignores a difference after \{cp}",52 )53 }54}5556///|57async test "DerivedNormalizationProps.txt NFKC_CF for every scalar" {58 let expected : Map[Int32, String] = Map([])59 let content = @support.read_data_file("DerivedNormalizationProps.txt")60 assert_true(content.has_prefix("# DerivedNormalizationProps-18.0.0.txt"))61 for raw_line in content.split("\n") {62 let line = raw_line.split("#").next().unwrap().trim()63 if line.is_empty() {64 continue65 }66 guard line.split(";").collect() is [range, property, .. fields] else {67 fail("Invalid derived normalization record")68 }69 if property.trim() != "NFKC_CF" {70 continue71 }72 guard fields is [target, ..] else { fail("Missing NFKC_CF target") }73 let target = mapping_string(target)74 let (first, last) = if range.trim().split_once("..") is Some((a, b)) {75 (@support.parse_hex_int(a), @support.parse_hex_int(b))76 } else {77 let cp = @support.parse_hex_int(range)78 (cp, cp)79 }80 for cp in first..<=last {81 expected[cp] = target82 }83 }84 assert_true(expected.length() > 10000)85 for cp in 0..<=0x10FFFF {86 if cp >= 0xD800 && cp <= 0xDFFF {87 continue88 }89 let input = cp.unsafe_to_char().to_string()90 let target = expected.get(cp).unwrap_or(input)91 let actual = @casefold.nfkc_casefold(input)92 @debug.assert_eq(actual, target, msg="NFKC_CF mismatch at \{cp}")93 @debug.assert_eq(94 @casefold.nfkc_casefold(actual),95 actual,96 msg="NFKC_CF not idempotent at \{cp}",97 )98 assert_true(@normalization.is_normalized(actual, @normalization.NFC))99 }100}101102///|103async test "canonical caseless keys respect NormalizationTest canonical equivalents" {104 for raw_line in @support.read_data_file("NormalizationTest.txt").split("\n") {105 let line = raw_line.trim()106 if line.is_empty() || line.has_prefix("#") || line.has_prefix("@") {107 continue108 }109 guard line.split(";").collect() is [c1, c2, c3, c4, c5, ..] else {110 fail("Invalid NormalizationTest record")111 }112 let first = mapping_string(c1)113 let composed = mapping_string(c2)114 let decomposed = mapping_string(c3)115 assert_true(@casefold.canonical_caseless_match(first, composed))116 assert_true(@casefold.canonical_caseless_match(first, decomposed))117 assert_true(118 @casefold.canonical_caseless_match(mapping_string(c4), mapping_string(c5)),119 )120 assert_true(@casefold.identifier_caseless_match(first, decomposed))121 }122}123
///|
/// Read the official Unicode 18.0.0 files independently of the generator.
fn mapping_string(field : StringView) -> String raise {
guard @support.code_points_to_string(
@support.parse_code_point_sequence(field),
)
is Some(text) else {
fail("Invalid scalar in folding mapping")
}
text
}
///|
async test "CaseFolding.txt full default mappings and identity for every scalar" {
let expected : Map[Int32, String] = Map([])
let content = @support.read_data_file("CaseFolding.txt")
assert_true(content.has_prefix("# CaseFolding-18.0.0.txt"))
for raw_line in content.split("\n") {
let line = raw_line.split("#").next().unwrap().trim()
if line.is_empty() {
continue
}
guard line.split(";").collect() is [cp, status, target, ..] else {
fail("Invalid CaseFolding record")
}
if status.trim() == "C" || status.trim() == "F" {
expected[@support.parse_hex_int(cp)] = mapping_string(target)
}
}
assert_true(expected.length() > 1500)
for cp in 0..<=0x10FFFF {
if cp >= 0xD800 && cp <= 0xDFFF {
continue
}
let input = cp.unsafe_to_char().to_string()
let target = expected.get(cp).unwrap_or(input)
let actual = @casefold.case_fold(input)
@debug.assert_eq(actual, target, msg="CaseFolding mismatch at \{cp}")
@debug.assert_eq(
@casefold.case_fold(actual),
actual,
msg="CaseFolding not idempotent at \{cp}",
)
// Matching walks code points instead of folding whole strings.
assert_true(
@casefold.caseless_match("A\{input}b", "a\{target}B"),
msg="caseless_match rejects its folding at \{cp}",
)
assert_true(
!@casefold.caseless_match("\{input}a", "\{target}b"),
msg="caseless_match ignores a difference after \{cp}",
)
}
}
///|
async test "DerivedNormalizationProps.txt NFKC_CF for every scalar" {
let expected : Map[Int32, String] = Map([])
let content = @support.read_data_file("DerivedNormalizationProps.txt")
assert_true(content.has_prefix("# DerivedNormalizationProps-18.0.0.txt"))
for raw_line in content.split("\n") {
let line = raw_line.split("#").next().unwrap().trim()
if line.is_empty() {
continue
}
guard line.split(";").collect() is [range, property, .. fields] else {
fail("Invalid derived normalization record")
}
if property.trim() != "NFKC_CF" {
continue
}
guard fields is [target, ..] else { fail("Missing NFKC_CF target") }
let target = mapping_string(target)
let (first, last) = if range.trim().split_once("..") is Some((a, b)) {
(@support.parse_hex_int(a), @support.parse_hex_int(b))
} else {
let cp = @support.parse_hex_int(range)
(cp, cp)
}
for cp in first..<=last {
expected[cp] = target
}
}
assert_true(expected.length() > 10000)
for cp in 0..<=0x10FFFF {
if cp >= 0xD800 && cp <= 0xDFFF {
continue
}
let input = cp.unsafe_to_char().to_string()
let target = expected.get(cp).unwrap_or(input)
let actual = @casefold.nfkc_casefold(input)
@debug.assert_eq(actual, target, msg="NFKC_CF mismatch at \{cp}")
@debug.assert_eq(
@casefold.nfkc_casefold(actual),
actual,
msg="NFKC_CF not idempotent at \{cp}",
)
assert_true(@normalization.is_normalized(actual, @normalization.NFC))
}
}
///|
async test "canonical caseless keys respect NormalizationTest canonical equivalents" {
for raw_line in @support.read_data_file("NormalizationTest.txt").split("\n") {
let line = raw_line.trim()
if line.is_empty() || line.has_prefix("#") || line.has_prefix("@") {
continue
}
guard line.split(";").collect() is [c1, c2, c3, c4, c5, ..] else {
fail("Invalid NormalizationTest record")
}
let first = mapping_string(c1)
let composed = mapping_string(c2)
let decomposed = mapping_string(c3)
assert_true(@casefold.canonical_caseless_match(first, composed))
assert_true(@casefold.canonical_caseless_match(first, decomposed))
assert_true(
@casefold.canonical_caseless_match(mapping_string(c4), mapping_string(c5)),
)
assert_true(@casefold.identifier_caseless_match(first, decomposed))
}
}