tests/case_mapping_tests/case_mapping_test.mbt1///|2/// Full case mappings of one scalar value, read from the official Unicode3/// 18.0.0 files independently of the generator.4priv struct Expected {5 upper : Map[Int32, String]6 lower : Map[Int32, String]7 title : Map[Int32, String]8}910///|11fn mapping_string(field : StringView) -> String raise {12 guard @support.code_points_to_string(13 @support.parse_code_point_sequence(field),14 )15 is Some(text) else {16 fail("Invalid scalar in case mapping")17 }18 text19}2021///|22/// The simple mappings of UnicodeData.txt, replaced by the unconditional23/// full mappings of SpecialCasing.txt. These are the character-level mappings;24/// language-independent conditional string mappings are tested separately.25async fn read_expected() -> Expected {26 let expected = { upper: Map([]), lower: Map([]), title: Map([]), }27 let unicode_data = @support.read_data_file("UnicodeData.txt")28 for line in unicode_data.split("\n") {29 if line.is_empty() {30 continue31 }32 let fields = line.split(";").collect()33 guard fields.length() == 15 else { fail("Invalid UnicodeData record") }34 let cp = @support.parse_hex_int(fields[0])35 let upper = fields[12]36 let lower = fields[13]37 // An empty titlecase field means the uppercase mapping.38 let title = if fields[14].is_empty() { upper } else { fields[14] }39 if !upper.is_empty() {40 expected.upper[cp] = mapping_string(upper)41 }42 if !lower.is_empty() {43 expected.lower[cp] = mapping_string(lower)44 }45 if !title.is_empty() {46 expected.title[cp] = mapping_string(title)47 }48 }49 let special_casing = @support.read_data_file("SpecialCasing.txt")50 assert_true(special_casing.has_prefix("# SpecialCasing-18.0.0.txt"))51 let mut unconditional = 052 for raw_line in special_casing.split("\n") {53 let line = raw_line.split("#").next().unwrap().trim()54 if line.is_empty() {55 continue56 }57 guard line.split(";").collect() is [cp, lower, title, upper, condition, ..] else {58 fail("Invalid SpecialCasing record")59 }60 if !condition.trim().is_empty() {61 continue62 }63 let cp = @support.parse_hex_int(cp)64 expected.lower[cp] = mapping_string(lower)65 expected.title[cp] = mapping_string(title)66 expected.upper[cp] = mapping_string(upper)67 unconditional += 168 }69 assert_true(unconditional > 100)70 expected71}7273///|74async test "UnicodeData.txt and SpecialCasing.txt full mappings for every scalar" {75 let expected = read_expected()76 assert_true(expected.upper.length() > 1400)77 // All scalars in one string, to map runs of them at once.78 let all = StringBuilder()79 let all_upper = StringBuilder()80 let all_lower = StringBuilder()81 for cp in 0..<=0x10FFFF {82 if cp >= 0xD800 && cp <= 0xDFFF {83 continue84 }85 let c = cp.unsafe_to_char()86 let input = c.to_string()87 let upper = expected.upper.get(cp).unwrap_or(input)88 let lower = expected.lower.get(cp).unwrap_or(input)89 let title = expected.title.get(cp).unwrap_or(input)90 @debug.assert_eq(@ucd.to_uppercase(c), upper, msg="to_uppercase at \{cp}")91 @debug.assert_eq(@ucd.to_lowercase(c), lower, msg="to_lowercase at \{cp}")92 @debug.assert_eq(@ucd.to_titlecase(c), title, msg="to_titlecase at \{cp}")93 @debug.assert_eq(@ucd.uppercase(input), upper, msg="uppercase at \{cp}")94 @debug.assert_eq(@ucd.lowercase(input), lower, msg="lowercase at \{cp}")95 // Next to ASCII, which the string functions map separately.96 @debug.assert_eq(97 @ucd.uppercase("a\{input}Z"),98 "A\{upper}Z",99 msg="uppercase in context at \{cp}",100 )101 @debug.assert_eq(102 @ucd.lowercase("a\{input}Z"),103 "a\{lower}z",104 msg="lowercase in context at \{cp}",105 )106 all.write_string(input)107 all_upper.write_string(upper)108 all_lower.write_string(lower)109 }110 let all = all.to_string()111 assert_true(@ucd.uppercase(all) == all_upper.to_string())112 assert_true(@ucd.lowercase(all) == all_lower.to_string())113}114115///|116async test "SpecialCasing.txt language-independent conditional string mappings" {117 let unconditional = read_expected()118 let special_casing = @support.read_data_file("SpecialCasing.txt")119 let mut conditional = 0120 for raw_line in special_casing.split("\n") {121 let line = raw_line.split("#").next().unwrap().trim()122 if line.is_empty() {123 continue124 }125 guard line.split(";").collect() is [cp, lower, _, _, condition, ..] else {126 fail("Invalid SpecialCasing record")127 }128 let condition = condition.trim()129 if condition.is_empty() {130 continue131 }132 // Default casing does not apply Turkish, Azeri or Lithuanian tailorings.133 if condition =~ re"^(tr|az|lt)( |$)" {134 continue135 }136 guard condition == "Final_Sigma" else {137 fail("Untested language-independent casing condition: \{condition}")138 }139 let input = mapping_string(cp)140 let cp = @support.parse_hex_int(cp)141 let final_form = mapping_string(lower)142 let ordinary_form = unconditional.lower.get(cp).unwrap_or(input)143 // Prefixes and suffixes are already lowercase. The booleans encode the144 // contexts from Unicode Table 3-17, independently of the implementation.145 let prefixes = [146 ("", false),147 ("a", true),148 ("a\u{0301}", true),149 ("\u{0301}", false),150 ("\u{0345}", false),151 ("a\u{0345}", true),152 ("a ", false),153 ("a0", false),154 ("a中", false),155 ("a'", true),156 ("a\u{1D185}", true),157 ("a\u{E0100}", true),158 ("\u{10428}", true),159 ("ª", true),160 ("a\u{200D}", true),161 ]162 let suffixes = [163 ("", true),164 ("a", false),165 ("\u{0301}", true),166 ("\u{0301}a", false),167 ("\u{0345}", true),168 ("\u{0345}a", false),169 (" a", true),170 ("0a", true),171 ("中a", true),172 ("'a", false),173 ("\u{1D185}a", false),174 ("\u{E0100}a", false),175 ("\u{10428}", false),176 ("ª", false),177 ("\u{200D}", true),178 ]179 for (prefix, cased_before) in prefixes {180 for (suffix, no_cased_after) in suffixes {181 let mapped = if cased_before && no_cased_after {182 final_form183 } else {184 ordinary_form185 }186 @debug.assert_eq(187 @ucd.lowercase("\{prefix}\{input}\{suffix}"),188 "\{prefix}\{mapped}\{suffix}",189 msg="\{condition}: prefix=\{Repr(prefix)}, suffix=\{Repr(suffix)}",190 )191 }192 }193 conditional += 1194 }195 assert_true(conditional > 0)196}197198///|199test "lowercase: long case-ignorable runs between successive sigmas" {200 let marks = "\u{0345}\u{E0100}".repeat(2048)201 @debug.assert_eq(202 @ucd.lowercase("AΣ\{marks}Σ\{marks}"),203 "aσ\{marks}ς\{marks}",204 )205}206
///|
/// Full case mappings of one scalar value, read from the official Unicode
/// 18.0.0 files independently of the generator.
priv struct Expected {
upper : Map[Int32, String]
lower : Map[Int32, String]
title : Map[Int32, String]
}
///|
fn mapping_string(field : StringView) -> String raise {
guard @support.code_points_to_string(
@support.parse_code_point_sequence(field),
)
is Some(text) else {
fail("Invalid scalar in case mapping")
}
text
}
///|
/// The simple mappings of UnicodeData.txt, replaced by the unconditional
/// full mappings of SpecialCasing.txt. These are the character-level mappings;
/// language-independent conditional string mappings are tested separately.
async fn read_expected() -> Expected {
let expected = { upper: Map([]), lower: Map([]), title: Map([]), }
let unicode_data = @support.read_data_file("UnicodeData.txt")
for line in unicode_data.split("\n") {
if line.is_empty() {
continue
}
let fields = line.split(";").collect()
guard fields.length() == 15 else { fail("Invalid UnicodeData record") }
let cp = @support.parse_hex_int(fields[0])
let upper = fields[12]
let lower = fields[13]
// An empty titlecase field means the uppercase mapping.
let title = if fields[14].is_empty() { upper } else { fields[14] }
if !upper.is_empty() {
expected.upper[cp] = mapping_string(upper)
}
if !lower.is_empty() {
expected.lower[cp] = mapping_string(lower)
}
if !title.is_empty() {
expected.title[cp] = mapping_string(title)
}
}
let special_casing = @support.read_data_file("SpecialCasing.txt")
assert_true(special_casing.has_prefix("# SpecialCasing-18.0.0.txt"))
let mut unconditional = 0
for raw_line in special_casing.split("\n") {
let line = raw_line.split("#").next().unwrap().trim()
if line.is_empty() {
continue
}
guard line.split(";").collect() is [cp, lower, title, upper, condition, ..] else {
fail("Invalid SpecialCasing record")
}
if !condition.trim().is_empty() {
continue
}
let cp = @support.parse_hex_int(cp)
expected.lower[cp] = mapping_string(lower)
expected.title[cp] = mapping_string(title)
expected.upper[cp] = mapping_string(upper)
unconditional += 1
}
assert_true(unconditional > 100)
expected
}
///|
async test "UnicodeData.txt and SpecialCasing.txt full mappings for every scalar" {
let expected = read_expected()
assert_true(expected.upper.length() > 1400)
// All scalars in one string, to map runs of them at once.
let all = StringBuilder()
let all_upper = StringBuilder()
let all_lower = StringBuilder()
for cp in 0..<=0x10FFFF {
if cp >= 0xD800 && cp <= 0xDFFF {
continue
}
let c = cp.unsafe_to_char()
let input = c.to_string()
let upper = expected.upper.get(cp).unwrap_or(input)
let lower = expected.lower.get(cp).unwrap_or(input)
let title = expected.title.get(cp).unwrap_or(input)
@debug.assert_eq(@ucd.to_uppercase(c), upper, msg="to_uppercase at \{cp}")
@debug.assert_eq(@ucd.to_lowercase(c), lower, msg="to_lowercase at \{cp}")
@debug.assert_eq(@ucd.to_titlecase(c), title, msg="to_titlecase at \{cp}")
@debug.assert_eq(@ucd.uppercase(input), upper, msg="uppercase at \{cp}")
@debug.assert_eq(@ucd.lowercase(input), lower, msg="lowercase at \{cp}")
// Next to ASCII, which the string functions map separately.
@debug.assert_eq(
@ucd.uppercase("a\{input}Z"),
"A\{upper}Z",
msg="uppercase in context at \{cp}",
)
@debug.assert_eq(
@ucd.lowercase("a\{input}Z"),
"a\{lower}z",
msg="lowercase in context at \{cp}",
)
all.write_string(input)
all_upper.write_string(upper)
all_lower.write_string(lower)
}
let all = all.to_string()
assert_true(@ucd.uppercase(all) == all_upper.to_string())
assert_true(@ucd.lowercase(all) == all_lower.to_string())
}
///|
async test "SpecialCasing.txt language-independent conditional string mappings" {
let unconditional = read_expected()
let special_casing = @support.read_data_file("SpecialCasing.txt")
let mut conditional = 0
for raw_line in special_casing.split("\n") {
let line = raw_line.split("#").next().unwrap().trim()
if line.is_empty() {
continue
}
guard line.split(";").collect() is [cp, lower, _, _, condition, ..] else {
fail("Invalid SpecialCasing record")
}
let condition = condition.trim()
if condition.is_empty() {
continue
}
// Default casing does not apply Turkish, Azeri or Lithuanian tailorings.
if condition =~ re"^(tr|az|lt)( |$)" {
continue
}
guard condition == "Final_Sigma" else {
fail("Untested language-independent casing condition: \{condition}")
}
let input = mapping_string(cp)
let cp = @support.parse_hex_int(cp)
let final_form = mapping_string(lower)
let ordinary_form = unconditional.lower.get(cp).unwrap_or(input)
// Prefixes and suffixes are already lowercase. The booleans encode the
// contexts from Unicode Table 3-17, independently of the implementation.
let prefixes = [
("", false),
("a", true),
("a\u{0301}", true),
("\u{0301}", false),
("\u{0345}", false),
("a\u{0345}", true),
("a ", false),
("a0", false),
("a中", false),
("a'", true),
("a\u{1D185}", true),
("a\u{E0100}", true),
("\u{10428}", true),
("ª", true),
("a\u{200D}", true),
]
let suffixes = [
("", true),
("a", false),
("\u{0301}", true),
("\u{0301}a", false),
("\u{0345}", true),
("\u{0345}a", false),
(" a", true),
("0a", true),
("中a", true),
("'a", false),
("\u{1D185}a", false),
("\u{E0100}a", false),
("\u{10428}", false),
("ª", false),
("\u{200D}", true),
]
for (prefix, cased_before) in prefixes {
for (suffix, no_cased_after) in suffixes {
let mapped = if cased_before && no_cased_after {
final_form
} else {
ordinary_form
}
@debug.assert_eq(
@ucd.lowercase("\{prefix}\{input}\{suffix}"),
"\{prefix}\{mapped}\{suffix}",
msg="\{condition}: prefix=\{Repr(prefix)}, suffix=\{Repr(suffix)}",
)
}
}
conditional += 1
}
assert_true(conditional > 0)
}
///|
test "lowercase: long case-ignorable runs between successive sigmas" {
let marks = "\u{0345}\u{E0100}".repeat(2048)
@debug.assert_eq(
@ucd.lowercase("AΣ\{marks}Σ\{marks}"),
"aσ\{marks}ς\{marks}",
)
}