unicode

Branch: main

Switch branches or tags

Branch: main
Go to file t

413 files indexed

History
Code

tests/case_mapping_tests/case_mapping_test.mbt
1///|2/// Full case mappings of one scalar value, read from the official Unicode3/// 18.0.0 files independently of the generator.4priv struct Expected {5  upper : Map[Int32, String]6  lower : Map[Int32, String]7  title : Map[Int32, String]8}910///|11fn mapping_string(field : StringView) -> String raise {12  guard @support.code_points_to_string(13      @support.parse_code_point_sequence(field),14    )15    is Some(text) else {16    fail("Invalid scalar in case mapping")17  }18  text19}2021///|22/// The simple mappings of UnicodeData.txt, replaced by the unconditional23/// full mappings of SpecialCasing.txt. These are the character-level mappings;24/// language-independent conditional string mappings are tested separately.25async fn read_expected() -> Expected {26  let expected = { upper: Map([]), lower: Map([]), title: Map([]), }27  let unicode_data = @support.read_data_file("UnicodeData.txt")28  for line in unicode_data.split("\n") {29    if line.is_empty() {30      continue31    }32    let fields = line.split(";").collect()33    guard fields.length() == 15 else { fail("Invalid UnicodeData record") }34    let cp = @support.parse_hex_int(fields[0])35    let upper = fields[12]36    let lower = fields[13]37    // An empty titlecase field means the uppercase mapping.38    let title = if fields[14].is_empty() { upper } else { fields[14] }39    if !upper.is_empty() {40      expected.upper[cp] = mapping_string(upper)41    }42    if !lower.is_empty() {43      expected.lower[cp] = mapping_string(lower)44    }45    if !title.is_empty() {46      expected.title[cp] = mapping_string(title)47    }48  }49  let special_casing = @support.read_data_file("SpecialCasing.txt")50  assert_true(special_casing.has_prefix("# SpecialCasing-18.0.0.txt"))51  let mut unconditional = 052  for raw_line in special_casing.split("\n") {53    let line = raw_line.split("#").next().unwrap().trim()54    if line.is_empty() {55      continue56    }57    guard line.split(";").collect() is [cp, lower, title, upper, condition, ..] else {58      fail("Invalid SpecialCasing record")59    }60    if !condition.trim().is_empty() {61      continue62    }63    let cp = @support.parse_hex_int(cp)64    expected.lower[cp] = mapping_string(lower)65    expected.title[cp] = mapping_string(title)66    expected.upper[cp] = mapping_string(upper)67    unconditional += 168  }69  assert_true(unconditional > 100)70  expected71}7273///|74async test "UnicodeData.txt and SpecialCasing.txt full mappings for every scalar" {75  let expected = read_expected()76  assert_true(expected.upper.length() > 1400)77  // All scalars in one string, to map runs of them at once.78  let all = StringBuilder()79  let all_upper = StringBuilder()80  let all_lower = StringBuilder()81  for cp in 0..<=0x10FFFF {82    if cp >= 0xD800 && cp <= 0xDFFF {83      continue84    }85    let c = cp.unsafe_to_char()86    let input = c.to_string()87    let upper = expected.upper.get(cp).unwrap_or(input)88    let lower = expected.lower.get(cp).unwrap_or(input)89    let title = expected.title.get(cp).unwrap_or(input)90    @debug.assert_eq(@ucd.to_uppercase(c), upper, msg="to_uppercase at \{cp}")91    @debug.assert_eq(@ucd.to_lowercase(c), lower, msg="to_lowercase at \{cp}")92    @debug.assert_eq(@ucd.to_titlecase(c), title, msg="to_titlecase at \{cp}")93    @debug.assert_eq(@ucd.uppercase(input), upper, msg="uppercase at \{cp}")94    @debug.assert_eq(@ucd.lowercase(input), lower, msg="lowercase at \{cp}")95    // Next to ASCII, which the string functions map separately.96    @debug.assert_eq(97      @ucd.uppercase("a\{input}Z"),98      "A\{upper}Z",99      msg="uppercase in context at \{cp}",100    )101    @debug.assert_eq(102      @ucd.lowercase("a\{input}Z"),103      "a\{lower}z",104      msg="lowercase in context at \{cp}",105    )106    all.write_string(input)107    all_upper.write_string(upper)108    all_lower.write_string(lower)109  }110  let all = all.to_string()111  assert_true(@ucd.uppercase(all) == all_upper.to_string())112  assert_true(@ucd.lowercase(all) == all_lower.to_string())113}114115///|116async test "SpecialCasing.txt language-independent conditional string mappings" {117  let unconditional = read_expected()118  let special_casing = @support.read_data_file("SpecialCasing.txt")119  let mut conditional = 0120  for raw_line in special_casing.split("\n") {121    let line = raw_line.split("#").next().unwrap().trim()122    if line.is_empty() {123      continue124    }125    guard line.split(";").collect() is [cp, lower, _, _, condition, ..] else {126      fail("Invalid SpecialCasing record")127    }128    let condition = condition.trim()129    if condition.is_empty() {130      continue131    }132    // Default casing does not apply Turkish, Azeri or Lithuanian tailorings.133    if condition =~ re"^(tr|az|lt)( |$)" {134      continue135    }136    guard condition == "Final_Sigma" else {137      fail("Untested language-independent casing condition: \{condition}")138    }139    let input = mapping_string(cp)140    let cp = @support.parse_hex_int(cp)141    let final_form = mapping_string(lower)142    let ordinary_form = unconditional.lower.get(cp).unwrap_or(input)143    // Prefixes and suffixes are already lowercase. The booleans encode the144    // contexts from Unicode Table 3-17, independently of the implementation.145    let prefixes = [146      ("", false),147      ("a", true),148      ("a\u{0301}", true),149      ("\u{0301}", false),150      ("\u{0345}", false),151      ("a\u{0345}", true),152      ("a ", false),153      ("a0", false),154      ("a中", false),155      ("a'", true),156      ("a\u{1D185}", true),157      ("a\u{E0100}", true),158      ("\u{10428}", true),159      ("ª", true),160      ("a\u{200D}", true),161    ]162    let suffixes = [163      ("", true),164      ("a", false),165      ("\u{0301}", true),166      ("\u{0301}a", false),167      ("\u{0345}", true),168      ("\u{0345}a", false),169      (" a", true),170      ("0a", true),171      ("中a", true),172      ("'a", false),173      ("\u{1D185}a", false),174      ("\u{E0100}a", false),175      ("\u{10428}", false),176      ("ª", false),177      ("\u{200D}", true),178    ]179    for (prefix, cased_before) in prefixes {180      for (suffix, no_cased_after) in suffixes {181        let mapped = if cased_before && no_cased_after {182          final_form183        } else {184          ordinary_form185        }186        @debug.assert_eq(187          @ucd.lowercase("\{prefix}\{input}\{suffix}"),188          "\{prefix}\{mapped}\{suffix}",189          msg="\{condition}: prefix=\{Repr(prefix)}, suffix=\{Repr(suffix)}",190        )191      }192    }193    conditional += 1194  }195  assert_true(conditional > 0)196}197198///|199test "lowercase: long case-ignorable runs between successive sigmas" {200  let marks = "\u{0345}\u{E0100}".repeat(2048)201  @debug.assert_eq(202    @ucd.lowercase("AΣ\{marks}Σ\{marks}"),203    "aσ\{marks}ς\{marks}",204  )205}206