unicode

Branch: main

Switch branches or tags

Branch: main
Go to file t

413 files indexed

History
Code

tests/case_folding_tests/case_folding_test.mbt
1///|2/// Read the official Unicode 18.0.0 files independently of the generator.3fn mapping_string(field : StringView) -> String raise {4  guard @support.code_points_to_string(5      @support.parse_code_point_sequence(field),6    )7    is Some(text) else {8    fail("Invalid scalar in folding mapping")9  }10  text11}1213///|14async test "CaseFolding.txt full default mappings and identity for every scalar" {15  let expected : Map[Int32, String] = Map([])16  let content = @support.read_data_file("CaseFolding.txt")17  assert_true(content.has_prefix("# CaseFolding-18.0.0.txt"))18  for raw_line in content.split("\n") {19    let line = raw_line.split("#").next().unwrap().trim()20    if line.is_empty() {21      continue22    }23    guard line.split(";").collect() is [cp, status, target, ..] else {24      fail("Invalid CaseFolding record")25    }26    if status.trim() == "C" || status.trim() == "F" {27      expected[@support.parse_hex_int(cp)] = mapping_string(target)28    }29  }30  assert_true(expected.length() > 1500)31  for cp in 0..<=0x10FFFF {32    if cp >= 0xD800 && cp <= 0xDFFF {33      continue34    }35    let input = cp.unsafe_to_char().to_string()36    let target = expected.get(cp).unwrap_or(input)37    let actual = @casefold.case_fold(input)38    @debug.assert_eq(actual, target, msg="CaseFolding mismatch at \{cp}")39    @debug.assert_eq(40      @casefold.case_fold(actual),41      actual,42      msg="CaseFolding not idempotent at \{cp}",43    )44    // Matching walks code points instead of folding whole strings.45    assert_true(46      @casefold.caseless_match("A\{input}b", "a\{target}B"),47      msg="caseless_match rejects its folding at \{cp}",48    )49    assert_true(50      !@casefold.caseless_match("\{input}a", "\{target}b"),51      msg="caseless_match ignores a difference after \{cp}",52    )53  }54}5556///|57async test "DerivedNormalizationProps.txt NFKC_CF for every scalar" {58  let expected : Map[Int32, String] = Map([])59  let content = @support.read_data_file("DerivedNormalizationProps.txt")60  assert_true(content.has_prefix("# DerivedNormalizationProps-18.0.0.txt"))61  for raw_line in content.split("\n") {62    let line = raw_line.split("#").next().unwrap().trim()63    if line.is_empty() {64      continue65    }66    guard line.split(";").collect() is [range, property, .. fields] else {67      fail("Invalid derived normalization record")68    }69    if property.trim() != "NFKC_CF" {70      continue71    }72    guard fields is [target, ..] else { fail("Missing NFKC_CF target") }73    let target = mapping_string(target)74    let (first, last) = if range.trim().split_once("..") is Some((a, b)) {75      (@support.parse_hex_int(a), @support.parse_hex_int(b))76    } else {77      let cp = @support.parse_hex_int(range)78      (cp, cp)79    }80    for cp in first..<=last {81      expected[cp] = target82    }83  }84  assert_true(expected.length() > 10000)85  for cp in 0..<=0x10FFFF {86    if cp >= 0xD800 && cp <= 0xDFFF {87      continue88    }89    let input = cp.unsafe_to_char().to_string()90    let target = expected.get(cp).unwrap_or(input)91    let actual = @casefold.nfkc_casefold(input)92    @debug.assert_eq(actual, target, msg="NFKC_CF mismatch at \{cp}")93    @debug.assert_eq(94      @casefold.nfkc_casefold(actual),95      actual,96      msg="NFKC_CF not idempotent at \{cp}",97    )98    assert_true(@normalization.is_normalized(actual, @normalization.NFC))99  }100}101102///|103async test "canonical caseless keys respect NormalizationTest canonical equivalents" {104  for raw_line in @support.read_data_file("NormalizationTest.txt").split("\n") {105    let line = raw_line.trim()106    if line.is_empty() || line.has_prefix("#") || line.has_prefix("@") {107      continue108    }109    guard line.split(";").collect() is [c1, c2, c3, c4, c5, ..] else {110      fail("Invalid NormalizationTest record")111    }112    let first = mapping_string(c1)113    let composed = mapping_string(c2)114    let decomposed = mapping_string(c3)115    assert_true(@casefold.canonical_caseless_match(first, composed))116    assert_true(@casefold.canonical_caseless_match(first, decomposed))117    assert_true(118      @casefold.canonical_caseless_match(mapping_string(c4), mapping_string(c5)),119    )120    assert_true(@casefold.identifier_caseless_match(first, decomposed))121  }122}123