unicode

Branch: main

Switch branches or tags

Branch: main
Go to file t

413 files indexed

History
Code

tests/normalization_tests/normalization_test.mbt
1///|2/// Unicode normalization conformance tests, read directly from3/// NormalizationTest.txt (Unicode 18.0.0).4/// Source: https://www.unicode.org/Public/18.0.0/ucd/NormalizationTest.txt5///6/// Each line contains five strings (c1..c5) that must satisfy:7/// - NFC:  c2 == NFC(c1) == NFC(c2) == NFC(c3)8/// - NFD:  c3 == NFD(c1) == NFD(c2) == NFD(c3)9/// - NFKC: c4 == NFKC(c1) == NFKC(c2) == NFKC(c3) == NFKC(c4) == NFKC(c5)10/// - NFKD: c5 == NFKD(c1) == NFKD(c2) == NFKD(c3) == NFKD(c4) == NFKD(c5)1112///|13struct NormalizationCase {14  line_number : Int3215  c1 : String16  c2 : String17  c3 : String18  c4 : String19  c5 : String20}2122///|23fn parse_field(field : StringView) -> String? raise {24  @support.code_points_to_string(@support.parse_code_point_sequence(field))25}2627///|28fn parse_case(29  line : StringView,30  line_number : Int32,31) -> NormalizationCase? raise {32  let line = line.trim()33  if line.is_empty() || line.has_prefix("#") || line.has_prefix("@") {34    return None35  }36  guard line.split(";").collect() is [c1, c2, c3, c4, c5, ..] else {37    return None38  }39  guard parse_field(c1) is Some(c1) else { return None }40  guard parse_field(c2) is Some(c2) else { return None }41  guard parse_field(c3) is Some(c3) else { return None }42  guard parse_field(c4) is Some(c4) else { return None }43  guard parse_field(c5) is Some(c5) else { return None }44  Some({ line_number, c1, c2, c3, c4, c5, })45}4647///|48fn parse_cases(content : String) -> Array[NormalizationCase] raise {49  let cases : Array[NormalizationCase] = []50  for line_number, raw_line in content.split("\n") {51    if parse_case(raw_line, line_number + 1) is Some(case) {52      cases.push(case)53    }54  }55  cases56}5758///|59async test "NormalizationTest.txt - NFC" {60  for case in parse_cases(@support.read_data_file("NormalizationTest.txt")) {61    @debug.assert_eq(62      @normalization.nfc(case.c1),63      case.c2,64      msg="NormalizationTest.txt line \{case.line_number}: NFC(c1) != c2",65    )66    @debug.assert_eq(67      @normalization.nfc(case.c2),68      case.c2,69      msg="NormalizationTest.txt line \{case.line_number}: NFC(c2) != c2",70    )71    @debug.assert_eq(72      @normalization.nfc(case.c3),73      case.c2,74      msg="NormalizationTest.txt line \{case.line_number}: NFC(c3) != c2",75    )76  }77}7879///|80async test "NormalizationTest.txt - NFD" {81  for case in parse_cases(@support.read_data_file("NormalizationTest.txt")) {82    @debug.assert_eq(83      @normalization.nfd(case.c1),84      case.c3,85      msg="NormalizationTest.txt line \{case.line_number}: NFD(c1) != c3",86    )87    @debug.assert_eq(88      @normalization.nfd(case.c2),89      case.c3,90      msg="NormalizationTest.txt line \{case.line_number}: NFD(c2) != c3",91    )92    @debug.assert_eq(93      @normalization.nfd(case.c3),94      case.c3,95      msg="NormalizationTest.txt line \{case.line_number}: NFD(c3) != c3",96    )97  }98}99100///|101async test "NormalizationTest.txt - NFKC" {102  for case in parse_cases(@support.read_data_file("NormalizationTest.txt")) {103    @debug.assert_eq(104      @normalization.nfkc(case.c1),105      case.c4,106      msg="NormalizationTest.txt line \{case.line_number}: NFKC(c1) != c4",107    )108    @debug.assert_eq(109      @normalization.nfkc(case.c2),110      case.c4,111      msg="NormalizationTest.txt line \{case.line_number}: NFKC(c2) != c4",112    )113    @debug.assert_eq(114      @normalization.nfkc(case.c3),115      case.c4,116      msg="NormalizationTest.txt line \{case.line_number}: NFKC(c3) != c4",117    )118    @debug.assert_eq(119      @normalization.nfkc(case.c4),120      case.c4,121      msg="NormalizationTest.txt line \{case.line_number}: NFKC(c4) != c4",122    )123    @debug.assert_eq(124      @normalization.nfkc(case.c5),125      case.c4,126      msg="NormalizationTest.txt line \{case.line_number}: NFKC(c5) != c4",127    )128  }129}130131///|132async test "NormalizationTest.txt - NFKD" {133  for case in parse_cases(@support.read_data_file("NormalizationTest.txt")) {134    @debug.assert_eq(135      @normalization.nfkd(case.c1),136      case.c5,137      msg="NormalizationTest.txt line \{case.line_number}: NFKD(c1) != c5",138    )139    @debug.assert_eq(140      @normalization.nfkd(case.c2),141      case.c5,142      msg="NormalizationTest.txt line \{case.line_number}: NFKD(c2) != c5",143    )144    @debug.assert_eq(145      @normalization.nfkd(case.c3),146      case.c5,147      msg="NormalizationTest.txt line \{case.line_number}: NFKD(c3) != c5",148    )149    @debug.assert_eq(150      @normalization.nfkd(case.c4),151      case.c5,152      msg="NormalizationTest.txt line \{case.line_number}: NFKD(c4) != c5",153    )154    @debug.assert_eq(155      @normalization.nfkd(case.c5),156      case.c5,157      msg="NormalizationTest.txt line \{case.line_number}: NFKD(c5) != c5",158    )159  }160}161162///|163async test "NormalizationTest.txt - quick check and copied span boundaries" {164  for case in parse_cases(@support.read_data_file("NormalizationTest.txt")) {165    let inputs = [case.c1, case.c2, case.c3, case.c4, case.c5]166    for167      form in [168        @normalization.NFC,169        @normalization.NFD,170        @normalization.NFKC,171        @normalization.NFKD,172      ] {173      for i, input in inputs {174        let expected = match form {175          NFC => if i < 3 { case.c2 } else { case.c4 }176          NFD => if i < 3 { case.c3 } else { case.c5 }177          NFKC => case.c4178          NFKD => case.c5179        }180        @debug.assert_eq(181          @normalization.is_normalized(input, form),182          input == expected,183          msg="NormalizationTest.txt line \{case.line_number}: is_normalized",184        )185        @debug.assert_eq(186          @normalization.normalize("😀prefix|\{input}|suffix😀", form),187          "😀prefix|\{expected}|suffix😀",188          msg="NormalizationTest.txt line \{case.line_number}: copied boundaries",189        )190      }191    }192  }193}194