unicode

Branch: main

Switch branches or tags

Branch: main
Go to file t

413 files indexed

History
Code

tests/segmentation_tests/grapheme_test.mbt
1///|2/// Unicode Text Segmentation conformance tests for extended grapheme3/// clusters, read directly from GraphemeBreakTest.txt (Unicode 18.0.0).4/// Source: https://www.unicode.org/Public/18.0.0/ucd/auxiliary/GraphemeBreakTest.txt56///|7/// Parse one test line such as `÷ 0020 × 0308 ÷ 0020 ÷` into its clusters:8/// `÷` marks a boundary and `×` joins the surrounding code points. Returns9/// `None` if a cluster contains a surrogate, which MoonBit strings cannot10/// represent.11fn parse_boundary_segments(line : StringView) -> Array[String]? raise {12  let clusters : Array[String] = []13  let code_points : Array[Int32] = []14  for token in line.split(" ") {15    match token.trim() {16      "" | "×" => ()17      "÷" =>18        if !code_points.is_empty() {19          guard @support.code_points_to_string(code_points) is Some(cluster) else {20            return None21          }22          clusters.push(cluster)23          code_points.clear()24        }25      code_point => code_points.push(@support.parse_hex_int(code_point))26    }27  }28  Some(clusters)29}3031///|32async test "GraphemeBreakTest.txt conformance" {33  let content = @support.read_data_file("GraphemeBreakTest.txt")34  let mut cases = 035  for line_number, raw_line in content.split("\n") {36    let line = match raw_line.split_once("#") {37      Some((before, _)) => before.trim()38      None => raw_line.trim()39    }40    if line.is_empty() {41      continue42    }43    guard parse_boundary_segments(line) is Some(expected) else { continue }44    let input = expected.join("")45    let actual = @segmentation.graphemes(input)46      .iter()47      .map(cluster => cluster.to_owned())48      .collect()49    @debug.assert_eq(50      actual,51      expected,52      msg="GraphemeBreakTest.txt line \{line_number + 1}",53    )54    cases += 155  }56  // Guard against a parser that silently skips every line.57  assert_true(cases > 500)58}59