1///|2/// Unicode Text Segmentation conformance tests for extended grapheme3/// clusters, read directly from GraphemeBreakTest.txt (Unicode 18.0.0).4/// Source: https://www.unicode.org/Public/18.0.0/ucd/auxiliary/GraphemeBreakTest.txt56///|7/// Parse one test line such as `÷ 0020 × 0308 ÷ 0020 ÷` into its clusters:8/// `÷` marks a boundary and `×` joins the surrounding code points. Returns9/// `None` if a cluster contains a surrogate, which MoonBit strings cannot10/// represent.11fn parse_boundary_segments(line : StringView) -> Array[String]? raise {12 let clusters : Array[String] = []13 let code_points : Array[Int32] = []14 for token in line.split(" ") {15 match token.trim() {16 "" | "×" => ()17 "÷" =>18 if !code_points.is_empty() {19 guard @support.code_points_to_string(code_points) is Some(cluster) else {20 return None21 }22 clusters.push(cluster)23 code_points.clear()24 }25 code_point => code_points.push(@support.parse_hex_int(code_point))26 }27 }28 Some(clusters)29}3031///|32async test "GraphemeBreakTest.txt conformance" {33 let content = @support.read_data_file("GraphemeBreakTest.txt")34 let mut cases = 035 for line_number, raw_line in content.split("\n") {36 let line = match raw_line.split_once("#") {37 Some((before, _)) => before.trim()38 None => raw_line.trim()39 }40 if line.is_empty() {41 continue42 }43 guard parse_boundary_segments(line) is Some(expected) else { continue }44 let input = expected.join("")45 let actual = @segmentation.graphemes(input)46 .iter()47 .map(cluster => cluster.to_owned())48 .collect()49 @debug.assert_eq(50 actual,51 expected,52 msg="GraphemeBreakTest.txt line \{line_number + 1}",53 )54 cases += 155 }56 // Guard against a parser that silently skips every line.57 assert_true(cases > 500)58}59
///|
/// Unicode Text Segmentation conformance tests for extended grapheme
/// clusters, read directly from GraphemeBreakTest.txt (Unicode 18.0.0).
/// Source: https://www.unicode.org/Public/18.0.0/ucd/auxiliary/GraphemeBreakTest.txt
///|
/// Parse one test line such as `÷ 0020 × 0308 ÷ 0020 ÷` into its clusters:
/// `÷` marks a boundary and `×` joins the surrounding code points. Returns
/// `None` if a cluster contains a surrogate, which MoonBit strings cannot
/// represent.
fn parse_boundary_segments(line : StringView) -> Array[String]? raise {
let clusters : Array[String] = []
let code_points : Array[Int32] = []
for token in line.split(" ") {
match token.trim() {
"" | "×" => ()
"÷" =>
if !code_points.is_empty() {
guard @support.code_points_to_string(code_points) is Some(cluster) else {
return None
}
clusters.push(cluster)
code_points.clear()
}
code_point => code_points.push(@support.parse_hex_int(code_point))
}
}
Some(clusters)
}
///|
async test "GraphemeBreakTest.txt conformance" {
let content = @support.read_data_file("GraphemeBreakTest.txt")
let mut cases = 0
for line_number, raw_line in content.split("\n") {
let line = match raw_line.split_once("#") {
Some((before, _)) => before.trim()
None => raw_line.trim()
}
if line.is_empty() {
continue
}
guard parse_boundary_segments(line) is Some(expected) else { continue }
let input = expected.join("")
let actual = @segmentation.graphemes(input)
.iter()
.map(cluster => cluster.to_owned())
.collect()
@debug.assert_eq(
actual,
expected,
msg="GraphemeBreakTest.txt line \{line_number + 1}",
)
cases += 1
}
// Guard against a parser that silently skips every line.
assert_true(cases > 500)
}