tests/segmentation_tests/text_break_test.mbt1///|2/// Official Unicode 18.0.0 cases from3/// https://www.unicode.org/Public/18.0.0/ucd/auxiliary/WordBreakTest.txt and4/// https://www.unicode.org/Public/18.0.0/ucd/auxiliary/SentenceBreakTest.txt.5async fn check_text_breaks(6 filename : String,7 segment : (StringView) -> Iter[StringView],8 expected_cases : Int32,9) -> Unit {10 let content = @support.read_data_file(filename)11 let mut cases = 012 for line_number, raw_line in content.split("\n") {13 let line = if raw_line.split_once("#") is Some((before, _)) {14 before.trim()15 } else {16 raw_line.trim()17 }18 if line.is_empty() {19 continue20 }21 guard parse_boundary_segments(line) is Some(expected) else {22 fail("Unexpected surrogate in \{filename} line \{line_number + 1}")23 }24 let input = expected.join("")25 @debug.assert_eq(26 segment(input).map(s => s.to_owned()).collect(),27 expected,28 msg="\{filename} line \{line_number + 1}",29 )30 cases += 131 }32 assert_eq(cases, expected_cases)33}3435///|36async test "WordBreakTest.txt conformance" {37 check_text_breaks(38 "WordBreakTest.txt",39 text => @segmentation.word_segments(text).iter(),40 1944,41 )42}4344///|45async test "SentenceBreakTest.txt conformance" {46 check_text_breaks(47 "SentenceBreakTest.txt",48 text => @segmentation.sentences(text).iter(),49 512,50 )51}52
///|
/// Official Unicode 18.0.0 cases from
/// https://www.unicode.org/Public/18.0.0/ucd/auxiliary/WordBreakTest.txt and
/// https://www.unicode.org/Public/18.0.0/ucd/auxiliary/SentenceBreakTest.txt.
async fn check_text_breaks(
filename : String,
segment : (StringView) -> Iter[StringView],
expected_cases : Int32,
) -> Unit {
let content = @support.read_data_file(filename)
let mut cases = 0
for line_number, raw_line in content.split("\n") {
let line = if raw_line.split_once("#") is Some((before, _)) {
before.trim()
} else {
raw_line.trim()
}
if line.is_empty() {
continue
}
guard parse_boundary_segments(line) is Some(expected) else {
fail("Unexpected surrogate in \{filename} line \{line_number + 1}")
}
let input = expected.join("")
@debug.assert_eq(
segment(input).map(s => s.to_owned()).collect(),
expected,
msg="\{filename} line \{line_number + 1}",
)
cases += 1
}
assert_eq(cases, expected_cases)
}
///|
async test "WordBreakTest.txt conformance" {
check_text_breaks(
"WordBreakTest.txt",
text => @segmentation.word_segments(text).iter(),
1944,
)
}
///|
async test "SentenceBreakTest.txt conformance" {
check_text_breaks(
"SentenceBreakTest.txt",
text => @segmentation.sentences(text).iter(),
512,
)
}