unicode

Branch: main

Switch branches or tags

Branch: main
Go to file t

413 files indexed

History
Code

tests/segmentation_tests/text_break_test.mbt
1///|2/// Official Unicode 18.0.0 cases from3/// https://www.unicode.org/Public/18.0.0/ucd/auxiliary/WordBreakTest.txt and4/// https://www.unicode.org/Public/18.0.0/ucd/auxiliary/SentenceBreakTest.txt.5async fn check_text_breaks(6  filename : String,7  segment : (StringView) -> Iter[StringView],8  expected_cases : Int32,9) -> Unit {10  let content = @support.read_data_file(filename)11  let mut cases = 012  for line_number, raw_line in content.split("\n") {13    let line = if raw_line.split_once("#") is Some((before, _)) {14      before.trim()15    } else {16      raw_line.trim()17    }18    if line.is_empty() {19      continue20    }21    guard parse_boundary_segments(line) is Some(expected) else {22      fail("Unexpected surrogate in \{filename} line \{line_number + 1}")23    }24    let input = expected.join("")25    @debug.assert_eq(26      segment(input).map(s => s.to_owned()).collect(),27      expected,28      msg="\{filename} line \{line_number + 1}",29    )30    cases += 131  }32  assert_eq(cases, expected_cases)33}3435///|36async test "WordBreakTest.txt conformance" {37  check_text_breaks(38    "WordBreakTest.txt",39    text => @segmentation.word_segments(text).iter(),40    1944,41  )42}4344///|45async test "SentenceBreakTest.txt conformance" {46  check_text_breaks(47    "SentenceBreakTest.txt",48    text => @segmentation.sentences(text).iter(),49    512,50  )51}52