unicode

Branch: main

Switch branches or tags

Branch: main
Go to file t

413 files indexed

History
Code

segmentation/sentence_test.mbt
1///|2test "sentences retain closing punctuation, spaces, and paragraph separators" {3  @debug.assert_eq(4    @segmentation.sentences("Hello!  \"Next.\"\r\n第三句。尾巴")5    .iter()6    .map(s => s.to_owned())7    .collect(),8    ["Hello!  ", "\"Next.\"\r\n", "第三句。", "尾巴"],9  )10  @debug.assert_eq(11    @segmentation.sentences("\r\n\n\u{2029}tail")12    .iter()13    .map(s => s.to_owned())14    .collect(),15    ["\r\n", "\n", "\u{2029}", "tail"],16  )17}1819///|20test "sentence full stops use numeric, abbreviation, and lowercase context" {21  @debug.assert_eq(22    @segmentation.sentences("3.14 is pi. U.S.A. is here. Next!")23    .iter()24    .map(s => s.to_owned())25    .collect(),26    ["3.14 is pi. ", "U.S.A. is here. ", "Next!"],27  )28  @debug.assert_eq(29    @segmentation.sentences("Go.)  'again! Yes?! No")30    .iter()31    .map(s => s.to_owned())32    .collect(),33    ["Go.)  'again! ", "Yes?! ", "No"],34  )35}3637///|38test "sentence cursor exhaustion and shared iteration" {39  let cursor = @segmentation.sentences("One! Two? Three.")40  let iterator = cursor.iter()41  inspect(cursor.next(), content="One! ")42  @debug.assert_eq(iterator.next(), Some("Two? "))43  inspect(cursor.next(), content="Three.")44  assert_true(cursor.next().is_empty())45  assert_true(cursor.next().is_empty())46  @debug.assert_eq(iterator.next(), None)47  assert_eq(cursor.next().start_offset(), 16)48  let empty = @segmentation.sentences("")49  assert_true(empty.next().is_empty())50  assert_true(empty.next().is_empty())51  assert_true(empty.iter().collect().is_empty())52  @debug.assert_eq(53    @segmentation.sentences("   ").iter().map(s => s.to_owned()).collect(),54    ["   "],55  )56}5758///|59test "sentence views retain offsets and do not read preceding punctuation" {60  let backing = "Done!   Next.😀"61  let text = backing.unsafe_view(start=6, end=backing.length())62  let cursor = @segmentation.sentences(text)63  @debug.assert_eq(64    cursor.iter().map(s => (s.to_owned(), s.start_offset())).collect(),65    [("  Next.", 6), ("😀", 13)],66  )67  assert_eq(cursor.next().start_offset(), backing.length())68}6970///|71test "sentence ignore exceptions and non-nested word boundaries" {72  @debug.assert_eq(73    @segmentation.sentences("A!\u{301} B\r\u{301}C")74    .iter()75    .map(s => s.to_owned())76    .collect(),77    ["A!\u{301} ", "B\r", "\u{301}C"],78  )79  let text = "1\u{589}2"80  assert_eq(@segmentation.word_segments(text).iter().count(), 1)81  @debug.assert_eq(82    @segmentation.sentences(text).iter().map(s => s.to_owned()).collect(),83    ["1\u{589}", "2"],84  )85}8687///|88test "sentence cursors preserve unpaired surrogates" {89  let text = "😀!😀"90  let view = text.unsafe_view(start=1, end=4)91  @debug.assert_eq(92    @segmentation.sentences(view)93    .iter()94    .map(s => (s.start_offset(), s.length()))95    .collect(),96    [(1, 2), (3, 1)],97  )98}99100///|101test "long sentence suffixes and lowercase lookahead" {102  let closers = ")".repeat(20000)103  let spaces = " ".repeat(20000)104  let text = "A." + closers + spaces + "lower"105  let cursor = @segmentation.sentences(text)106  assert_eq(cursor.next().length(), text.length())107  assert_true(cursor.next().is_empty())108}109