segmentation/sentence_test.mbt1///|2test "sentences retain closing punctuation, spaces, and paragraph separators" {3 @debug.assert_eq(4 @segmentation.sentences("Hello! \"Next.\"\r\n第三句。尾巴")5 .iter()6 .map(s => s.to_owned())7 .collect(),8 ["Hello! ", "\"Next.\"\r\n", "第三句。", "尾巴"],9 )10 @debug.assert_eq(11 @segmentation.sentences("\r\n\n\u{2029}tail")12 .iter()13 .map(s => s.to_owned())14 .collect(),15 ["\r\n", "\n", "\u{2029}", "tail"],16 )17}1819///|20test "sentence full stops use numeric, abbreviation, and lowercase context" {21 @debug.assert_eq(22 @segmentation.sentences("3.14 is pi. U.S.A. is here. Next!")23 .iter()24 .map(s => s.to_owned())25 .collect(),26 ["3.14 is pi. ", "U.S.A. is here. ", "Next!"],27 )28 @debug.assert_eq(29 @segmentation.sentences("Go.) 'again! Yes?! No")30 .iter()31 .map(s => s.to_owned())32 .collect(),33 ["Go.) 'again! ", "Yes?! ", "No"],34 )35}3637///|38test "sentence cursor exhaustion and shared iteration" {39 let cursor = @segmentation.sentences("One! Two? Three.")40 let iterator = cursor.iter()41 inspect(cursor.next(), content="One! ")42 @debug.assert_eq(iterator.next(), Some("Two? "))43 inspect(cursor.next(), content="Three.")44 assert_true(cursor.next().is_empty())45 assert_true(cursor.next().is_empty())46 @debug.assert_eq(iterator.next(), None)47 assert_eq(cursor.next().start_offset(), 16)48 let empty = @segmentation.sentences("")49 assert_true(empty.next().is_empty())50 assert_true(empty.next().is_empty())51 assert_true(empty.iter().collect().is_empty())52 @debug.assert_eq(53 @segmentation.sentences(" ").iter().map(s => s.to_owned()).collect(),54 [" "],55 )56}5758///|59test "sentence views retain offsets and do not read preceding punctuation" {60 let backing = "Done! Next.😀"61 let text = backing.unsafe_view(start=6, end=backing.length())62 let cursor = @segmentation.sentences(text)63 @debug.assert_eq(64 cursor.iter().map(s => (s.to_owned(), s.start_offset())).collect(),65 [(" Next.", 6), ("😀", 13)],66 )67 assert_eq(cursor.next().start_offset(), backing.length())68}6970///|71test "sentence ignore exceptions and non-nested word boundaries" {72 @debug.assert_eq(73 @segmentation.sentences("A!\u{301} B\r\u{301}C")74 .iter()75 .map(s => s.to_owned())76 .collect(),77 ["A!\u{301} ", "B\r", "\u{301}C"],78 )79 let text = "1\u{589}2"80 assert_eq(@segmentation.word_segments(text).iter().count(), 1)81 @debug.assert_eq(82 @segmentation.sentences(text).iter().map(s => s.to_owned()).collect(),83 ["1\u{589}", "2"],84 )85}8687///|88test "sentence cursors preserve unpaired surrogates" {89 let text = "😀!😀"90 let view = text.unsafe_view(start=1, end=4)91 @debug.assert_eq(92 @segmentation.sentences(view)93 .iter()94 .map(s => (s.start_offset(), s.length()))95 .collect(),96 [(1, 2), (3, 1)],97 )98}99100///|101test "long sentence suffixes and lowercase lookahead" {102 let closers = ")".repeat(20000)103 let spaces = " ".repeat(20000)104 let text = "A." + closers + spaces + "lower"105 let cursor = @segmentation.sentences(text)106 assert_eq(cursor.next().length(), text.length())107 assert_true(cursor.next().is_empty())108}109
///|
test "sentences retain closing punctuation, spaces, and paragraph separators" {
@debug.assert_eq(
@segmentation.sentences("Hello! \"Next.\"\r\n第三句。尾巴")
.iter()
.map(s => s.to_owned())
.collect(),
["Hello! ", "\"Next.\"\r\n", "第三句。", "尾巴"],
)
@debug.assert_eq(
@segmentation.sentences("\r\n\n\u{2029}tail")
.iter()
.map(s => s.to_owned())
.collect(),
["\r\n", "\n", "\u{2029}", "tail"],
)
}
///|
test "sentence full stops use numeric, abbreviation, and lowercase context" {
@debug.assert_eq(
@segmentation.sentences("3.14 is pi. U.S.A. is here. Next!")
.iter()
.map(s => s.to_owned())
.collect(),
["3.14 is pi. ", "U.S.A. is here. ", "Next!"],
)
@debug.assert_eq(
@segmentation.sentences("Go.) 'again! Yes?! No")
.iter()
.map(s => s.to_owned())
.collect(),
["Go.) 'again! ", "Yes?! ", "No"],
)
}
///|
test "sentence cursor exhaustion and shared iteration" {
let cursor = @segmentation.sentences("One! Two? Three.")
let iterator = cursor.iter()
inspect(cursor.next(), content="One! ")
@debug.assert_eq(iterator.next(), Some("Two? "))
inspect(cursor.next(), content="Three.")
assert_true(cursor.next().is_empty())
assert_true(cursor.next().is_empty())
@debug.assert_eq(iterator.next(), None)
assert_eq(cursor.next().start_offset(), 16)
let empty = @segmentation.sentences("")
assert_true(empty.next().is_empty())
assert_true(empty.next().is_empty())
assert_true(empty.iter().collect().is_empty())
@debug.assert_eq(
@segmentation.sentences(" ").iter().map(s => s.to_owned()).collect(),
[" "],
)
}
///|
test "sentence views retain offsets and do not read preceding punctuation" {
let backing = "Done! Next.😀"
let text = backing.unsafe_view(start=6, end=backing.length())
let cursor = @segmentation.sentences(text)
@debug.assert_eq(
cursor.iter().map(s => (s.to_owned(), s.start_offset())).collect(),
[(" Next.", 6), ("😀", 13)],
)
assert_eq(cursor.next().start_offset(), backing.length())
}
///|
test "sentence ignore exceptions and non-nested word boundaries" {
@debug.assert_eq(
@segmentation.sentences("A!\u{301} B\r\u{301}C")
.iter()
.map(s => s.to_owned())
.collect(),
["A!\u{301} ", "B\r", "\u{301}C"],
)
let text = "1\u{589}2"
assert_eq(@segmentation.word_segments(text).iter().count(), 1)
@debug.assert_eq(
@segmentation.sentences(text).iter().map(s => s.to_owned()).collect(),
["1\u{589}", "2"],
)
}
///|
test "sentence cursors preserve unpaired surrogates" {
let text = "😀!😀"
let view = text.unsafe_view(start=1, end=4)
@debug.assert_eq(
@segmentation.sentences(view)
.iter()
.map(s => (s.start_offset(), s.length()))
.collect(),
[(1, 2), (3, 1)],
)
}
///|
test "long sentence suffixes and lowercase lookahead" {
let closers = ")".repeat(20000)
let spaces = " ".repeat(20000)
let text = "A." + closers + spaces + "lower"
let cursor = @segmentation.sentences(text)
assert_eq(cursor.next().length(), text.length())
assert_true(cursor.next().is_empty())
}