segmentation/word_test.mbt1///|2test "word segments retain punctuation, spaces, and numeric context" {3 @debug.assert_eq(4 @segmentation.word_segments(5 "can't 3.14 foo_bar אב\"ג カタカナ\r\n中文",6 )7 .iter()8 .map(s => s.to_owned())9 .collect(),10 [11 "can't", " ", "3.14", " ", "foo_bar", " ", "אב\"ג", " ", "カタカナ",12 "\r\n", "中", "文",13 ],14 )15}1617///|18test "words filter by Alphabetic or Number and preserve complete segments" {19 @debug.assert_eq(20 @segmentation.words(21 "👩👩👧👦 -- can't 42 Ⅷ ² _ \u{301} e\u{301} 中文",22 )23 .iter()24 .map(s => s.to_owned())25 .collect(),26 ["can't", "42", "Ⅷ", "²", "e\u{301}", "中", "文"],27 )28 assert_true(@segmentation.words("!? 👩👧 🇺🇳").next().is_empty())29 @debug.assert_eq(30 @segmentation.words("1️⃣ #️⃣")31 .iter()32 .map(s => s.to_owned())33 .collect(),34 ["1️⃣"],35 )36}3738///|39test "word cursor exhaustion and interleaved iterators" {40 let cursor = @segmentation.word_segments("one two!")41 let iterator = cursor.iter()42 inspect(cursor.next(), content="one")43 @debug.assert_eq(iterator.next(), Some(" "))44 inspect(cursor.next(), content="two")45 @debug.assert_eq(iterator.next(), Some("!"))46 assert_true(cursor.next().is_empty())47 assert_true(cursor.next().is_empty())48 @debug.assert_eq(iterator.next(), None)49 let filtered = @segmentation.words("one two!")50 let iterator = filtered.iter()51 inspect(filtered.next(), content="one")52 @debug.assert_eq(iterator.next(), Some("two"))53 let end = filtered.next()54 assert_true(end.is_empty())55 assert_eq(end.start_offset(), 8)56 assert_true(filtered.next().is_empty())57 @debug.assert_eq(iterator.next(), None)58 let empty_segments = @segmentation.word_segments("")59 let empty_words = @segmentation.words("")60 assert_true(empty_segments.next().is_empty())61 assert_true(empty_segments.next().is_empty())62 assert_true(empty_words.next().is_empty())63 assert_true(empty_words.next().is_empty())64}6566///|67test "word views retain UTF-16 offsets and use independent context" {68 let backing = "xa.b😀z"69 let text = backing.unsafe_view(start=2, end=6)70 let cursor = @segmentation.word_segments(text)71 @debug.assert_eq(72 cursor73 .iter()74 .map(s => (s.to_owned(), s.start_offset(), s.length()))75 .collect(),76 [(".", 2, 1), ("b", 3, 1), ("😀", 4, 2)],77 )78 assert_eq(cursor.next().start_offset(), 6)79 let flags = "🇦🇧🇨🇩🇪"80 @debug.assert_eq(81 @segmentation.word_segments(flags.unsafe_view(start=2, end=flags.length()))82 .iter()83 .map(s => s.to_owned())84 .collect(),85 ["🇧🇨", "🇩🇪"],86 )87}8889///|90test "word ignored characters and RI pairing keep the required context" {91 @debug.assert_eq(92 @segmentation.word_segments("\u{301}a\r\n\u{301}b a\u{301}'\u{301}b")93 .iter()94 .map(s => s.to_owned())95 .collect(),96 ["\u{301}", "a", "\r\n", "\u{301}", "b", " ", "a\u{301}'\u{301}b"],97 )98 @debug.assert_eq(99 @segmentation.word_segments("🇦\u{301}🇧🇨\u{301}🇩🇪")100 .iter()101 .map(s => s.to_owned())102 .collect(),103 ["🇦\u{301}🇧", "🇨\u{301}🇩", "🇪"],104 )105 // WB3d is before WB4: marks between spaces prevent their grouping.106 @debug.assert_eq(107 @segmentation.word_segments(" \u{301} ")108 .iter()109 .map(s => s.to_owned())110 .collect(),111 [" \u{301}", " "],112 )113}114115///|116test "word boundaries can split an extended grapheme" {117 let text = "\u{6DD} "118 assert_eq(@segmentation.graphemes(text).iter().count(), 1)119 @debug.assert_eq(120 @segmentation.word_segments(text).iter().map(s => s.to_owned()).collect(),121 ["\u{6DD}", " "],122 )123}124125///|126test "word cursors preserve unpaired surrogates" {127 let text = "a😀\u{301}b"128 let leading = text.unsafe_view(start=0, end=2)129 @debug.assert_eq(130 @segmentation.word_segments(leading).iter().map(s => s.length()).collect(),131 [1, 1],132 )133 let trailing = text.unsafe_view(start=2, end=text.length())134 @debug.assert_eq(135 @segmentation.word_segments(trailing).iter().map(s => s.length()).collect(),136 [2, 1],137 )138 @debug.assert_eq(139 @segmentation.words(trailing).iter().map(s => s.to_owned()).collect(),140 ["b"],141 )142}143144///|145test "long ignored runs do not change word punctuation lookaround" {146 let marks = "\u{301}".repeat(20000)147 let text = "a" + marks + "'" + marks + "b"148 let cursor = @segmentation.word_segments(text)149 assert_eq(cursor.next().length(), text.length())150 assert_true(cursor.next().is_empty())151}152
///|
test "word segments retain punctuation, spaces, and numeric context" {
@debug.assert_eq(
@segmentation.word_segments(
"can't 3.14 foo_bar אב\"ג カタカナ\r\n中文",
)
.iter()
.map(s => s.to_owned())
.collect(),
[
"can't", " ", "3.14", " ", "foo_bar", " ", "אב\"ג", " ", "カタカナ",
"\r\n", "中", "文",
],
)
}
///|
test "words filter by Alphabetic or Number and preserve complete segments" {
@debug.assert_eq(
@segmentation.words(
"👩👩👧👦 -- can't 42 Ⅷ ² _ \u{301} e\u{301} 中文",
)
.iter()
.map(s => s.to_owned())
.collect(),
["can't", "42", "Ⅷ", "²", "e\u{301}", "中", "文"],
)
assert_true(@segmentation.words("!? 👩👧 🇺🇳").next().is_empty())
@debug.assert_eq(
@segmentation.words("1️⃣ #️⃣")
.iter()
.map(s => s.to_owned())
.collect(),
["1️⃣"],
)
}
///|
test "word cursor exhaustion and interleaved iterators" {
let cursor = @segmentation.word_segments("one two!")
let iterator = cursor.iter()
inspect(cursor.next(), content="one")
@debug.assert_eq(iterator.next(), Some(" "))
inspect(cursor.next(), content="two")
@debug.assert_eq(iterator.next(), Some("!"))
assert_true(cursor.next().is_empty())
assert_true(cursor.next().is_empty())
@debug.assert_eq(iterator.next(), None)
let filtered = @segmentation.words("one two!")
let iterator = filtered.iter()
inspect(filtered.next(), content="one")
@debug.assert_eq(iterator.next(), Some("two"))
let end = filtered.next()
assert_true(end.is_empty())
assert_eq(end.start_offset(), 8)
assert_true(filtered.next().is_empty())
@debug.assert_eq(iterator.next(), None)
let empty_segments = @segmentation.word_segments("")
let empty_words = @segmentation.words("")
assert_true(empty_segments.next().is_empty())
assert_true(empty_segments.next().is_empty())
assert_true(empty_words.next().is_empty())
assert_true(empty_words.next().is_empty())
}
///|
test "word views retain UTF-16 offsets and use independent context" {
let backing = "xa.b😀z"
let text = backing.unsafe_view(start=2, end=6)
let cursor = @segmentation.word_segments(text)
@debug.assert_eq(
cursor
.iter()
.map(s => (s.to_owned(), s.start_offset(), s.length()))
.collect(),
[(".", 2, 1), ("b", 3, 1), ("😀", 4, 2)],
)
assert_eq(cursor.next().start_offset(), 6)
let flags = "🇦🇧🇨🇩🇪"
@debug.assert_eq(
@segmentation.word_segments(flags.unsafe_view(start=2, end=flags.length()))
.iter()
.map(s => s.to_owned())
.collect(),
["🇧🇨", "🇩🇪"],
)
}
///|
test "word ignored characters and RI pairing keep the required context" {
@debug.assert_eq(
@segmentation.word_segments("\u{301}a\r\n\u{301}b a\u{301}'\u{301}b")
.iter()
.map(s => s.to_owned())
.collect(),
["\u{301}", "a", "\r\n", "\u{301}", "b", " ", "a\u{301}'\u{301}b"],
)
@debug.assert_eq(
@segmentation.word_segments("🇦\u{301}🇧🇨\u{301}🇩🇪")
.iter()
.map(s => s.to_owned())
.collect(),
["🇦\u{301}🇧", "🇨\u{301}🇩", "🇪"],
)
// WB3d is before WB4: marks between spaces prevent their grouping.
@debug.assert_eq(
@segmentation.word_segments(" \u{301} ")
.iter()
.map(s => s.to_owned())
.collect(),
[" \u{301}", " "],
)
}
///|
test "word boundaries can split an extended grapheme" {
let text = "\u{6DD} "
assert_eq(@segmentation.graphemes(text).iter().count(), 1)
@debug.assert_eq(
@segmentation.word_segments(text).iter().map(s => s.to_owned()).collect(),
["\u{6DD}", " "],
)
}
///|
test "word cursors preserve unpaired surrogates" {
let text = "a😀\u{301}b"
let leading = text.unsafe_view(start=0, end=2)
@debug.assert_eq(
@segmentation.word_segments(leading).iter().map(s => s.length()).collect(),
[1, 1],
)
let trailing = text.unsafe_view(start=2, end=text.length())
@debug.assert_eq(
@segmentation.word_segments(trailing).iter().map(s => s.length()).collect(),
[2, 1],
)
@debug.assert_eq(
@segmentation.words(trailing).iter().map(s => s.to_owned()).collect(),
["b"],
)
}
///|
test "long ignored runs do not change word punctuation lookaround" {
let marks = "\u{301}".repeat(20000)
let text = "a" + marks + "'" + marks + "b"
let cursor = @segmentation.word_segments(text)
assert_eq(cursor.next().length(), text.length())
assert_true(cursor.next().is_empty())
}