segmentation/word.mbt1///|2/// A forward cursor over all default UAX #29 word segments, including spaces3/// and punctuation. Create with `word_segments`.4struct WordSegments(TextSegments)56///|7/// Split text at the default Unicode 18.0.0 word boundaries. Segments are8/// contiguous views covering the input exactly. The input view is processed9/// independently of surrounding text, without dictionary or locale tailoring.10/// Construction takes O(n) time and space; traversal does not copy the text.11/// Unpaired surrogates have Word_Break=Other; valid pairs stay together.12///13/// ```mbt check14/// test {15/// debug_inspect(16/// @segmentation.word_segments("Hello, world!")17/// .iter()18/// .map(s => s.to_owned())19/// .collect(),20/// content="[\"Hello\", \",\", \" \", \"world\", \"!\"]",21/// )22/// }23/// ```24pub fn word_segments(text : StringView) -> WordSegments {25 WordSegments({ text, ends: word_segment_ends(text), index: 0, offset: 0, })26}2728///|29/// Return the next segment, or an empty view at the input's end on exhaustion.30pub fn WordSegments::next(self : WordSegments) -> StringView {31 self.0.next()32}3334///|35/// Iterate over the remaining segments, consuming the same cursor as `next`.36pub fn WordSegments::iter(self : WordSegments) -> Iter[StringView] {37 self.0.iter()38}3940///|41/// A forward cursor over word segments containing a Unicode Alphabetic or42/// Number character. Create with `words`.43struct Words(WordSegments)4445///|46/// Filter default word segments to those containing at least one Alphabetic47/// character or a character in General_Category=Number (Nd, Nl, or No).48/// The entire matching segment is retained, including internal punctuation.49/// This filtering policy is a convenience over UAX #29 word boundaries, not50/// dictionary word recognition. Construction takes O(n) time and space;51/// traversal scans the segments without copying text.52///53/// ```mbt check54/// test {55/// debug_inspect(56/// @segmentation.words("Hello, world! 42")57/// .iter()58/// .map(s => s.to_owned())59/// .collect(),60/// content="[\"Hello\", \"world\", \"42\"]",61/// )62/// }63/// ```64pub fn words(text : StringView) -> Words {65 Words(word_segments(text))66}6768///|69/// Return the next word, skipping other segments. Returns an empty view at70/// the input's end once exhausted, including when the input has no words.71pub fn Words::next(self : Words) -> StringView {72 for segment = self.0.next() {73 if segment.is_empty() {74 return segment75 }76 let mut offset = 077 while offset < segment.length() {78 let cp = code_point_at(segment, offset, segment.length())79 offset += if cp >= 0x10000 { 2 } else { 1 }80 if cp >= 0xD800 && cp <= 0xDFFF {81 continue82 }83 let character = cp.unsafe_to_char()84 if @ucd.is_alphabetic(character) ||85 @ucd.general_category(character) is (@ucd.Nd | @ucd.Nl | @ucd.No) {86 return segment87 }88 }89 continue self.0.next()90 }91}9293///|94/// Iterate over the remaining words, consuming the same cursor as `next`.95pub fn Words::iter(self : Words) -> Iter[StringView] {96 Iter::new(() => {97 let word = self.next()98 if word.is_empty() {99 None100 } else {101 Some(word)102 }103 })104}105
///|
/// A forward cursor over all default UAX #29 word segments, including spaces
/// and punctuation. Create with `word_segments`.
struct WordSegments(TextSegments)
///|
/// Split text at the default Unicode 18.0.0 word boundaries. Segments are
/// contiguous views covering the input exactly. The input view is processed
/// independently of surrounding text, without dictionary or locale tailoring.
/// Construction takes O(n) time and space; traversal does not copy the text.
/// Unpaired surrogates have Word_Break=Other; valid pairs stay together.
///
/// ```mbt check
/// test {
/// debug_inspect(
/// @segmentation.word_segments("Hello, world!")
/// .iter()
/// .map(s => s.to_owned())
/// .collect(),
/// content="[\"Hello\", \",\", \" \", \"world\", \"!\"]",
/// )
/// }
/// ```
pub fn word_segments(text : StringView) -> WordSegments {
WordSegments({ text, ends: word_segment_ends(text), index: 0, offset: 0, })
}
///|
/// Return the next segment, or an empty view at the input's end on exhaustion.
pub fn WordSegments::next(self : WordSegments) -> StringView {
self.0.next()
}
///|
/// Iterate over the remaining segments, consuming the same cursor as `next`.
pub fn WordSegments::iter(self : WordSegments) -> Iter[StringView] {
self.0.iter()
}
///|
/// A forward cursor over word segments containing a Unicode Alphabetic or
/// Number character. Create with `words`.
struct Words(WordSegments)
///|
/// Filter default word segments to those containing at least one Alphabetic
/// character or a character in General_Category=Number (Nd, Nl, or No).
/// The entire matching segment is retained, including internal punctuation.
/// This filtering policy is a convenience over UAX #29 word boundaries, not
/// dictionary word recognition. Construction takes O(n) time and space;
/// traversal scans the segments without copying text.
///
/// ```mbt check
/// test {
/// debug_inspect(
/// @segmentation.words("Hello, world! 42")
/// .iter()
/// .map(s => s.to_owned())
/// .collect(),
/// content="[\"Hello\", \"world\", \"42\"]",
/// )
/// }
/// ```
pub fn words(text : StringView) -> Words {
Words(word_segments(text))
}
///|
/// Return the next word, skipping other segments. Returns an empty view at
/// the input's end once exhausted, including when the input has no words.
pub fn Words::next(self : Words) -> StringView {
for segment = self.0.next() {
if segment.is_empty() {
return segment
}
let mut offset = 0
while offset < segment.length() {
let cp = code_point_at(segment, offset, segment.length())
offset += if cp >= 0x10000 { 2 } else { 1 }
if cp >= 0xD800 && cp <= 0xDFFF {
continue
}
let character = cp.unsafe_to_char()
if @ucd.is_alphabetic(character) ||
@ucd.general_category(character) is (@ucd.Nd | @ucd.Nl | @ucd.No) {
return segment
}
}
continue self.0.next()
}
}
///|
/// Iterate over the remaining words, consuming the same cursor as `next`.
pub fn Words::iter(self : Words) -> Iter[StringView] {
Iter::new(() => {
let word = self.next()
if word.is_empty() {
None
} else {
Some(word)
}
})
}