unicode

Branch: main

Switch branches or tags

Branch: main
Go to file t

413 files indexed

History
Code

segmentation/word.mbt
1///|2/// A forward cursor over all default UAX #29 word segments, including spaces3/// and punctuation. Create with `word_segments`.4struct WordSegments(TextSegments)56///|7/// Split text at the default Unicode 18.0.0 word boundaries. Segments are8/// contiguous views covering the input exactly. The input view is processed9/// independently of surrounding text, without dictionary or locale tailoring.10/// Construction takes O(n) time and space; traversal does not copy the text.11/// Unpaired surrogates have Word_Break=Other; valid pairs stay together.12///13/// ```mbt check14/// test {15///   debug_inspect(16///     @segmentation.word_segments("Hello, world!")17///     .iter()18///     .map(s => s.to_owned())19///     .collect(),20///     content="[\"Hello\", \",\", \" \", \"world\", \"!\"]",21///   )22/// }23/// ```24pub fn word_segments(text : StringView) -> WordSegments {25  WordSegments({ text, ends: word_segment_ends(text), index: 0, offset: 0, })26}2728///|29/// Return the next segment, or an empty view at the input's end on exhaustion.30pub fn WordSegments::next(self : WordSegments) -> StringView {31  self.0.next()32}3334///|35/// Iterate over the remaining segments, consuming the same cursor as `next`.36pub fn WordSegments::iter(self : WordSegments) -> Iter[StringView] {37  self.0.iter()38}3940///|41/// A forward cursor over word segments containing a Unicode Alphabetic or42/// Number character. Create with `words`.43struct Words(WordSegments)4445///|46/// Filter default word segments to those containing at least one Alphabetic47/// character or a character in General_Category=Number (Nd, Nl, or No).48/// The entire matching segment is retained, including internal punctuation.49/// This filtering policy is a convenience over UAX #29 word boundaries, not50/// dictionary word recognition. Construction takes O(n) time and space;51/// traversal scans the segments without copying text.52///53/// ```mbt check54/// test {55///   debug_inspect(56///     @segmentation.words("Hello, world! 42")57///     .iter()58///     .map(s => s.to_owned())59///     .collect(),60///     content="[\"Hello\", \"world\", \"42\"]",61///   )62/// }63/// ```64pub fn words(text : StringView) -> Words {65  Words(word_segments(text))66}6768///|69/// Return the next word, skipping other segments. Returns an empty view at70/// the input's end once exhausted, including when the input has no words.71pub fn Words::next(self : Words) -> StringView {72  for segment = self.0.next() {73    if segment.is_empty() {74      return segment75    }76    let mut offset = 077    while offset < segment.length() {78      let cp = code_point_at(segment, offset, segment.length())79      offset += if cp >= 0x10000 { 2 } else { 1 }80      if cp >= 0xD800 && cp <= 0xDFFF {81        continue82      }83      let character = cp.unsafe_to_char()84      if @ucd.is_alphabetic(character) ||85        @ucd.general_category(character) is (@ucd.Nd | @ucd.Nl | @ucd.No) {86        return segment87      }88    }89    continue self.0.next()90  }91}9293///|94/// Iterate over the remaining words, consuming the same cursor as `next`.95pub fn Words::iter(self : Words) -> Iter[StringView] {96  Iter::new(() => {97    let word = self.next()98    if word.is_empty() {99      None100    } else {101      Some(word)102    }103  })104}105