unicode

Branch: main

Switch branches or tags

Branch: main
Go to file t

413 files indexed

History
Code

segmentation/text_segments.mbt
1///|2/// Shared storage for the word and sentence cursors. Only segment ends are3/// retained after boundary analysis; slices keep the original backing text.4priv struct TextSegments {5  text : StringView6  ends : Array[Int32]7  mut index : Int328  mut offset : Int329}1011///|12fn TextSegments::next(self : TextSegments) -> StringView {13  let start = self.offset14  if self.index < self.ends.length() {15    self.offset = self.ends[self.index]16    self.index += 117  }18  // Every entry of `ends` is a boundary recorded by the analysis, so the slice19  // is in range and never splits a character: no clamping is needed.20  self.text.unsafe_view(start~, end=self.offset)21}2223///|24fn TextSegments::iter(self : TextSegments) -> Iter[StringView] {25  Iter::new(() => {26    let segment = self.next()27    if segment.is_empty() {28      None29    } else {30      Some(segment)31    }32  })33}3435///|36/// One entry per decoded code point, with UTF-16 end offsets. Unpaired37/// surrogates have the unlisted (Other) property; valid pairs stay together.38priv struct TextBreakInput {39  points : Array[Int32]40  properties : Array[Int32]41  ends : Array[Int32]42}4344///|45fn text_break_input(46  text : StringView,47  property : (Char) -> Int32,48) -> TextBreakInput {49  let points = []50  let properties = []51  let ends = []52  let mut offset = 053  while offset < text.length() {54    let cp = code_point_at(text, offset, text.length())55    offset += if cp >= 0x10000 { 2 } else { 1 }56    points.push(cp)57    properties.push(58      if cp >= 0xD800 && cp <= 0xDFFF {59        060      } else {61        property(cp.unsafe_to_char())62      },63    )64    ends.push(offset)65  }66  { points, properties, ends, }67}68