unicode

Branch: main

Switch branches or tags

Branch: main
Go to file t

413 files indexed

History
Code

segmentation/line_break.mbt
1///|2/// Whether a line may end at a text boundary. Mandatory includes the end of3/// nonempty text, even when the text does not end in a newline character.4#derive(Eq, Debug)5pub(all) enum BreakOpportunity {6  Prohibited7  Allowed8  Mandatory9}1011///|12pub extend BreakOpportunity with Eq::{equal}1314///|15#deprecated("Use `!=` instead", skip_current_package=true)16#doc(hidden)17pub extend BreakOpportunity with Eq::{not_equal}1819///|20#deprecated("Use `Debug::to_repr` instead", skip_current_package=true)21#doc(hidden)22pub extend BreakOpportunity with Debug::{to_repr}2324///|25/// A forward cursor over text segments ending at default UAX #14 opportunities.26struct LineBreaks {27  text : StringView28  units : Array[BreakUnit]29  mut index : Int3230  mut offset : Int3231  mut opportunity : BreakOpportunity32}3334///|35/// Find the default Unicode 18.0.0 UAX #14 line break opportunities.36///37/// Segments are contiguous views covering the input, retaining spaces and38/// newline characters. The view is processed independently of its backing39/// string. Default line breaks may fall inside extended grapheme clusters;40/// use Graphemes::line_break to query only the ends of complete clusters.41///42/// Nonempty input ends with one Mandatory boundary; empty input yields nothing.43/// Unpaired surrogates are resolved as SG -> AL; valid pairs stay together.44/// Construction prepares character context in O(n) time and space; iteration45/// scans it once without copying the text. No locale tailoring, dictionary46/// segmentation, or width-based wrapping is applied. The unicode-data47/// line_break property query remains unresolved.48///49/// ```mbt check50/// test {51///   let breaks = @segmentation.line_breaks("Hello world!")52///   inspect(breaks.next(), content="Hello ")53///   debug_inspect(breaks.opportunity(), content="Allowed")54///   inspect(breaks.next(), content="world!")55///   debug_inspect(breaks.opportunity(), content="Mandatory")56///   assert_true(breaks.next().is_empty())57/// }58/// ```59pub fn line_breaks(text : StringView) -> LineBreaks {60  {61    text,62    units: prepare_units(text),63    index: 1,64    offset: 0,65    opportunity: Prohibited,66  }67}6869///|70/// Return the next segment, or an empty view at the end of the input once71/// exhausted. The returned view retains its offsets in the backing string.72/// Empty results do not change opportunity().73pub fn LineBreaks::next(self : LineBreaks) -> StringView {74  while self.index <= self.units.length() {75    let index = self.index76    self.index += 177    let kind = if index == self.units.length() {78      Mandatory79    } else {80      boundary(self.units, index).unwrap_or(Prohibited)81    }82    if kind != Prohibited {83      let start = self.offset84      self.offset = self.units[index - 1].end85      self.opportunity = kind86      // `units` carry the UTF-16 end of every code point, so both offsets are87      // in range and on character boundaries: a raw view needs no checking.88      return self.text.unsafe_view(start~, end=self.offset)89    }90  }91  self.text.unsafe_view(start=self.offset, end=self.offset)92}9394///|95/// Iterate over the remaining segments, advancing this same cursor. Query96/// opportunity() in the loop body for the segment just yielded. Collected97/// views retain their text and positions, but do not store the break kind.98pub fn LineBreaks::iter(self : LineBreaks) -> Iter[StringView] {99  Iter::new(() => {100    let segment = self.next()101    if segment.is_empty() {102      None103    } else {104      Some(segment)105    }106  })107}108109///|110/// The break kind at the end of the last nonempty segment returned by next()111/// or iter(). Before the first segment, including for empty input, returns112/// Prohibited. Exhaustion preserves the last kind; querying does not advance.113pub fn LineBreaks::opportunity(self : LineBreaks) -> BreakOpportunity {114  self.opportunity115}116