unicode

Branch: main

Switch branches or tags

Branch: main
Go to file t

413 files indexed

History
Code

segmentation/grapheme.mbt
1///|2/// State of the GB11 emoji sequence scan within a cluster: no3/// Extended_Pictographic code point is pending.4const EmojiNone : Int32 = 056///|7/// The cluster so far ends in `\p{Extended_Pictographic} Extend*`.8const EmojiPictographic : Int32 = 1910///|11/// The cluster so far ends in `\p{Extended_Pictographic} Extend* ZWJ`.12const EmojiZWJ : Int32 = 21314///|15/// Code points below this limit are CR, LF, Control, or Other, and none has16/// an Indic_Conjunct_Break value, so only GB3, GB4, GB5, and GB999 apply17/// between two of them. Combining marks start at U+0300.18const SimpleLimit : Int32 = 0x3001920///|21/// Decode the code point starting at `index`. An unpaired surrogate is22/// returned as is; like any code point GraphemeBreakProperty.txt does not23/// list, its Grapheme_Cluster_Break value is Other.24fn code_point_at(text : StringView, index : Int32, length : Int32) -> Int32 {25  // SAFETY: callers guarantee 0 <= index < length.26  let first = text.unsafe_get(index)27  if first.is_leading_surrogate() && index + 1 < length {28    // SAFETY: index + 1 < length is checked above.29    let second = text.unsafe_get(index + 1)30    if second.is_trailing_surrogate() {31      return 0x10000 +32        ((first.to_int() - 0xD800) << 10) +33        second.to_int() -34        0xDC0035    }36  }37  first.to_int()38}3940///|41/// Return the end offset of the extended grapheme cluster starting at `start`,42/// following the boundary rules of UAX #29. Requires `start < text.length()`.43fn next_grapheme_boundary(text : StringView, start : Int32) -> Int32 {44  let length = text.length()45  // Fast path: two adjacent code points below `SimpleLimit` always break,46  // except CR LF (GB3).47  // SAFETY: the caller guarantees start < length.48  let first_unit = text.unsafe_get(start).to_int()49  if first_unit < SimpleLimit {50    if start + 1 == length {51      return length52    }53    // SAFETY: start + 1 < length is checked above.54    let second_unit = text.unsafe_get(start + 1).to_int()55    if second_unit < SimpleLimit {56      return if first_unit == '\r' && second_unit == '\n' {57        start + 258      } else {59        start + 160      }61    }62  }63  let first = code_point_at(text, start, length)64  let mut index = start + (if first >= 0x10000 { 2 } else { 1 })65  let mut previous = grapheme_break_value(first)66  // Whether the cluster ends in an odd number of Regional_Indicator code67  // points (GB12, GB13).68  let mut regional_odd = (previous & GcbMask) == GcbRegionalIndicator69  let mut emoji = if (previous & ExtendedPictographicBit) != 0 {70    EmojiPictographic71  } else {72    EmojiNone73  }74  // Whether the cluster ends in `\p{InCB=Linker} \p{InCB=Extend}*` (GB9c).75  let mut linker = ((previous >> InCBShift) & InCBMask) == InCBLinker76  while index < length {77    let code_point = code_point_at(text, index, length)78    let current = grapheme_break_value(code_point)79    let before = previous & GcbMask80    let after = current & GcbMask81    let conjunct_break = (current >> InCBShift) & InCBMask82    let pictographic = (current & ExtendedPictographicBit) != 083    // GB3, GB4, GB5: break around controls, except between CR and LF.84    match before {85      GcbCR => return if after == GcbLF { index + 1 } else { index }86      GcbLF | GcbControl => break87      _ => ()88    }89    if after is (GcbCR | GcbLF | GcbControl) {90      break91    }92    let joined = match (before, after) {93      // GB9, GB9a: do not break before extending characters, ZWJ, or94      // spacing marks.95      (_, GcbExtend | GcbZWJ | GcbSpacingMark) => true96      // GB9b: do not break after prepend characters.97      (GcbPrepend, _) => true98      // GB6, GB7, GB8: do not break Hangul syllable sequences.99      (GcbL, GcbL | GcbV | GcbLV | GcbLVT) => true100      (GcbLV | GcbV, GcbV | GcbT) => true101      (GcbLVT | GcbT, GcbT) => true102      // GB12, GB13: do not break within a pair of regional indicators.103      (GcbRegionalIndicator, GcbRegionalIndicator) => regional_odd104      _ =>105        // GB9c: do not break within an Indic conjunct.106        (linker && conjunct_break == InCBConsonant) ||107        // GB11: do not break within an emoji ZWJ sequence.108        (emoji == EmojiZWJ && pictographic)109    }110    // GB999: otherwise, break everywhere.111    if !joined {112      break113    }114    regional_odd = after == GcbRegionalIndicator && !regional_odd115    emoji = if pictographic {116      EmojiPictographic117    } else if emoji == EmojiPictographic && after == GcbExtend {118      EmojiPictographic119    } else if emoji == EmojiPictographic && after == GcbZWJ {120      EmojiZWJ121    } else {122      EmojiNone123    }124    linker = match conjunct_break {125      InCBLinker => true126      InCBExtend => linker127      _ => false128    }129    index += if code_point >= 0x10000 { 2 } else { 1 }130    previous = current131  }132  index133}134135///|136/// A single-pass cursor over the extended grapheme clusters of a text, the137/// user-perceived characters defined by Unicode Text Segmentation (UAX #29).138///139/// Create one with `graphemes`. Use it in a `for .. in` loop, or call `next`140/// directly to avoid allocating an `Option` per cluster.141struct Graphemes {142  text : StringView143  /// Offset in `text` of the next cluster to return.144  mut offset : Int32145  // Initialized only when line_break is queried.146  mut breaks : LineBreaks?147}148149///|150/// Split `text` into extended grapheme clusters.151///152/// Each cluster is a view into `text`, so `StringView::start_offset` gives its153/// position in the underlying string. The clusters are contiguous and cover154/// `text` exactly. `text` is segmented on its own: its start is a boundary155/// even if the underlying string continues before it. Unpaired surrogates have156/// Grapheme_Cluster_Break=Other; valid pairs stay together.157///158/// ```mbt check159/// test {160///   let clusters = []161///   for cluster in @segmentation.graphemes("e\u{301}🇺🇳x") {162///     clusters.push(cluster.to_owned())163///   }164///   debug_inspect(165///     clusters,166///     content=(167///       #|["é", "🇺🇳", "x"]168///     ),169///   )170/// }171/// ```172pub fn graphemes(text : StringView) -> Graphemes {173  { text, offset: 0, breaks: None, }174}175176///|177/// Return the next cluster and advance past it, or return an empty view once178/// the text is exhausted. A cluster is never empty, so an empty result always179/// means the end.180///181/// ```mbt check182/// test {183///   let clusters = @segmentation.graphemes("e\u{301}x")184///   inspect(185///     clusters.next(),186///     content=(187///       #|é188///     ),189///   )190///   inspect(191///     clusters.next(),192///     content=(193///       #|x194///     ),195///   )196///   inspect(197///     clusters.next().is_empty(),198///     content=(199///       #|true200///     ),201///   )202/// }203/// ```204pub fn Graphemes::next(self : Graphemes) -> StringView {205  let text = self.text206  let start = self.offset207  let end = if start < text.length() {208    next_grapheme_boundary(text, start)209  } else {210    start211  }212  self.offset = end213  // `start` and `end` come from `next_grapheme_boundary`, so they are in range214  // and on cluster boundaries: this needs no check or clamping per cluster.215  text.unsafe_view(start~, end~)216}217218///|219/// Iterate over the remaining clusters. The iterator advances this cursor, so220/// clusters it yields are not returned by `next` again.221pub fn Graphemes::iter(self : Graphemes) -> Iter[StringView] {222  Iter::new(() => {223    let cluster = self.next()224    if cluster.is_empty() {225      None226    } else {227      Some(cluster)228    }229  })230}231