unicode

Branch: main

Switch branches or tags

Branch: main
Go to file t

413 files indexed

History
Code

segmentation/line_break_context.mbt
1///|2/// One significant code point plus any marks ignored by LB9. Auxiliary3/// properties belong to the base; only the end and the last raw ZWJ flag4/// change when a combining mark is attached.5priv struct BreakUnit {6  class_ : @ucd.LineBreak7  code_point : Int328  initial_quote : Bool9  final_quote : Bool10  east_asian : Bool11  potential_emoji : Bool12  mut end : Int3213  mut ends_with_zwj : Bool14  // Last non-SP unit, including this unit. -1 means start of text.15  space_base : Int3216  // Whether the significant prefix ends in NU (SY | IS)*.17  numeric : Bool18  // Length of the significant RI suffix (LB9 marks do not count).19  regional_count : Int3220}2122///|23fn prepare_units(text : StringView) -> Array[BreakUnit] {24  let units : Array[BreakUnit] = []25  let mut offset = 026  while offset < text.length() {27    // SAFETY: offset is in the view; a second code unit is checked below.28    let first = text.unsafe_get(offset)29    let mut cp = first.to_int()30    offset += 131    if first.is_leading_surrogate() && offset < text.length() {32      let second = text.unsafe_get(offset)33      if second.is_trailing_surrogate() {34        cp = 0x10000 + ((cp - 0xD800) << 10) + second.to_int() - 0xDC0035        offset += 136      }37    }38    // Unpaired surrogates have SG, which LB1 resolves to AL. They cannot39    // participate in the auxiliary-property rules, so use A as their proxy.40    let char = (if cp >= 0xD800 && cp <= 0xDFFF { 0x41 } else { cp }).unsafe_to_char()41    let original = @ucd.line_break(char)42    let category = @ucd.general_category(char)43    // LB1: default resolution only; never alter the UCD property query.44    let mut class_ = match original {45      AI | SG | XX => @ucd.AL46      CJ => @ucd.NS47      SA => if category is (Mn | Mc) { @ucd.CM } else { @ucd.AL }48      _ => original49    }50    let last = units.last()51    if class_ is (CM | ZWJ) {52      // LB9. Keep the raw final ZWJ separately for the earlier rule LB8a.53      if last is Some(base) && !(base.class_ is (BK | CR | LF | NL | SP | ZW)) {54        base.end = offset55        base.ends_with_zwj = class_ == @ucd.ZWJ56        continue57      }58    }59    // LB10 replaces *all* auxiliary properties of remaining marks with A's.60    let remaining_mark = class_ is (CM | ZWJ)61    if remaining_mark {62      class_ = @ucd.AL63    }64    let effective_char = if remaining_mark { 'A' } else { char }65    let east_asian = @ucd.east_asian_width(effective_char) is (F | W | H)66    let index = units.length()67    units.push({68      class_,69      code_point: effective_char.to_int(),70      initial_quote: !remaining_mark && category == @ucd.Pi,71      final_quote: !remaining_mark && category == @ucd.Pf,72      east_asian,73      potential_emoji: !remaining_mark &&74      category == @ucd.Cn &&75      @ucd.is_extended_pictographic(char),76      end: offset,77      ends_with_zwj: original == @ucd.ZWJ,78      space_base: if class_ == @ucd.SP {79        last.map(u => u.space_base).unwrap_or(-1)80      } else {81        index82      },83      numeric: class_ == @ucd.NU ||84      (class_ is (SY | IS) && last.map(u => u.numeric).unwrap_or(false)),85      regional_count: if class_ == @ucd.RI {86        last.map(u => u.regional_count).unwrap_or(0) + 187      } else {88        089      },90    })91  }92  units93}9495///|96/// XX is a sentinel here: LB1 has removed it from the actual input.97fn class_at(units : Array[BreakUnit], index : Int32) -> @ucd.LineBreak {98  units.get(index).map(u => u.class_).unwrap_or(@ucd.XX)99}100101///|102fn BreakUnit::aksara(self : BreakUnit) -> Bool {103  self.class_ is (AK | AS) || self.code_point == 0x25CC104}105