unicode

Branch: main

Switch branches or tags

Branch: main
Go to file t

413 files indexed

History
Code

casefold/case_folding.mbt
1///|2/// Unicode default full case folding (section 3.13, R4).3/// Locale-independent: uses the C and F mappings in CaseFolding.txt, including4/// expansions such as ß -> ss. Does not normalize or apply Turkic tailoring.5pub fn case_fold(s : StringView) -> String {6  let index = Full.first_change(s)7  if index == s.length() {8    s.to_owned()9  } else {10    Full.fold_from(s, index)11  }12}1314///|15/// The folding tables of this package. Both fold ASCII alike, mapping only16/// A-Z, to a-z; the generator checks this.17priv enum Folding {18  /// Full default case folding, from CaseFolding.txt.19  Full20  /// NFKC_Casefold, before the final NFC.21  Nfkc22}2324///|25/// The generated kind of a code point that folds to itself.26const KindIdentity : Int32 = 02728///|29/// The generated kind of a code point whose mapping `Folding::lookup` finds.30const KindSearched : Int32 = 13132///|33fn Folding::kind(self : Folding, c : Char) -> Int32 {34  match self {35    Full => case_folding_kind(c)36    Nfkc => nfkc_casefold_kind(c)37  }38}3940///|41/// The code point delta of a kind from 2 on.42fn Folding::delta(self : Folding, kind : Int32) -> Int32 {43  match self {44    Full => case_folding_deltas[kind]45    Nfkc => nfkc_casefold_deltas[kind]46  }47}4849///|50/// The mapping of a code point of kind `KindSearched`.51fn Folding::lookup(self : Folding, c : Char) -> StringView? {52  match self {53    Full => lookup_case_folding(c)54    Nfkc => lookup_nfkc_casefold(c)55  }56}5758///|59/// The code point starting at `index`. An unpaired surrogate is returned as60/// it is, as iterating over the view does.61fn code_point_at(s : StringView, index : Int32, length : Int32) -> Int32 {62  // SAFETY: callers guarantee 0 <= index < length.63  let first = s.unsafe_get(index)64  if first.is_leading_surrogate() && index + 1 < length {65    // SAFETY: index + 1 < length is checked above.66    let second = s.unsafe_get(index + 1)67    if second.is_trailing_surrogate() {68      return 0x10000 +69        ((first.to_int() - 0xD800) << 10) +70        second.to_int() -71        0xDC0072    }73  }74  first.to_int()75}7677///|78/// The number of UTF-16 code units of `cp`.79fn utf16_width(cp : Int32) -> Int32 {80  if cp >= 0x10000 {81    282  } else {83    184  }85}8687///|88/// What the ASCII code point `cp` folds to, under either folding.89fn ascii_fold(cp : Int32) -> Int32 {90  if cp >= 'A' && cp <= 'Z' {91    cp + 3292  } else {93    cp94  }95}9697///|98/// The single code point `cp` folds to, itself included, or None when its99/// mapping is found with `Folding::lookup`.100fn Folding::fold_code_point(self : Folding, cp : Int32) -> Int32? {101  if cp < 0x80 {102    return Some(ascii_fold(cp))103  }104  match self.kind(cp.unsafe_to_char()) {105    KindIdentity => Some(cp)106    KindSearched => None107    kind => Some(cp + self.delta(kind))108  }109}110111///|112/// `Full.fold_code_point(cp)`, without the dispatch on the folding.113/// `caseless_match` calls it for every pair of code points that differ, and114/// there the dispatch costs about a fifth of the time.115fn simple_fold(cp : Int32) -> Int32? {116  if cp < 0x80 {117    return Some(ascii_fold(cp))118  }119  match case_folding_kind(cp.unsafe_to_char()) {120    KindIdentity => Some(cp)121    KindSearched => None122    kind => Some(cp + case_folding_deltas[kind])123  }124}125126///|127/// The offset of the first code point of `s` that does not fold to itself,128/// or the length of `s` when there is none.129fn Folding::first_change(self : Folding, s : StringView) -> Int32 {130  let length = s.length()131  let mut index = 0132  while index < length {133    // ASCII, most of most text, needs neither decoding nor the tables.134    // SAFETY: index < length.135    let unit = s.unsafe_get(index).to_int()136    if unit < 0x80 {137      if ascii_fold(unit) != unit {138        break139      }140      index += 1141      continue142    }143    let cp = code_point_at(s, index, length)144    if !(self.fold_code_point(cp) is Some(target) && target == cp) {145      break146    }147    index += utf16_width(cp)148  }149  index150}151152///|153/// Fold `s`, whose code points before `index` fold to themselves. Runs of154/// code points that fold to themselves are copied at once.155fn Folding::fold_from(self : Folding, s : StringView, index : Int32) -> String {156  let length = s.length()157  let out = StringBuilder(capacity=length)158  // Code units before `copied` are in `out`.159  let mut copied = 0160  for index = index; index < length; {161    // SAFETY: index < length.162    let unit = s.unsafe_get(index).to_int()163    if unit < 0x80 {164      let target = ascii_fold(unit)165      if target != unit {166        // SAFETY: copied <= index < length, both on code point boundaries.167        out.write_view(s.unsafe_view(start=copied, end=index))168        out.write_char(target.unsafe_to_char())169        copied = index + 1170      }171      continue index + 1172    }173    let cp = code_point_at(s, index, length)174    let width = utf16_width(cp)175    match self.fold_code_point(cp) {176      Some(target) if target == cp => ()177      folded => {178        // SAFETY: copied <= index < length, both on code point boundaries.179        out.write_view(s.unsafe_view(start=copied, end=index))180        if folded is Some(target) {181          out.write_char(target.unsafe_to_char())182        } else {183          let c = cp.unsafe_to_char()184          // Every code point of kind 1 has a mapping; a missing one would185          // mean identity.186          if self.lookup(c) is Some(mapping) {187            out.write_view(mapping)188          } else {189            out.write_char(c)190          }191        }192        copied = index + width193      }194    }195    continue index + width196  }197  // SAFETY: copied <= length, on a code point boundary.198  out.write_view(s.unsafe_view(start=copied, end=length))199  out.to_string()200}201