unicode

Branch: main

Switch branches or tags

Branch: main
Go to file t

413 files indexed

History
Code

ucd/case_mapping/string_case_mapping.mbt
1///|2/// Full uppercase mapping of a string: every code point is replaced with its3/// full uppercase mapping, as `full_uppercase` gives it, so "Straße" becomes4/// "STRASSE". Builds one string, with nothing allocated per character.5///6/// Context-free: the conditional mappings of SpecialCasing.txt, which depend7/// on the surrounding text or on the language, are not applied. Unpaired8/// surrogates are copied as they are.9pub fn uppercase(s : StringView) -> String {10  map_string(11    s,12    case_upper_deltas,13    1,14    special_upper_cps,15    special_upper_strings,16    AsciiCase::Upper,17  )18}1920///|21/// Unicode default full lowercase conversion of a string, including the22/// context-dependent Final_Sigma mapping: "ΟΣ" becomes "ος", while "ΟΣΑ"23/// becomes "οσα". Full mappings can expand characters, so "İ" becomes24/// "i\u{0307}". Language-specific mappings are not applied. Unpaired25/// surrogates are copied as they are.26pub fn lowercase(s : StringView) -> String {27  map_string(28    s,29    case_lower_deltas,30    2,31    special_lower_cps,32    special_lower_strings,33    AsciiCase::Lower,34  )35}3637///|38/// How a case maps ASCII: only letters change, a-z to A-Z or A-Z to a-z. A39/// test checks this against the tables.40priv enum AsciiCase {41  Upper42  Lower43}4445///|46/// Map `s` with one case's tables: `deltas` and the `flag` bit of47/// `case_special_flags` select the case, and the `special_*` arrays hold its48/// full mappings. `ascii` is how the same case maps ASCII.49///50/// The standard library maps ASCII letters several code units at a time on51/// these backends, so ASCII is mapped first, with one copy, and only the52/// other code points are looked up.53#cfg(not(target="js"))54fn map_string(55  s : StringView,56  deltas : ReadOnlyArray[Int32],57  flag : Int32,58  special_cps : ReadOnlyArray[Int32],59  special_strings : ReadOnlyArray[String],60  ascii : AsciiCase,61) -> String {62  let mapped = match ascii {63    Upper => s.to_upper()64    Lower => s.to_lower()65  }66  map_non_ascii(mapped, deltas, flag, special_cps, special_strings)67}6869///|70/// Map `s` with one case's tables, as the other backends do. On JavaScript71/// the standard library maps ASCII one character at a time into a new72/// string, so mapping it first would cost a second pass: ASCII letters are73/// mapped in the same pass instead.74#cfg(target="js")75fn map_string(76  s : StringView,77  deltas : ReadOnlyArray[Int32],78  flag : Int32,79  special_cps : ReadOnlyArray[Int32],80  special_strings : ReadOnlyArray[String],81  ascii : AsciiCase,82) -> String {83  let (ascii_first, ascii_delta) = match ascii {84    Upper => ('a'.to_int(), -32)85    Lower => ('A'.to_int(), 32)86  }87  let length = s.length()88  let out = StringBuilder(capacity=length)89  // Code units before `copied` are in `out`.90  let mut copied = 091  for index = 0; index < length; {92    // SAFETY: index < length.93    let unit = s.unsafe_get(index).to_int()94    if unit < 0x80 {95      if (unit - ascii_first).reinterpret_as_uint() < 26 {96        if copied < index {97          // SAFETY: copied < index < length, both on code point boundaries.98          out.write_view(s.unsafe_view(start=copied, end=index))99        }100        out.write_char((unit + ascii_delta).unsafe_to_char())101        copied = index + 1102      }103      continue index + 1104    }105    let cp = code_point_at(s, index, length)106    let width = utf16_width(cp)107    let record = lookup_case_record(cp.unsafe_to_char())108    let delta = if cp == 0x03A3 && flag == 2 && final_sigma(s, index) {109      0x03C2 - 0x03A3110    } else {111      deltas[record]112    }113    let special = (case_special_flags[record].to_int() & flag) != 0114    if delta != 0 || special {115      if copied < index {116        // SAFETY: copied < index < length, both on code point boundaries.117        out.write_view(s.unsafe_view(start=copied, end=index))118      }119      let found = if special { find_special(special_cps, cp) } else { -1 }120      if found >= 0 {121        out.write_string(special_strings[found])122      } else {123        out.write_char((cp + delta).unsafe_to_char())124      }125      copied = index + width126    }127    continue index + width128  }129  if copied == 0 {130    // Nothing changed.131    return s.to_owned()132  }133  // SAFETY: copied <= length, on a code point boundary.134  out.write_view(s.unsafe_view(start=copied, end=length))135  out.to_string()136}137138///|139/// Map the non-ASCII code points of `s`, whose ASCII characters are mapped140/// already, with one case's tables: `deltas` and the `flag` bit of141/// `case_special_flags` select the case, and the `special_*` arrays hold its142/// full mappings.143///144/// The standard library maps ASCII several code units at a time on native145/// and Wasm, so text that is mostly ASCII costs little more than one copy.146/// The unchanged prefix is found without building anything; from the first147/// change on, runs of unchanged code points are copied at once.148#cfg(not(target="js"))149fn map_non_ascii(150  s : StringView,151  deltas : ReadOnlyArray[Int32],152  flag : Int32,153  special_cps : ReadOnlyArray[Int32],154  special_strings : ReadOnlyArray[String],155) -> String {156  let length = s.length()157  let first_change = for index = 0; index < length; {158    // SAFETY: index < length.159    let unit = s.unsafe_get(index)160    if unit < 0x80 {161      continue index + ascii_run(s, unit, index, length)162    }163    let cp = code_point_at(s, index, length)164    let record = lookup_case_record(cp.unsafe_to_char())165    if deltas[record] != 0 || (case_special_flags[record].to_int() & flag) != 0 {166      break index167    }168    continue index + utf16_width(cp)169  } nobreak {170    // A view of a whole string gives that string back without a copy.171    return s.to_owned()172  }173  let out = StringBuilder(capacity=length)174  // Code units before `copied` are in `out`.175  let mut copied = 0176  for index = first_change; index < length; {177    // SAFETY: index < length.178    let unit = s.unsafe_get(index)179    if unit < 0x80 {180      continue index + ascii_run(s, unit, index, length)181    }182    let cp = code_point_at(s, index, length)183    let width = utf16_width(cp)184    let record = lookup_case_record(cp.unsafe_to_char())185    let delta = if cp == 0x03A3 && flag == 2 && final_sigma(s, index) {186      0x03C2 - 0x03A3187    } else {188      deltas[record]189    }190    let special = (case_special_flags[record].to_int() & flag) != 0191    if delta != 0 || special {192      // Changed characters often come in runs: skip the empty copy between.193      if copied < index {194        // SAFETY: copied < index < length, both on code point boundaries.195        out.write_view(s.unsafe_view(start=copied, end=index))196      }197      let found = if special { find_special(special_cps, cp) } else { -1 }198      if found >= 0 {199        out.write_string(special_strings[found])200      } else {201        out.write_char((cp + delta).unsafe_to_char())202      }203      copied = index + width204    }205    continue index + width206  }207  // SAFETY: copied <= length, on a code point boundary.208  out.write_view(s.unsafe_view(start=copied, end=length))209  out.to_string()210}211212///|213/// Unicode Final_Sigma context, evaluated only at a capital Σ. Ignorable214/// characters must be skipped before testing Cased: U+0345 has both properties.215/// ASCII pre-mapping preserves these properties, so the native/Wasm path can216/// inspect its ASCII-mapped string without changing the original context.217///218/// Σ itself is not ignorable. Each ignorable run is scanned at most once from219/// either end, keeping the complete conversion linear without a character array.220fn final_sigma(s : StringView, index : Int32) -> Bool {221  let preceded_by_cased = for end = index; end > 0; {222    let mut start = end - 1223    // SAFETY: 0 <= start < end <= index < s.length(). Do not pair a224    // trailing surrogate with a code unit outside this view's start.225    if s.unsafe_get(start).is_trailing_surrogate() &&226      start > 0 &&227      s.unsafe_get(start - 1).is_leading_surrogate() {228      start -= 1229    }230    let c = code_point_at(s, start, end).unsafe_to_char()231    if !is_case_ignorable(c) {232      break is_cased(c)233    }234    continue start235  } nobreak {236    false237  }238  if !preceded_by_cased {239    return false240  }241  let length = s.length()242  // The caller found a BMP Σ, so the following code point starts at index + 1.243  for start = index + 1; start < length; {244    let cp = code_point_at(s, start, length)245    if !is_case_ignorable(cp.unsafe_to_char()) {246      return !is_cased(cp.unsafe_to_char())247    }248    continue start + utf16_width(cp)249  }250  true251}252253///|254/// How far to skip from the ASCII code unit `unit` at `index`: four code255/// units when they are all ASCII, otherwise one. A single expression, so256/// that it is inlined into the loops.257#cfg(not(target="js"))258fn ascii_run(259  s : StringView,260  unit : UInt16,261  index : Int32,262  length : Int32,263) -> Int32 {264  // `length - index` cannot overflow, unlike `index + 4`.265  // SAFETY: index + 3 < length when the first test holds.266  if length - index >= 4 &&267    (268      unit |269      s.unsafe_get(index + 1) |270      s.unsafe_get(index + 2) |271      s.unsafe_get(index + 3)272    ) <273    0x80 {274    4275  } else {276    1277  }278}279280///|281/// The code point starting at `index`. An unpaired surrogate is returned as282/// it is, as iterating over the view does.283fn code_point_at(s : StringView, index : Int32, length : Int32) -> Int32 {284  // SAFETY: callers guarantee 0 <= index < length.285  let first = s.unsafe_get(index)286  if first.is_leading_surrogate() && index + 1 < length {287    // SAFETY: index + 1 < length is checked above.288    let second = s.unsafe_get(index + 1)289    if second.is_trailing_surrogate() {290      return 0x10000 +291        ((first.to_int() - 0xD800) << 10) +292        second.to_int() -293        0xDC00294    }295  }296  first.to_int()297}298299///|300/// The number of UTF-16 code units of `cp`.301fn utf16_width(cp : Int32) -> Int32 {302  if cp >= 0x10000 {303    2304  } else {305    1306  }307}308309///|310/// Index of `key` in the sorted array `cps`, or -1 if it is absent.311fn find_special(cps : ReadOnlyArray[Int32], key : Int32) -> Int32 {312  let mut left = 0313  let mut right = cps.length() - 1314  while left <= right {315    let mid = (left + right) / 2316    let value = cps[mid]317    if key < value {318      right = mid - 1319    } else if key > value {320      left = mid + 1321    } else {322      return mid323    }324  }325  -1326}327