casefold/case_folding.mbt1///|2/// Unicode default full case folding (section 3.13, R4).3/// Locale-independent: uses the C and F mappings in CaseFolding.txt, including4/// expansions such as ß -> ss. Does not normalize or apply Turkic tailoring.5pub fn case_fold(s : StringView) -> String {6 let index = Full.first_change(s)7 if index == s.length() {8 s.to_owned()9 } else {10 Full.fold_from(s, index)11 }12}1314///|15/// The folding tables of this package. Both fold ASCII alike, mapping only16/// A-Z, to a-z; the generator checks this.17priv enum Folding {18 /// Full default case folding, from CaseFolding.txt.19 Full20 /// NFKC_Casefold, before the final NFC.21 Nfkc22}2324///|25/// The generated kind of a code point that folds to itself.26const KindIdentity : Int32 = 02728///|29/// The generated kind of a code point whose mapping `Folding::lookup` finds.30const KindSearched : Int32 = 13132///|33fn Folding::kind(self : Folding, c : Char) -> Int32 {34 match self {35 Full => case_folding_kind(c)36 Nfkc => nfkc_casefold_kind(c)37 }38}3940///|41/// The code point delta of a kind from 2 on.42fn Folding::delta(self : Folding, kind : Int32) -> Int32 {43 match self {44 Full => case_folding_deltas[kind]45 Nfkc => nfkc_casefold_deltas[kind]46 }47}4849///|50/// The mapping of a code point of kind `KindSearched`.51fn Folding::lookup(self : Folding, c : Char) -> StringView? {52 match self {53 Full => lookup_case_folding(c)54 Nfkc => lookup_nfkc_casefold(c)55 }56}5758///|59/// The code point starting at `index`. An unpaired surrogate is returned as60/// it is, as iterating over the view does.61fn code_point_at(s : StringView, index : Int32, length : Int32) -> Int32 {62 // SAFETY: callers guarantee 0 <= index < length.63 let first = s.unsafe_get(index)64 if first.is_leading_surrogate() && index + 1 < length {65 // SAFETY: index + 1 < length is checked above.66 let second = s.unsafe_get(index + 1)67 if second.is_trailing_surrogate() {68 return 0x10000 +69 ((first.to_int() - 0xD800) << 10) +70 second.to_int() -71 0xDC0072 }73 }74 first.to_int()75}7677///|78/// The number of UTF-16 code units of `cp`.79fn utf16_width(cp : Int32) -> Int32 {80 if cp >= 0x10000 {81 282 } else {83 184 }85}8687///|88/// What the ASCII code point `cp` folds to, under either folding.89fn ascii_fold(cp : Int32) -> Int32 {90 if cp >= 'A' && cp <= 'Z' {91 cp + 3292 } else {93 cp94 }95}9697///|98/// The single code point `cp` folds to, itself included, or None when its99/// mapping is found with `Folding::lookup`.100fn Folding::fold_code_point(self : Folding, cp : Int32) -> Int32? {101 if cp < 0x80 {102 return Some(ascii_fold(cp))103 }104 match self.kind(cp.unsafe_to_char()) {105 KindIdentity => Some(cp)106 KindSearched => None107 kind => Some(cp + self.delta(kind))108 }109}110111///|112/// `Full.fold_code_point(cp)`, without the dispatch on the folding.113/// `caseless_match` calls it for every pair of code points that differ, and114/// there the dispatch costs about a fifth of the time.115fn simple_fold(cp : Int32) -> Int32? {116 if cp < 0x80 {117 return Some(ascii_fold(cp))118 }119 match case_folding_kind(cp.unsafe_to_char()) {120 KindIdentity => Some(cp)121 KindSearched => None122 kind => Some(cp + case_folding_deltas[kind])123 }124}125126///|127/// The offset of the first code point of `s` that does not fold to itself,128/// or the length of `s` when there is none.129fn Folding::first_change(self : Folding, s : StringView) -> Int32 {130 let length = s.length()131 let mut index = 0132 while index < length {133 // ASCII, most of most text, needs neither decoding nor the tables.134 // SAFETY: index < length.135 let unit = s.unsafe_get(index).to_int()136 if unit < 0x80 {137 if ascii_fold(unit) != unit {138 break139 }140 index += 1141 continue142 }143 let cp = code_point_at(s, index, length)144 if !(self.fold_code_point(cp) is Some(target) && target == cp) {145 break146 }147 index += utf16_width(cp)148 }149 index150}151152///|153/// Fold `s`, whose code points before `index` fold to themselves. Runs of154/// code points that fold to themselves are copied at once.155fn Folding::fold_from(self : Folding, s : StringView, index : Int32) -> String {156 let length = s.length()157 let out = StringBuilder(capacity=length)158 // Code units before `copied` are in `out`.159 let mut copied = 0160 for index = index; index < length; {161 // SAFETY: index < length.162 let unit = s.unsafe_get(index).to_int()163 if unit < 0x80 {164 let target = ascii_fold(unit)165 if target != unit {166 // SAFETY: copied <= index < length, both on code point boundaries.167 out.write_view(s.unsafe_view(start=copied, end=index))168 out.write_char(target.unsafe_to_char())169 copied = index + 1170 }171 continue index + 1172 }173 let cp = code_point_at(s, index, length)174 let width = utf16_width(cp)175 match self.fold_code_point(cp) {176 Some(target) if target == cp => ()177 folded => {178 // SAFETY: copied <= index < length, both on code point boundaries.179 out.write_view(s.unsafe_view(start=copied, end=index))180 if folded is Some(target) {181 out.write_char(target.unsafe_to_char())182 } else {183 let c = cp.unsafe_to_char()184 // Every code point of kind 1 has a mapping; a missing one would185 // mean identity.186 if self.lookup(c) is Some(mapping) {187 out.write_view(mapping)188 } else {189 out.write_char(c)190 }191 }192 copied = index + width193 }194 }195 continue index + width196 }197 // SAFETY: copied <= length, on a code point boundary.198 out.write_view(s.unsafe_view(start=copied, end=length))199 out.to_string()200}201
///|
/// Unicode default full case folding (section 3.13, R4).
/// Locale-independent: uses the C and F mappings in CaseFolding.txt, including
/// expansions such as ß -> ss. Does not normalize or apply Turkic tailoring.
pub fn case_fold(s : StringView) -> String {
let index = Full.first_change(s)
if index == s.length() {
s.to_owned()
} else {
Full.fold_from(s, index)
}
}
///|
/// The folding tables of this package. Both fold ASCII alike, mapping only
/// A-Z, to a-z; the generator checks this.
priv enum Folding {
/// Full default case folding, from CaseFolding.txt.
Full
/// NFKC_Casefold, before the final NFC.
Nfkc
}
///|
/// The generated kind of a code point that folds to itself.
const KindIdentity : Int32 = 0
///|
/// The generated kind of a code point whose mapping `Folding::lookup` finds.
const KindSearched : Int32 = 1
///|
fn Folding::kind(self : Folding, c : Char) -> Int32 {
match self {
Full => case_folding_kind(c)
Nfkc => nfkc_casefold_kind(c)
}
}
///|
/// The code point delta of a kind from 2 on.
fn Folding::delta(self : Folding, kind : Int32) -> Int32 {
match self {
Full => case_folding_deltas[kind]
Nfkc => nfkc_casefold_deltas[kind]
}
}
///|
/// The mapping of a code point of kind `KindSearched`.
fn Folding::lookup(self : Folding, c : Char) -> StringView? {
match self {
Full => lookup_case_folding(c)
Nfkc => lookup_nfkc_casefold(c)
}
}
///|
/// The code point starting at `index`. An unpaired surrogate is returned as
/// it is, as iterating over the view does.
fn code_point_at(s : StringView, index : Int32, length : Int32) -> Int32 {
// SAFETY: callers guarantee 0 <= index < length.
let first = s.unsafe_get(index)
if first.is_leading_surrogate() && index + 1 < length {
// SAFETY: index + 1 < length is checked above.
let second = s.unsafe_get(index + 1)
if second.is_trailing_surrogate() {
return 0x10000 +
((first.to_int() - 0xD800) << 10) +
second.to_int() -
0xDC00
}
}
first.to_int()
}
///|
/// The number of UTF-16 code units of `cp`.
fn utf16_width(cp : Int32) -> Int32 {
if cp >= 0x10000 {
2
} else {
1
}
}
///|
/// What the ASCII code point `cp` folds to, under either folding.
fn ascii_fold(cp : Int32) -> Int32 {
if cp >= 'A' && cp <= 'Z' {
cp + 32
} else {
cp
}
}
///|
/// The single code point `cp` folds to, itself included, or None when its
/// mapping is found with `Folding::lookup`.
fn Folding::fold_code_point(self : Folding, cp : Int32) -> Int32? {
if cp < 0x80 {
return Some(ascii_fold(cp))
}
match self.kind(cp.unsafe_to_char()) {
KindIdentity => Some(cp)
KindSearched => None
kind => Some(cp + self.delta(kind))
}
}
///|
/// `Full.fold_code_point(cp)`, without the dispatch on the folding.
/// `caseless_match` calls it for every pair of code points that differ, and
/// there the dispatch costs about a fifth of the time.
fn simple_fold(cp : Int32) -> Int32? {
if cp < 0x80 {
return Some(ascii_fold(cp))
}
match case_folding_kind(cp.unsafe_to_char()) {
KindIdentity => Some(cp)
KindSearched => None
kind => Some(cp + case_folding_deltas[kind])
}
}
///|
/// The offset of the first code point of `s` that does not fold to itself,
/// or the length of `s` when there is none.
fn Folding::first_change(self : Folding, s : StringView) -> Int32 {
let length = s.length()
let mut index = 0
while index < length {
// ASCII, most of most text, needs neither decoding nor the tables.
// SAFETY: index < length.
let unit = s.unsafe_get(index).to_int()
if unit < 0x80 {
if ascii_fold(unit) != unit {
break
}
index += 1
continue
}
let cp = code_point_at(s, index, length)
if !(self.fold_code_point(cp) is Some(target) && target == cp) {
break
}
index += utf16_width(cp)
}
index
}
///|
/// Fold `s`, whose code points before `index` fold to themselves. Runs of
/// code points that fold to themselves are copied at once.
fn Folding::fold_from(self : Folding, s : StringView, index : Int32) -> String {
let length = s.length()
let out = StringBuilder(capacity=length)
// Code units before `copied` are in `out`.
let mut copied = 0
for index = index; index < length; {
// SAFETY: index < length.
let unit = s.unsafe_get(index).to_int()
if unit < 0x80 {
let target = ascii_fold(unit)
if target != unit {
// SAFETY: copied <= index < length, both on code point boundaries.
out.write_view(s.unsafe_view(start=copied, end=index))
out.write_char(target.unsafe_to_char())
copied = index + 1
}
continue index + 1
}
let cp = code_point_at(s, index, length)
let width = utf16_width(cp)
match self.fold_code_point(cp) {
Some(target) if target == cp => ()
folded => {
// SAFETY: copied <= index < length, both on code point boundaries.
out.write_view(s.unsafe_view(start=copied, end=index))
if folded is Some(target) {
out.write_char(target.unsafe_to_char())
} else {
let c = cp.unsafe_to_char()
// Every code point of kind 1 has a mapping; a missing one would
// mean identity.
if self.lookup(c) is Some(mapping) {
out.write_view(mapping)
} else {
out.write_char(c)
}
}
copied = index + width
}
}
continue index + width
}
// SAFETY: copied <= length, on a code point boundary.
out.write_view(s.unsafe_view(start=copied, end=length))
out.to_string()
}