ucd/case_mapping/string_case_mapping.mbt1///|2/// Full uppercase mapping of a string: every code point is replaced with its3/// full uppercase mapping, as `full_uppercase` gives it, so "Straße" becomes4/// "STRASSE". Builds one string, with nothing allocated per character.5///6/// Context-free: the conditional mappings of SpecialCasing.txt, which depend7/// on the surrounding text or on the language, are not applied. Unpaired8/// surrogates are copied as they are.9pub fn uppercase(s : StringView) -> String {10 map_string(11 s,12 case_upper_deltas,13 1,14 special_upper_cps,15 special_upper_strings,16 AsciiCase::Upper,17 )18}1920///|21/// Unicode default full lowercase conversion of a string, including the22/// context-dependent Final_Sigma mapping: "ΟΣ" becomes "ος", while "ΟΣΑ"23/// becomes "οσα". Full mappings can expand characters, so "İ" becomes24/// "i\u{0307}". Language-specific mappings are not applied. Unpaired25/// surrogates are copied as they are.26pub fn lowercase(s : StringView) -> String {27 map_string(28 s,29 case_lower_deltas,30 2,31 special_lower_cps,32 special_lower_strings,33 AsciiCase::Lower,34 )35}3637///|38/// How a case maps ASCII: only letters change, a-z to A-Z or A-Z to a-z. A39/// test checks this against the tables.40priv enum AsciiCase {41 Upper42 Lower43}4445///|46/// Map `s` with one case's tables: `deltas` and the `flag` bit of47/// `case_special_flags` select the case, and the `special_*` arrays hold its48/// full mappings. `ascii` is how the same case maps ASCII.49///50/// The standard library maps ASCII letters several code units at a time on51/// these backends, so ASCII is mapped first, with one copy, and only the52/// other code points are looked up.53#cfg(not(target="js"))54fn map_string(55 s : StringView,56 deltas : ReadOnlyArray[Int32],57 flag : Int32,58 special_cps : ReadOnlyArray[Int32],59 special_strings : ReadOnlyArray[String],60 ascii : AsciiCase,61) -> String {62 let mapped = match ascii {63 Upper => s.to_upper()64 Lower => s.to_lower()65 }66 map_non_ascii(mapped, deltas, flag, special_cps, special_strings)67}6869///|70/// Map `s` with one case's tables, as the other backends do. On JavaScript71/// the standard library maps ASCII one character at a time into a new72/// string, so mapping it first would cost a second pass: ASCII letters are73/// mapped in the same pass instead.74#cfg(target="js")75fn map_string(76 s : StringView,77 deltas : ReadOnlyArray[Int32],78 flag : Int32,79 special_cps : ReadOnlyArray[Int32],80 special_strings : ReadOnlyArray[String],81 ascii : AsciiCase,82) -> String {83 let (ascii_first, ascii_delta) = match ascii {84 Upper => ('a'.to_int(), -32)85 Lower => ('A'.to_int(), 32)86 }87 let length = s.length()88 let out = StringBuilder(capacity=length)89 // Code units before `copied` are in `out`.90 let mut copied = 091 for index = 0; index < length; {92 // SAFETY: index < length.93 let unit = s.unsafe_get(index).to_int()94 if unit < 0x80 {95 if (unit - ascii_first).reinterpret_as_uint() < 26 {96 if copied < index {97 // SAFETY: copied < index < length, both on code point boundaries.98 out.write_view(s.unsafe_view(start=copied, end=index))99 }100 out.write_char((unit + ascii_delta).unsafe_to_char())101 copied = index + 1102 }103 continue index + 1104 }105 let cp = code_point_at(s, index, length)106 let width = utf16_width(cp)107 let record = lookup_case_record(cp.unsafe_to_char())108 let delta = if cp == 0x03A3 && flag == 2 && final_sigma(s, index) {109 0x03C2 - 0x03A3110 } else {111 deltas[record]112 }113 let special = (case_special_flags[record].to_int() & flag) != 0114 if delta != 0 || special {115 if copied < index {116 // SAFETY: copied < index < length, both on code point boundaries.117 out.write_view(s.unsafe_view(start=copied, end=index))118 }119 let found = if special { find_special(special_cps, cp) } else { -1 }120 if found >= 0 {121 out.write_string(special_strings[found])122 } else {123 out.write_char((cp + delta).unsafe_to_char())124 }125 copied = index + width126 }127 continue index + width128 }129 if copied == 0 {130 // Nothing changed.131 return s.to_owned()132 }133 // SAFETY: copied <= length, on a code point boundary.134 out.write_view(s.unsafe_view(start=copied, end=length))135 out.to_string()136}137138///|139/// Map the non-ASCII code points of `s`, whose ASCII characters are mapped140/// already, with one case's tables: `deltas` and the `flag` bit of141/// `case_special_flags` select the case, and the `special_*` arrays hold its142/// full mappings.143///144/// The standard library maps ASCII several code units at a time on native145/// and Wasm, so text that is mostly ASCII costs little more than one copy.146/// The unchanged prefix is found without building anything; from the first147/// change on, runs of unchanged code points are copied at once.148#cfg(not(target="js"))149fn map_non_ascii(150 s : StringView,151 deltas : ReadOnlyArray[Int32],152 flag : Int32,153 special_cps : ReadOnlyArray[Int32],154 special_strings : ReadOnlyArray[String],155) -> String {156 let length = s.length()157 let first_change = for index = 0; index < length; {158 // SAFETY: index < length.159 let unit = s.unsafe_get(index)160 if unit < 0x80 {161 continue index + ascii_run(s, unit, index, length)162 }163 let cp = code_point_at(s, index, length)164 let record = lookup_case_record(cp.unsafe_to_char())165 if deltas[record] != 0 || (case_special_flags[record].to_int() & flag) != 0 {166 break index167 }168 continue index + utf16_width(cp)169 } nobreak {170 // A view of a whole string gives that string back without a copy.171 return s.to_owned()172 }173 let out = StringBuilder(capacity=length)174 // Code units before `copied` are in `out`.175 let mut copied = 0176 for index = first_change; index < length; {177 // SAFETY: index < length.178 let unit = s.unsafe_get(index)179 if unit < 0x80 {180 continue index + ascii_run(s, unit, index, length)181 }182 let cp = code_point_at(s, index, length)183 let width = utf16_width(cp)184 let record = lookup_case_record(cp.unsafe_to_char())185 let delta = if cp == 0x03A3 && flag == 2 && final_sigma(s, index) {186 0x03C2 - 0x03A3187 } else {188 deltas[record]189 }190 let special = (case_special_flags[record].to_int() & flag) != 0191 if delta != 0 || special {192 // Changed characters often come in runs: skip the empty copy between.193 if copied < index {194 // SAFETY: copied < index < length, both on code point boundaries.195 out.write_view(s.unsafe_view(start=copied, end=index))196 }197 let found = if special { find_special(special_cps, cp) } else { -1 }198 if found >= 0 {199 out.write_string(special_strings[found])200 } else {201 out.write_char((cp + delta).unsafe_to_char())202 }203 copied = index + width204 }205 continue index + width206 }207 // SAFETY: copied <= length, on a code point boundary.208 out.write_view(s.unsafe_view(start=copied, end=length))209 out.to_string()210}211212///|213/// Unicode Final_Sigma context, evaluated only at a capital Σ. Ignorable214/// characters must be skipped before testing Cased: U+0345 has both properties.215/// ASCII pre-mapping preserves these properties, so the native/Wasm path can216/// inspect its ASCII-mapped string without changing the original context.217///218/// Σ itself is not ignorable. Each ignorable run is scanned at most once from219/// either end, keeping the complete conversion linear without a character array.220fn final_sigma(s : StringView, index : Int32) -> Bool {221 let preceded_by_cased = for end = index; end > 0; {222 let mut start = end - 1223 // SAFETY: 0 <= start < end <= index < s.length(). Do not pair a224 // trailing surrogate with a code unit outside this view's start.225 if s.unsafe_get(start).is_trailing_surrogate() &&226 start > 0 &&227 s.unsafe_get(start - 1).is_leading_surrogate() {228 start -= 1229 }230 let c = code_point_at(s, start, end).unsafe_to_char()231 if !is_case_ignorable(c) {232 break is_cased(c)233 }234 continue start235 } nobreak {236 false237 }238 if !preceded_by_cased {239 return false240 }241 let length = s.length()242 // The caller found a BMP Σ, so the following code point starts at index + 1.243 for start = index + 1; start < length; {244 let cp = code_point_at(s, start, length)245 if !is_case_ignorable(cp.unsafe_to_char()) {246 return !is_cased(cp.unsafe_to_char())247 }248 continue start + utf16_width(cp)249 }250 true251}252253///|254/// How far to skip from the ASCII code unit `unit` at `index`: four code255/// units when they are all ASCII, otherwise one. A single expression, so256/// that it is inlined into the loops.257#cfg(not(target="js"))258fn ascii_run(259 s : StringView,260 unit : UInt16,261 index : Int32,262 length : Int32,263) -> Int32 {264 // `length - index` cannot overflow, unlike `index + 4`.265 // SAFETY: index + 3 < length when the first test holds.266 if length - index >= 4 &&267 (268 unit |269 s.unsafe_get(index + 1) |270 s.unsafe_get(index + 2) |271 s.unsafe_get(index + 3)272 ) <273 0x80 {274 4275 } else {276 1277 }278}279280///|281/// The code point starting at `index`. An unpaired surrogate is returned as282/// it is, as iterating over the view does.283fn code_point_at(s : StringView, index : Int32, length : Int32) -> Int32 {284 // SAFETY: callers guarantee 0 <= index < length.285 let first = s.unsafe_get(index)286 if first.is_leading_surrogate() && index + 1 < length {287 // SAFETY: index + 1 < length is checked above.288 let second = s.unsafe_get(index + 1)289 if second.is_trailing_surrogate() {290 return 0x10000 +291 ((first.to_int() - 0xD800) << 10) +292 second.to_int() -293 0xDC00294 }295 }296 first.to_int()297}298299///|300/// The number of UTF-16 code units of `cp`.301fn utf16_width(cp : Int32) -> Int32 {302 if cp >= 0x10000 {303 2304 } else {305 1306 }307}308309///|310/// Index of `key` in the sorted array `cps`, or -1 if it is absent.311fn find_special(cps : ReadOnlyArray[Int32], key : Int32) -> Int32 {312 let mut left = 0313 let mut right = cps.length() - 1314 while left <= right {315 let mid = (left + right) / 2316 let value = cps[mid]317 if key < value {318 right = mid - 1319 } else if key > value {320 left = mid + 1321 } else {322 return mid323 }324 }325 -1326}327
///|
/// Full uppercase mapping of a string: every code point is replaced with its
/// full uppercase mapping, as `full_uppercase` gives it, so "Straße" becomes
/// "STRASSE". Builds one string, with nothing allocated per character.
///
/// Context-free: the conditional mappings of SpecialCasing.txt, which depend
/// on the surrounding text or on the language, are not applied. Unpaired
/// surrogates are copied as they are.
pub fn uppercase(s : StringView) -> String {
map_string(
s,
case_upper_deltas,
1,
special_upper_cps,
special_upper_strings,
AsciiCase::Upper,
)
}
///|
/// Unicode default full lowercase conversion of a string, including the
/// context-dependent Final_Sigma mapping: "ΟΣ" becomes "ος", while "ΟΣΑ"
/// becomes "οσα". Full mappings can expand characters, so "İ" becomes
/// "i\u{0307}". Language-specific mappings are not applied. Unpaired
/// surrogates are copied as they are.
pub fn lowercase(s : StringView) -> String {
map_string(
s,
case_lower_deltas,
2,
special_lower_cps,
special_lower_strings,
AsciiCase::Lower,
)
}
///|
/// How a case maps ASCII: only letters change, a-z to A-Z or A-Z to a-z. A
/// test checks this against the tables.
priv enum AsciiCase {
Upper
Lower
}
///|
/// Map `s` with one case's tables: `deltas` and the `flag` bit of
/// `case_special_flags` select the case, and the `special_*` arrays hold its
/// full mappings. `ascii` is how the same case maps ASCII.
///
/// The standard library maps ASCII letters several code units at a time on
/// these backends, so ASCII is mapped first, with one copy, and only the
/// other code points are looked up.
#cfg(not(target="js"))
fn map_string(
s : StringView,
deltas : ReadOnlyArray[Int32],
flag : Int32,
special_cps : ReadOnlyArray[Int32],
special_strings : ReadOnlyArray[String],
ascii : AsciiCase,
) -> String {
let mapped = match ascii {
Upper => s.to_upper()
Lower => s.to_lower()
}
map_non_ascii(mapped, deltas, flag, special_cps, special_strings)
}
///|
/// Map `s` with one case's tables, as the other backends do. On JavaScript
/// the standard library maps ASCII one character at a time into a new
/// string, so mapping it first would cost a second pass: ASCII letters are
/// mapped in the same pass instead.
#cfg(target="js")
fn map_string(
s : StringView,
deltas : ReadOnlyArray[Int32],
flag : Int32,
special_cps : ReadOnlyArray[Int32],
special_strings : ReadOnlyArray[String],
ascii : AsciiCase,
) -> String {
let (ascii_first, ascii_delta) = match ascii {
Upper => ('a'.to_int(), -32)
Lower => ('A'.to_int(), 32)
}
let length = s.length()
let out = StringBuilder(capacity=length)
// Code units before `copied` are in `out`.
let mut copied = 0
for index = 0; index < length; {
// SAFETY: index < length.
let unit = s.unsafe_get(index).to_int()
if unit < 0x80 {
if (unit - ascii_first).reinterpret_as_uint() < 26 {
if copied < index {
// SAFETY: copied < index < length, both on code point boundaries.
out.write_view(s.unsafe_view(start=copied, end=index))
}
out.write_char((unit + ascii_delta).unsafe_to_char())
copied = index + 1
}
continue index + 1
}
let cp = code_point_at(s, index, length)
let width = utf16_width(cp)
let record = lookup_case_record(cp.unsafe_to_char())
let delta = if cp == 0x03A3 && flag == 2 && final_sigma(s, index) {
0x03C2 - 0x03A3
} else {
deltas[record]
}
let special = (case_special_flags[record].to_int() & flag) != 0
if delta != 0 || special {
if copied < index {
// SAFETY: copied < index < length, both on code point boundaries.
out.write_view(s.unsafe_view(start=copied, end=index))
}
let found = if special { find_special(special_cps, cp) } else { -1 }
if found >= 0 {
out.write_string(special_strings[found])
} else {
out.write_char((cp + delta).unsafe_to_char())
}
copied = index + width
}
continue index + width
}
if copied == 0 {
// Nothing changed.
return s.to_owned()
}
// SAFETY: copied <= length, on a code point boundary.
out.write_view(s.unsafe_view(start=copied, end=length))
out.to_string()
}
///|
/// Map the non-ASCII code points of `s`, whose ASCII characters are mapped
/// already, with one case's tables: `deltas` and the `flag` bit of
/// `case_special_flags` select the case, and the `special_*` arrays hold its
/// full mappings.
///
/// The standard library maps ASCII several code units at a time on native
/// and Wasm, so text that is mostly ASCII costs little more than one copy.
/// The unchanged prefix is found without building anything; from the first
/// change on, runs of unchanged code points are copied at once.
#cfg(not(target="js"))
fn map_non_ascii(
s : StringView,
deltas : ReadOnlyArray[Int32],
flag : Int32,
special_cps : ReadOnlyArray[Int32],
special_strings : ReadOnlyArray[String],
) -> String {
let length = s.length()
let first_change = for index = 0; index < length; {
// SAFETY: index < length.
let unit = s.unsafe_get(index)
if unit < 0x80 {
continue index + ascii_run(s, unit, index, length)
}
let cp = code_point_at(s, index, length)
let record = lookup_case_record(cp.unsafe_to_char())
if deltas[record] != 0 || (case_special_flags[record].to_int() & flag) != 0 {
break index
}
continue index + utf16_width(cp)
} nobreak {
// A view of a whole string gives that string back without a copy.
return s.to_owned()
}
let out = StringBuilder(capacity=length)
// Code units before `copied` are in `out`.
let mut copied = 0
for index = first_change; index < length; {
// SAFETY: index < length.
let unit = s.unsafe_get(index)
if unit < 0x80 {
continue index + ascii_run(s, unit, index, length)
}
let cp = code_point_at(s, index, length)
let width = utf16_width(cp)
let record = lookup_case_record(cp.unsafe_to_char())
let delta = if cp == 0x03A3 && flag == 2 && final_sigma(s, index) {
0x03C2 - 0x03A3
} else {
deltas[record]
}
let special = (case_special_flags[record].to_int() & flag) != 0
if delta != 0 || special {
// Changed characters often come in runs: skip the empty copy between.
if copied < index {
// SAFETY: copied < index < length, both on code point boundaries.
out.write_view(s.unsafe_view(start=copied, end=index))
}
let found = if special { find_special(special_cps, cp) } else { -1 }
if found >= 0 {
out.write_string(special_strings[found])
} else {
out.write_char((cp + delta).unsafe_to_char())
}
copied = index + width
}
continue index + width
}
// SAFETY: copied <= length, on a code point boundary.
out.write_view(s.unsafe_view(start=copied, end=length))
out.to_string()
}
///|
/// Unicode Final_Sigma context, evaluated only at a capital Σ. Ignorable
/// characters must be skipped before testing Cased: U+0345 has both properties.
/// ASCII pre-mapping preserves these properties, so the native/Wasm path can
/// inspect its ASCII-mapped string without changing the original context.
///
/// Σ itself is not ignorable. Each ignorable run is scanned at most once from
/// either end, keeping the complete conversion linear without a character array.
fn final_sigma(s : StringView, index : Int32) -> Bool {
let preceded_by_cased = for end = index; end > 0; {
let mut start = end - 1
// SAFETY: 0 <= start < end <= index < s.length(). Do not pair a
// trailing surrogate with a code unit outside this view's start.
if s.unsafe_get(start).is_trailing_surrogate() &&
start > 0 &&
s.unsafe_get(start - 1).is_leading_surrogate() {
start -= 1
}
let c = code_point_at(s, start, end).unsafe_to_char()
if !is_case_ignorable(c) {
break is_cased(c)
}
continue start
} nobreak {
false
}
if !preceded_by_cased {
return false
}
let length = s.length()
// The caller found a BMP Σ, so the following code point starts at index + 1.
for start = index + 1; start < length; {
let cp = code_point_at(s, start, length)
if !is_case_ignorable(cp.unsafe_to_char()) {
return !is_cased(cp.unsafe_to_char())
}
continue start + utf16_width(cp)
}
true
}
///|
/// How far to skip from the ASCII code unit `unit` at `index`: four code
/// units when they are all ASCII, otherwise one. A single expression, so
/// that it is inlined into the loops.
#cfg(not(target="js"))
fn ascii_run(
s : StringView,
unit : UInt16,
index : Int32,
length : Int32,
) -> Int32 {
// `length - index` cannot overflow, unlike `index + 4`.
// SAFETY: index + 3 < length when the first test holds.
if length - index >= 4 &&
(
unit |
s.unsafe_get(index + 1) |
s.unsafe_get(index + 2) |
s.unsafe_get(index + 3)
) <
0x80 {
4
} else {
1
}
}
///|
/// The code point starting at `index`. An unpaired surrogate is returned as
/// it is, as iterating over the view does.
fn code_point_at(s : StringView, index : Int32, length : Int32) -> Int32 {
// SAFETY: callers guarantee 0 <= index < length.
let first = s.unsafe_get(index)
if first.is_leading_surrogate() && index + 1 < length {
// SAFETY: index + 1 < length is checked above.
let second = s.unsafe_get(index + 1)
if second.is_trailing_surrogate() {
return 0x10000 +
((first.to_int() - 0xD800) << 10) +
second.to_int() -
0xDC00
}
}
first.to_int()
}
///|
/// The number of UTF-16 code units of `cp`.
fn utf16_width(cp : Int32) -> Int32 {
if cp >= 0x10000 {
2
} else {
1
}
}
///|
/// Index of `key` in the sorted array `cps`, or -1 if it is absent.
fn find_special(cps : ReadOnlyArray[Int32], key : Int32) -> Int32 {
let mut left = 0
let mut right = cps.length() - 1
while left <= right {
let mid = (left + right) / 2
let value = cps[mid]
if key < value {
right = mid - 1
} else if key > value {
left = mid + 1
} else {
return mid
}
}
-1
}