tools/ucd/emit.mbt1///|2pub fn emit_ccc() -> String {3 let out = @emitter.Emitter::new()4 out.write_file_header(5 "Canonical Combining Class (CCC) lookup", "UnicodeData.txt",6 )7 out.write(8 (9 #|///|10 #|pub using @normalization_data {lookup_normalization_properties}11 #|12 #|///|13 #|/// Look up Canonical Combining Class; 0 denotes a starter.14 #|pub fn lookup_ccc(c : Char) -> Int {15 #| @normalization_data.lookup_normalization_properties(c) & 25516 #|}17 #|18 ),19 )20 out.finish()21}2223///|24/// Bits of a packed decomposition entry: the code point occupies the low25/// 21 bits and its Canonical_Combining_Class the eight above them.26const DecompositionCccShift : Int32 = 212728///|29const DecompositionCodePointMask : Int32 = 0x1FFFFF3031///|32/// A packed decomposition entry: `code_point` and its combining class.33fn packed_entry(data : UnicodeData, code_point : Int32) -> Int32 {34 code_point | (data.ccc[code_point] << DecompositionCccShift)35}3637///|38/// Append the full decomposition of `code_point` to `dest` as packed39/// entries: its mapping, applied recursively, where a compatibility mapping40/// applies only when `compatibility` is set.41fn append_full_decomposition(42 data : UnicodeData,43 mappings : Map[Int32, Decomposition],44 code_point : Int32,45 compatibility : Bool,46 dest : Array[Int32],47) -> Unit raise {48 // Hangul syllables decompose algorithmically. The runtime does not look49 // them up, so a mapping that produced one would leave it composed.50 guard code_point < 0xAC00 || code_point > 0xD7A3 else {51 fail("A decomposition mapping produces a Hangul syllable")52 }53 if mappings.get(code_point) is Some(entry) &&54 (compatibility || !entry.compatibility) {55 for target in entry.code_points {56 append_full_decomposition(data, mappings, target, compatibility, dest)57 }58 } else {59 dest.push(packed_entry(data, code_point))60 }61}6263///|64pub fn emit_decomposition(data : UnicodeData) -> String raise {65 let mappings : Map[Int32, Decomposition] = Map([])66 for entry in data.decompositions {67 mappings[entry.code_point] = entry68 }69 // Each row's entries are its full canonical decomposition (canonical70 // mappings only), its full compatibility decomposition (compatibility71 // mappings, or canonical ones whose full compatibility decomposition72 // differs), and its own mapping, when that differs from the full73 // decomposition of its kind.74 let starts : Array[Int32] = []75 let canonical_lens : Array[Int32] = []76 let compat_lens : Array[Int32] = []77 let mapping_lens : Array[Int32] = []78 let entries : Array[Int32] = []79 for entry in data.decompositions {80 starts.push(entries.length())81 let compat : Array[Int32] = []82 append_full_decomposition(data, mappings, entry.code_point, true, compat)83 let full = if entry.compatibility {84 canonical_lens.push(0)85 compat_lens.push(compat.length())86 entries.append(compat)87 compat88 } else {89 let canonical : Array[Int32] = []90 append_full_decomposition(91 data,92 mappings,93 entry.code_point,94 false,95 canonical,96 )97 canonical_lens.push(canonical.length())98 entries.append(canonical)99 if compat == canonical {100 compat_lens.push(0)101 } else {102 compat_lens.push(compat.length())103 entries.append(compat)104 }105 canonical106 }107 let mapping = entry.code_points.map(code_point => {108 packed_entry(data, code_point)109 })110 if mapping == full {111 mapping_lens.push(0)112 } else {113 mapping_lens.push(mapping.length())114 entries.append(mapping)115 }116 }117 guard canonical_lens.all(len => len <= 255) &&118 compat_lens.all(len => len <= 255) &&119 mapping_lens.all(len => len <= 255) else {120 fail("Decomposition length exceeds its encoded byte width")121 }122 let out = @emitter.Emitter::new()123 out.write_file_header("Decomposition mapping data", "UnicodeData.txt")124 out.write_int_array(125 "Start of each decomposition row's entries",126 "decomp_data_starts",127 starts,128 hex=false,129 )130 out.write_byte_array(131 "Length of the full canonical decomposition (0 for a compatibility mapping)",132 "decomp_canonical_lens", canonical_lens,133 )134 out.write_byte_array(135 "Length of the full compatibility decomposition (0 if it is the canonical one)",136 "decomp_compat_lens", compat_lens,137 )138 out.write_byte_array(139 "Length of the mapping itself (0 if it is the full decomposition)", "decomp_mapping_lens",140 mapping_lens,141 )142 out.write_int_array(143 "Packed entries: code point in bits 0..20, its CCC in bits 21..28",144 "decomp_data",145 entries,146 hex=true,147 )148 // Lets the lookup skip everything below the first character with a mapping.149 let first_decomposable = data.decompositions.fold(150 init=@emitter.MaxCodePoint + 1,151 (first, entry) => Int32::min(first, entry.code_point),152 )153 out.write(154 (155 $|///|156 $|/// The mapping of row `idx` itself, which is canonical when the row has157 $|/// a full canonical decomposition.158 $|fn mapping_entries(idx : Int) -> ArrayView[Int] {159 $| let start = decomp_data_starts[idx]160 $| let canonical_len = decomp_canonical_lens[idx].to_int()161 $| let full_len = if canonical_len != 0 {162 $| canonical_len163 $| } else {164 $| decomp_compat_lens[idx].to_int()165 $| }166 $| let mapping_len = decomp_mapping_lens[idx].to_int()167 $| if mapping_len == 0 {168 $| return decomp_data.exact_view(start~, end=start + full_len)169 $| }170 $| let mapping_start = start +171 $| canonical_len +172 $| decomp_compat_lens[idx].to_int()173 $| decomp_data.exact_view(174 $| start=mapping_start,175 $| end=mapping_start + mapping_len,176 $| )177 $|}178 $|179 $|///|180 $|fn unpack_code_points(entries : ArrayView[Int]) -> Array[Char] {181 $| entries.map(entry => (entry & \{@emitter.hex_literal(DecompositionCodePointMask)}).unsafe_to_char())182 $|}183 $|184 $|///|185 $|/// Get canonical decomposition for a character186 $|/// Returns None if no decomposition exists187 $|pub fn get_canonical_decomposition(c : Char) -> Array[Char]? {188 $| let idx = decomposition_row(@normalization_data.lookup_normalization_record(c))189 $| if idx < 0 || decomp_canonical_lens[idx] == 0 {190 $| return None191 $| }192 $| Some(unpack_code_points(mapping_entries(idx)))193 $|}194 $|195 $|///|196 $|/// Get compatibility decomposition for a character197 $|/// Returns None if no decomposition exists198 $|pub fn get_compat_decomposition(c : Char) -> Array[Char]? {199 $| let idx = decomposition_row(@normalization_data.lookup_normalization_record(c))200 $| if idx < 0 || decomp_canonical_lens[idx] != 0 {201 $| return None202 $| }203 $| Some(unpack_code_points(mapping_entries(idx)))204 $|}205 $|206 ),207 )208 out.write(209 (210 $|///|211 $|/// Public only because unicode-normalization, a separate module, needs212 $|/// it. `#alert` does not apply to a re-export, so unlike213 $|/// `packed_decomposition` it does not warn.214 $|#doc(hidden)215 $|pub using @normalization_data {lookup_normalization_record}216 $|217 $|///|218 $|/// The decomposition row of a character with normalization record219 $|/// `record`, or -1 without one.220 $|fn decomposition_row(record : Int) -> Int {221 $| (record >> 16) - 1222 $|}223 $|224 $|///|225 $|/// The full decomposition of a character: its canonical mapping, or its226 $|/// compatibility mapping when asked for and present, applied227 $|/// recursively. `record` is the character's `lookup_normalization_record`,228 $|/// which tells where its decomposition is. Each entry packs a code point229 $|/// in bits 0..20 and its Canonical_Combining_Class in bits 21..28. The230 $|/// view is empty when the character has no such mapping.231 $|///232 $|/// Hangul syllables decompose algorithmically and have no mapping233 $|/// here.234 $|///235 $|/// Public only because unicode-normalization, a separate module, needs236 $|/// it. Other callers want `append_decomposition`.237 $|#doc(hidden)238 $|#alert(internal, "For unicode-normalization; the entry layout may change.")239 $|pub fn packed_decomposition(240 $| record : Int,241 $| compatibility~ : Bool,242 $|) -> ArrayView[Int] {243 $| let idx = decomposition_row(record)244 $| if idx < 0 {245 $| return decomp_data.exact_view(start=0, end=0)246 $| }247 $| let start = decomp_data_starts[idx]248 $| let canonical_len = decomp_canonical_lens[idx].to_int()249 $| let compat_len = decomp_compat_lens[idx].to_int()250 $| if compatibility && compat_len != 0 {251 $| decomp_data.exact_view(252 $| start=start + canonical_len,253 $| end=start + canonical_len + compat_len,254 $| )255 $| } else {256 $| // Empty for a compatibility mapping, which has no canonical part.257 $| decomp_data.exact_view(start~, end=start + canonical_len)258 $| }259 $|}260 $|261 $|///|262 $|/// Append the full decomposition of a character: its canonical mapping,263 $|/// or its compatibility mapping when asked for and present, applied264 $|/// recursively. A character without a mapping is appended as it is.265 $|///266 $|/// Hangul syllables decompose algorithmically and have no mapping267 $|/// here, so they are appended unchanged.268 $|pub fn append_decomposition(269 $| c : Char,270 $| dest : Array[Char],271 $| compatibility~ : Bool,272 $|) -> Unit {273 $| // No character below this one has a decomposition.274 $| if c.to_int() < \{@emitter.hex_literal(first_decomposable)} {275 $| dest.push(c)276 $| return277 $| }278 $| let entries = packed_decomposition(279 $| lookup_normalization_record(c),280 $| compatibility~,281 $| )282 $| if entries.length() == 0 {283 $| dest.push(c)284 $| return285 $| }286 $| for entry in entries {287 $| dest.push((entry & \{@emitter.hex_literal(DecompositionCodePointMask)}).unsafe_to_char())288 $| }289 $|}290 $|291 ),292 )293 out.finish()294}295296///|297/// Emit the canonical composition pairs and the full composition exclusions.298/// `exclusions` are the code points `CompositionExclusions.txt` lists.299pub fn emit_composition(300 data : UnicodeData,301 exclusions : Array[Int32],302) -> String {303 let compositions = build_composition_table(data, exclusions)304 let exclusions = build_full_composition_exclusions(data, exclusions)305 let out = @emitter.Emitter::new()306 out.write_file_header(307 "Composition lookup data", "UnicodeData.txt and CompositionExclusions.txt",308 )309 out.write_int_array(310 "Composition candidate group offsets, with an end sentinel",311 "comp_group_offsets",312 composition_group_offsets(compositions),313 hex=false,314 )315 out.write_int_array(316 "Second code points of composition pairs",317 "comp_seconds",318 compositions.map(entry => entry.second),319 hex=true,320 )321 out.write_int_array(322 "Composed result code points",323 "comp_results",324 compositions.map(entry => entry.result),325 hex=true,326 )327 out.write_int_array(328 "Composition exclusions (sorted for binary search)",329 "comp_exclusions",330 exclusions,331 hex=true,332 )333 out.write(334 (335 #|///|336 #|/// Look up composition for a pair of characters337 #|/// Returns Some(composed) if composition exists, None otherwise338 #|pub fn lookup_composition(starter : Char, combining : Char) -> Char? {339 #| let group = @normalization_data.normalization_composition_group(starter)340 #| if group < 0 {341 #| return None342 #| }343 #| let combining_cp = combining.to_int()344 #| // Search only this starter's candidates, not the global pair table.345 #| let mut left = comp_group_offsets[group]346 #| let mut right = comp_group_offsets[group + 1] - 1347 #|348 #| while left <= right {349 #| let mid = (left + right) / 2350 #| let mid_second = comp_seconds[mid]351 #|352 #| if combining_cp < mid_second {353 #| right = mid - 1354 #| } else if combining_cp > mid_second {355 #| left = mid + 1356 #| } else {357 #| return Some(comp_results[mid].unsafe_to_char())358 #| }359 #| }360 #|361 #| None362 #|}363 #|364 #|///|365 #|/// Check if a character is excluded from composition366 #|pub fn is_composition_excluded(c : Char) -> Bool {367 #| find_composition_exclusion(c.to_int()) >= 0368 #|}369 #|370 ),371 )372 out.write_sorted_search(373 name="find_composition_exclusion",374 values="comp_exclusions",375 )376 out.finish()377}378
///|
pub fn emit_ccc() -> String {
let out = @emitter.Emitter::new()
out.write_file_header(
"Canonical Combining Class (CCC) lookup", "UnicodeData.txt",
)
out.write(
(
#|///|
#|pub using @normalization_data {lookup_normalization_properties}
#|
#|///|
#|/// Look up Canonical Combining Class; 0 denotes a starter.
#|pub fn lookup_ccc(c : Char) -> Int {
#| @normalization_data.lookup_normalization_properties(c) & 255
#|}
#|
),
)
out.finish()
}
///|
/// Bits of a packed decomposition entry: the code point occupies the low
/// 21 bits and its Canonical_Combining_Class the eight above them.
const DecompositionCccShift : Int32 = 21
///|
const DecompositionCodePointMask : Int32 = 0x1FFFFF
///|
/// A packed decomposition entry: `code_point` and its combining class.
fn packed_entry(data : UnicodeData, code_point : Int32) -> Int32 {
code_point | (data.ccc[code_point] << DecompositionCccShift)
}
///|
/// Append the full decomposition of `code_point` to `dest` as packed
/// entries: its mapping, applied recursively, where a compatibility mapping
/// applies only when `compatibility` is set.
fn append_full_decomposition(
data : UnicodeData,
mappings : Map[Int32, Decomposition],
code_point : Int32,
compatibility : Bool,
dest : Array[Int32],
) -> Unit raise {
// Hangul syllables decompose algorithmically. The runtime does not look
// them up, so a mapping that produced one would leave it composed.
guard code_point < 0xAC00 || code_point > 0xD7A3 else {
fail("A decomposition mapping produces a Hangul syllable")
}
if mappings.get(code_point) is Some(entry) &&
(compatibility || !entry.compatibility) {
for target in entry.code_points {
append_full_decomposition(data, mappings, target, compatibility, dest)
}
} else {
dest.push(packed_entry(data, code_point))
}
}
///|
pub fn emit_decomposition(data : UnicodeData) -> String raise {
let mappings : Map[Int32, Decomposition] = Map([])
for entry in data.decompositions {
mappings[entry.code_point] = entry
}
// Each row's entries are its full canonical decomposition (canonical
// mappings only), its full compatibility decomposition (compatibility
// mappings, or canonical ones whose full compatibility decomposition
// differs), and its own mapping, when that differs from the full
// decomposition of its kind.
let starts : Array[Int32] = []
let canonical_lens : Array[Int32] = []
let compat_lens : Array[Int32] = []
let mapping_lens : Array[Int32] = []
let entries : Array[Int32] = []
for entry in data.decompositions {
starts.push(entries.length())
let compat : Array[Int32] = []
append_full_decomposition(data, mappings, entry.code_point, true, compat)
let full = if entry.compatibility {
canonical_lens.push(0)
compat_lens.push(compat.length())
entries.append(compat)
compat
} else {
let canonical : Array[Int32] = []
append_full_decomposition(
data,
mappings,
entry.code_point,
false,
canonical,
)
canonical_lens.push(canonical.length())
entries.append(canonical)
if compat == canonical {
compat_lens.push(0)
} else {
compat_lens.push(compat.length())
entries.append(compat)
}
canonical
}
let mapping = entry.code_points.map(code_point => {
packed_entry(data, code_point)
})
if mapping == full {
mapping_lens.push(0)
} else {
mapping_lens.push(mapping.length())
entries.append(mapping)
}
}
guard canonical_lens.all(len => len <= 255) &&
compat_lens.all(len => len <= 255) &&
mapping_lens.all(len => len <= 255) else {
fail("Decomposition length exceeds its encoded byte width")
}
let out = @emitter.Emitter::new()
out.write_file_header("Decomposition mapping data", "UnicodeData.txt")
out.write_int_array(
"Start of each decomposition row's entries",
"decomp_data_starts",
starts,
hex=false,
)
out.write_byte_array(
"Length of the full canonical decomposition (0 for a compatibility mapping)",
"decomp_canonical_lens", canonical_lens,
)
out.write_byte_array(
"Length of the full compatibility decomposition (0 if it is the canonical one)",
"decomp_compat_lens", compat_lens,
)
out.write_byte_array(
"Length of the mapping itself (0 if it is the full decomposition)", "decomp_mapping_lens",
mapping_lens,
)
out.write_int_array(
"Packed entries: code point in bits 0..20, its CCC in bits 21..28",
"decomp_data",
entries,
hex=true,
)
// Lets the lookup skip everything below the first character with a mapping.
let first_decomposable = data.decompositions.fold(
init=@emitter.MaxCodePoint + 1,
(first, entry) => Int32::min(first, entry.code_point),
)
out.write(
(
$|///|
$|/// The mapping of row `idx` itself, which is canonical when the row has
$|/// a full canonical decomposition.
$|fn mapping_entries(idx : Int) -> ArrayView[Int] {
$| let start = decomp_data_starts[idx]
$| let canonical_len = decomp_canonical_lens[idx].to_int()
$| let full_len = if canonical_len != 0 {
$| canonical_len
$| } else {
$| decomp_compat_lens[idx].to_int()
$| }
$| let mapping_len = decomp_mapping_lens[idx].to_int()
$| if mapping_len == 0 {
$| return decomp_data.exact_view(start~, end=start + full_len)
$| }
$| let mapping_start = start +
$| canonical_len +
$| decomp_compat_lens[idx].to_int()
$| decomp_data.exact_view(
$| start=mapping_start,
$| end=mapping_start + mapping_len,
$| )
$|}
$|
$|///|
$|fn unpack_code_points(entries : ArrayView[Int]) -> Array[Char] {
$| entries.map(entry => (entry & \{@emitter.hex_literal(DecompositionCodePointMask)}).unsafe_to_char())
$|}
$|
$|///|
$|/// Get canonical decomposition for a character
$|/// Returns None if no decomposition exists
$|pub fn get_canonical_decomposition(c : Char) -> Array[Char]? {
$| let idx = decomposition_row(@normalization_data.lookup_normalization_record(c))
$| if idx < 0 || decomp_canonical_lens[idx] == 0 {
$| return None
$| }
$| Some(unpack_code_points(mapping_entries(idx)))
$|}
$|
$|///|
$|/// Get compatibility decomposition for a character
$|/// Returns None if no decomposition exists
$|pub fn get_compat_decomposition(c : Char) -> Array[Char]? {
$| let idx = decomposition_row(@normalization_data.lookup_normalization_record(c))
$| if idx < 0 || decomp_canonical_lens[idx] != 0 {
$| return None
$| }
$| Some(unpack_code_points(mapping_entries(idx)))
$|}
$|
),
)
out.write(
(
$|///|
$|/// Public only because unicode-normalization, a separate module, needs
$|/// it. `#alert` does not apply to a re-export, so unlike
$|/// `packed_decomposition` it does not warn.
$|#doc(hidden)
$|pub using @normalization_data {lookup_normalization_record}
$|
$|///|
$|/// The decomposition row of a character with normalization record
$|/// `record`, or -1 without one.
$|fn decomposition_row(record : Int) -> Int {
$| (record >> 16) - 1
$|}
$|
$|///|
$|/// The full decomposition of a character: its canonical mapping, or its
$|/// compatibility mapping when asked for and present, applied
$|/// recursively. `record` is the character's `lookup_normalization_record`,
$|/// which tells where its decomposition is. Each entry packs a code point
$|/// in bits 0..20 and its Canonical_Combining_Class in bits 21..28. The
$|/// view is empty when the character has no such mapping.
$|///
$|/// Hangul syllables decompose algorithmically and have no mapping
$|/// here.
$|///
$|/// Public only because unicode-normalization, a separate module, needs
$|/// it. Other callers want `append_decomposition`.
$|#doc(hidden)
$|#alert(internal, "For unicode-normalization; the entry layout may change.")
$|pub fn packed_decomposition(
$| record : Int,
$| compatibility~ : Bool,
$|) -> ArrayView[Int] {
$| let idx = decomposition_row(record)
$| if idx < 0 {
$| return decomp_data.exact_view(start=0, end=0)
$| }
$| let start = decomp_data_starts[idx]
$| let canonical_len = decomp_canonical_lens[idx].to_int()
$| let compat_len = decomp_compat_lens[idx].to_int()
$| if compatibility && compat_len != 0 {
$| decomp_data.exact_view(
$| start=start + canonical_len,
$| end=start + canonical_len + compat_len,
$| )
$| } else {
$| // Empty for a compatibility mapping, which has no canonical part.
$| decomp_data.exact_view(start~, end=start + canonical_len)
$| }
$|}
$|
$|///|
$|/// Append the full decomposition of a character: its canonical mapping,
$|/// or its compatibility mapping when asked for and present, applied
$|/// recursively. A character without a mapping is appended as it is.
$|///
$|/// Hangul syllables decompose algorithmically and have no mapping
$|/// here, so they are appended unchanged.
$|pub fn append_decomposition(
$| c : Char,
$| dest : Array[Char],
$| compatibility~ : Bool,
$|) -> Unit {
$| // No character below this one has a decomposition.
$| if c.to_int() < \{@emitter.hex_literal(first_decomposable)} {
$| dest.push(c)
$| return
$| }
$| let entries = packed_decomposition(
$| lookup_normalization_record(c),
$| compatibility~,
$| )
$| if entries.length() == 0 {
$| dest.push(c)
$| return
$| }
$| for entry in entries {
$| dest.push((entry & \{@emitter.hex_literal(DecompositionCodePointMask)}).unsafe_to_char())
$| }
$|}
$|
),
)
out.finish()
}
///|
/// Emit the canonical composition pairs and the full composition exclusions.
/// `exclusions` are the code points `CompositionExclusions.txt` lists.
pub fn emit_composition(
data : UnicodeData,
exclusions : Array[Int32],
) -> String {
let compositions = build_composition_table(data, exclusions)
let exclusions = build_full_composition_exclusions(data, exclusions)
let out = @emitter.Emitter::new()
out.write_file_header(
"Composition lookup data", "UnicodeData.txt and CompositionExclusions.txt",
)
out.write_int_array(
"Composition candidate group offsets, with an end sentinel",
"comp_group_offsets",
composition_group_offsets(compositions),
hex=false,
)
out.write_int_array(
"Second code points of composition pairs",
"comp_seconds",
compositions.map(entry => entry.second),
hex=true,
)
out.write_int_array(
"Composed result code points",
"comp_results",
compositions.map(entry => entry.result),
hex=true,
)
out.write_int_array(
"Composition exclusions (sorted for binary search)",
"comp_exclusions",
exclusions,
hex=true,
)
out.write(
(
#|///|
#|/// Look up composition for a pair of characters
#|/// Returns Some(composed) if composition exists, None otherwise
#|pub fn lookup_composition(starter : Char, combining : Char) -> Char? {
#| let group = @normalization_data.normalization_composition_group(starter)
#| if group < 0 {
#| return None
#| }
#| let combining_cp = combining.to_int()
#| // Search only this starter's candidates, not the global pair table.
#| let mut left = comp_group_offsets[group]
#| let mut right = comp_group_offsets[group + 1] - 1
#|
#| while left <= right {
#| let mid = (left + right) / 2
#| let mid_second = comp_seconds[mid]
#|
#| if combining_cp < mid_second {
#| right = mid - 1
#| } else if combining_cp > mid_second {
#| left = mid + 1
#| } else {
#| return Some(comp_results[mid].unsafe_to_char())
#| }
#| }
#|
#| None
#|}
#|
#|///|
#|/// Check if a character is excluded from composition
#|pub fn is_composition_excluded(c : Char) -> Bool {
#| find_composition_exclusion(c.to_int()) >= 0
#|}
#|
),
)
out.write_sorted_search(
name="find_composition_exclusion",
values="comp_exclusions",
)
out.finish()
}