tools/ucd/tables.mbt1///|2/// Collect the maximal runs of code points whose value satisfies `keep`.3fn ranges_where(4 values : FixedArray[Int32],5 keep : (Int32) -> Bool,6) -> Array[@parsing.CodePointRange] {7 let ranges : Array[@parsing.CodePointRange] = []8 for cp, value in values {9 if !keep(value) {10 continue11 }12 if ranges.last() is Some(range) && range.end + 1 == cp {13 ranges[ranges.length() - 1] = { start: range.start, end: cp, }14 } else {15 ranges.push({ start: cp, end: cp, })16 }17 }18 ranges19}2021///|22fn build_composition_table(23 data : UnicodeData,24 exclusions : Array[Int32],25) -> Array[CompositionEntry] {26 let compositions : Array[CompositionEntry] = []27 for decomp in data.decompositions {28 if !decomp.compatibility &&29 decomp.code_points.length() == 2 &&30 !exclusions.contains(decomp.code_point) {31 compositions.push({32 first: decomp.code_points[0],33 second: decomp.code_points[1],34 result: decomp.code_point,35 })36 }37 }38 compositions.sort_by((left, right) => {39 let first_order = left.first - right.first40 if first_order != 0 {41 first_order42 } else {43 left.second - right.second44 }45 })46 compositions47}4849///|50fn build_full_composition_exclusions(51 data : UnicodeData,52 explicit_exclusions : Array[Int32],53) -> Array[Int32] {54 let exclusions = explicit_exclusions.copy()55 for decomp in data.decompositions {56 if decomp.compatibility {57 continue58 }59 // Singletons and decompositions starting with a non-starter never60 // recompose.61 let excluded = match decomp.code_points {62 [_] => true63 [first, ..] => data.ccc[first] > 064 [] => false65 }66 if excluded && !exclusions.contains(decomp.code_point) {67 exclusions.push(decomp.code_point)68 }69 }70 exclusions.sort()71 exclusions72}7374///|75/// Shared group order for norm16 records and the composition candidate table.76fn composition_group_offsets(77 compositions : Array[CompositionEntry],78) -> Array[Int32] {79 let offsets : Array[Int32] = []80 let mut previous = -181 for index, entry in compositions {82 if entry.first != previous {83 offsets.push(index)84 previous = entry.first85 }86 }87 offsets.push(compositions.length())88 offsets89}90
///|
/// Collect the maximal runs of code points whose value satisfies `keep`.
fn ranges_where(
values : FixedArray[Int32],
keep : (Int32) -> Bool,
) -> Array[@parsing.CodePointRange] {
let ranges : Array[@parsing.CodePointRange] = []
for cp, value in values {
if !keep(value) {
continue
}
if ranges.last() is Some(range) && range.end + 1 == cp {
ranges[ranges.length() - 1] = { start: range.start, end: cp, }
} else {
ranges.push({ start: cp, end: cp, })
}
}
ranges
}
///|
fn build_composition_table(
data : UnicodeData,
exclusions : Array[Int32],
) -> Array[CompositionEntry] {
let compositions : Array[CompositionEntry] = []
for decomp in data.decompositions {
if !decomp.compatibility &&
decomp.code_points.length() == 2 &&
!exclusions.contains(decomp.code_point) {
compositions.push({
first: decomp.code_points[0],
second: decomp.code_points[1],
result: decomp.code_point,
})
}
}
compositions.sort_by((left, right) => {
let first_order = left.first - right.first
if first_order != 0 {
first_order
} else {
left.second - right.second
}
})
compositions
}
///|
fn build_full_composition_exclusions(
data : UnicodeData,
explicit_exclusions : Array[Int32],
) -> Array[Int32] {
let exclusions = explicit_exclusions.copy()
for decomp in data.decompositions {
if decomp.compatibility {
continue
}
// Singletons and decompositions starting with a non-starter never
// recompose.
let excluded = match decomp.code_points {
[_] => true
[first, ..] => data.ccc[first] > 0
[] => false
}
if excluded && !exclusions.contains(decomp.code_point) {
exclusions.push(decomp.code_point)
}
}
exclusions.sort()
exclusions
}
///|
/// Shared group order for norm16 records and the composition candidate table.
fn composition_group_offsets(
compositions : Array[CompositionEntry],
) -> Array[Int32] {
let offsets : Array[Int32] = []
let mut previous = -1
for index, entry in compositions {
if entry.first != previous {
offsets.push(index)
previous = entry.first
}
}
offsets.push(compositions.length())
offsets
}