tools/ucd/parse.mbt1///|2fn validate_property_ranges(3 name : StringView,4 ranges : Array[@parsing.CodePointRange],5) -> Unit raise {6 if ranges.length() == 0 {7 fail("Missing \{name} ranges")8 }9 let mut previous_end = -110 for range in ranges {11 if range.start < 0 || range.start > range.end || range.end > 0x10FFFF {12 fail("Invalid \{name} range")13 }14 if range.start <= previous_end {15 fail("\{name} ranges must be sorted and non-overlapping")16 }17 previous_end = range.end18 }19}2021///|22fn validate_xid_subset(23 xid_start : Array[@parsing.CodePointRange],24 xid_continue : Array[@parsing.CodePointRange],25) -> Unit raise {26 let mut continue_index = 027 for start_range in xid_start {28 let mut next = start_range.start29 while next <= start_range.end {30 while continue_index < xid_continue.length() &&31 xid_continue[continue_index].end < next {32 continue_index += 133 }34 if continue_index >= xid_continue.length() ||35 xid_continue[continue_index].start > next {36 fail("XID_Start must be a subset of XID_Continue")37 }38 next = xid_continue[continue_index].end + 139 }40 }41}4243///|44fn is_identity_or_empty(cp : Int32, targets : Array[Int32]) -> Bool {45 targets.length() == 0 || (targets.length() == 1 && targets[0] == cp)46}4748///|49/// General_Category ordinal of Cn (Unassigned), the value of every code point50/// not listed in UnicodeData.txt.51const GcUnassigned : Int32 = 295253///|54/// Ordinal of a General_Category value in the `GeneralCategory` enum.55fn gc_ordinal(category : StringView) -> Int32 {56 match category {57 "Lu" => 058 "Ll" => 159 "Lt" => 260 "Lm" => 361 "Lo" => 462 "Mn" => 563 "Mc" => 664 "Me" => 765 "Nd" => 866 "Nl" => 967 "No" => 1068 "Zs" => 1169 "Zl" => 1270 "Zp" => 1371 "Pc" => 1472 "Pd" => 1573 "Ps" => 1674 "Pe" => 1775 "Pi" => 1876 "Pf" => 1977 "Po" => 2078 "Sm" => 2179 "Sc" => 2280 "Sk" => 2381 "So" => 2482 "Cc" => 2583 "Cf" => 2684 "Cs" => 2785 "Co" => 2886 _ => GcUnassigned87 }88}8990///|91/// Parse the code point sequence of a simple case mapping field, keeping it92/// only when it maps the code point elsewhere.93fn push_simple_mapping(94 mappings : Array[SimpleMappingEntry],95 code_point : Int32,96 field : StringView,97) -> Unit raise {98 if field.is_empty() {99 return100 }101 let target = @parsing.parse_hex_int(field)102 if target != code_point {103 mappings.push({ code_point, target, })104 }105}106107///|108pub fn parse_unicode_data(content : String) -> UnicodeData raise {109 let general_categories = FixedArray::make(110 @emitter.MaxCodePoint + 1,111 GcUnassigned,112 )113 let ccc = FixedArray::make(@emitter.MaxCodePoint + 1, 0)114 let decompositions : Array[Decomposition] = []115 let simple_uppercase : Array[SimpleMappingEntry] = []116 let simple_lowercase : Array[SimpleMappingEntry] = []117 let simple_titlecase : Array[SimpleMappingEntry] = []118 // Start and category of an open `<..., First>` range.119 let mut range_start : (Int32, StringView)? = None120 for fields in @parsing.data_lines(content) {121 if fields.length() < 15 {122 continue123 }124 let cp = @parsing.parse_hex_int(fields[0])125 let name = fields[1]126 let category = fields[2]127 if name.has_suffix(", First>") {128 if range_start is Some(_) {129 fail("Nested UnicodeData First range")130 }131 range_start = Some((cp, category))132 continue133 }134 if name.has_suffix(", Last>") {135 guard range_start is Some((start, start_category)) &&136 start_category == category else {137 fail("Mismatched UnicodeData First/Last range")138 }139 for range_cp in start..<=cp {140 general_categories[range_cp] = gc_ordinal(category)141 }142 range_start = None143 continue144 }145 general_categories[cp] = gc_ordinal(category)146 if !fields[3].is_empty() {147 let value = @parsing.parse_decimal_int(fields[3])148 guard value >= 0 && value <= 255 else {149 fail("Invalid Canonical_Combining_Class")150 }151 ccc[cp] = value152 }153 // A compatibility mapping starts with a `<tag>`.154 let (mapping, compatibility) = match fields[5].split_once(">") {155 Some((_, rest)) => (rest, true)156 None => (fields[5], false)157 }158 let code_points = @parsing.parse_code_point_sequence(mapping)159 if !code_points.is_empty() {160 decompositions.push({ code_point: cp, code_points, compatibility, })161 }162 push_simple_mapping(simple_uppercase, cp, fields[12])163 push_simple_mapping(simple_lowercase, cp, fields[13])164 push_simple_mapping(simple_titlecase, cp, fields[14])165 }166 if range_start is Some(_) {167 fail("Unterminated UnicodeData First range")168 }169 decompositions.sort_by((left, right) => left.code_point - right.code_point)170 {171 general_categories,172 ccc,173 decompositions,174 simple_uppercase,175 simple_lowercase,176 simple_titlecase,177 }178}179180///|181pub fn parse_composition_exclusions(content : String) -> Array[Int32] raise {182 let exclusions : Array[Int32] = []183 for fields in @parsing.data_lines(content) {184 let range = @parsing.parse_code_point_range(fields[0])185 for cp in range.start..<=range.end {186 exclusions.push(cp)187 }188 }189 exclusions190}191192///|193pub fn parse_special_casing(content : String) -> SpecialCasingData raise {194 let lower_mapping : Array[FullMappingEntry] = []195 let title_mapping : Array[FullMappingEntry] = []196 let upper_mapping : Array[FullMappingEntry] = []197 for fields in @parsing.data_lines(content) {198 // A fifth field holds the conditions of a conditional mapping.199 guard fields is [cp, lower, title, upper, .. rest] &&200 !(rest is [condition, ..] && !condition.is_empty()) else {201 continue202 }203 let cp = @parsing.parse_hex_int(cp)204 let lower = @parsing.parse_code_point_sequence(lower)205 let title = @parsing.parse_code_point_sequence(title)206 let upper = @parsing.parse_code_point_sequence(upper)207 if !is_identity_or_empty(cp, lower) {208 lower_mapping.push({ code_point: cp, targets: lower, })209 }210 if !is_identity_or_empty(cp, title) {211 title_mapping.push({ code_point: cp, targets: title, })212 }213 if !is_identity_or_empty(cp, upper) {214 upper_mapping.push({ code_point: cp, targets: upper, })215 }216 }217 { lower_mapping, title_mapping, upper_mapping, }218}219220///|221/// Collect the ranges listed for one binary property in a UCD property file222/// such as `PropList.txt` or `DerivedCoreProperties.txt`.223fn parse_property_ranges(224 content : String,225 property : String,226) -> Array[@parsing.CodePointRange] raise {227 let ranges : Array[@parsing.CodePointRange] = []228 for fields in @parsing.data_lines(content) {229 if fields is [range, value, ..] && value == property {230 ranges.push(@parsing.parse_code_point_range(range))231 }232 }233 validate_property_ranges(property, ranges)234 ranges235}236237///|238/// Collect the characters `emoji-variation-sequences.txt` lists a variation239/// sequence for, as sorted ranges.240///241/// Every data line is `base selector ; style`, where the selector is U+FE0E242/// for `text style` and U+FE0F for `emoji style`. Each base must have both243/// sequences, so that one set of characters describes the file.244fn parse_emoji_variation_sequences(245 content : String,246) -> Array[@parsing.CodePointRange] raise {247 let text_style : Array[Int32] = []248 let emoji_style : Array[Int32] = []249 for fields in @parsing.data_lines(content) {250 guard fields is [sequence, style, ..] else {251 fail("Unexpected emoji variation sequence: \{fields[0]}")252 }253 match (@parsing.parse_code_point_sequence(sequence), style) {254 ([base, 0xFE0E], "text style") => text_style.push(base)255 ([base, 0xFE0F], "emoji style") => emoji_style.push(base)256 _ => fail("Unexpected emoji variation sequence: \{sequence}")257 }258 }259 text_style.sort()260 emoji_style.sort()261 guard text_style == emoji_style else {262 fail("Text and emoji variation sequences have different base characters")263 }264 let ranges : Array[@parsing.CodePointRange] = []265 for base in text_style {266 if ranges.last() is Some(range) && range.end + 1 == base {267 ranges[ranges.length() - 1] = { start: range.start, end: base, }268 } else {269 ranges.push({ start: base, end: base, })270 }271 }272 validate_property_ranges("emoji variation sequence", ranges)273 ranges274}275
///|
fn validate_property_ranges(
name : StringView,
ranges : Array[@parsing.CodePointRange],
) -> Unit raise {
if ranges.length() == 0 {
fail("Missing \{name} ranges")
}
let mut previous_end = -1
for range in ranges {
if range.start < 0 || range.start > range.end || range.end > 0x10FFFF {
fail("Invalid \{name} range")
}
if range.start <= previous_end {
fail("\{name} ranges must be sorted and non-overlapping")
}
previous_end = range.end
}
}
///|
fn validate_xid_subset(
xid_start : Array[@parsing.CodePointRange],
xid_continue : Array[@parsing.CodePointRange],
) -> Unit raise {
let mut continue_index = 0
for start_range in xid_start {
let mut next = start_range.start
while next <= start_range.end {
while continue_index < xid_continue.length() &&
xid_continue[continue_index].end < next {
continue_index += 1
}
if continue_index >= xid_continue.length() ||
xid_continue[continue_index].start > next {
fail("XID_Start must be a subset of XID_Continue")
}
next = xid_continue[continue_index].end + 1
}
}
}
///|
fn is_identity_or_empty(cp : Int32, targets : Array[Int32]) -> Bool {
targets.length() == 0 || (targets.length() == 1 && targets[0] == cp)
}
///|
/// General_Category ordinal of Cn (Unassigned), the value of every code point
/// not listed in UnicodeData.txt.
const GcUnassigned : Int32 = 29
///|
/// Ordinal of a General_Category value in the `GeneralCategory` enum.
fn gc_ordinal(category : StringView) -> Int32 {
match category {
"Lu" => 0
"Ll" => 1
"Lt" => 2
"Lm" => 3
"Lo" => 4
"Mn" => 5
"Mc" => 6
"Me" => 7
"Nd" => 8
"Nl" => 9
"No" => 10
"Zs" => 11
"Zl" => 12
"Zp" => 13
"Pc" => 14
"Pd" => 15
"Ps" => 16
"Pe" => 17
"Pi" => 18
"Pf" => 19
"Po" => 20
"Sm" => 21
"Sc" => 22
"Sk" => 23
"So" => 24
"Cc" => 25
"Cf" => 26
"Cs" => 27
"Co" => 28
_ => GcUnassigned
}
}
///|
/// Parse the code point sequence of a simple case mapping field, keeping it
/// only when it maps the code point elsewhere.
fn push_simple_mapping(
mappings : Array[SimpleMappingEntry],
code_point : Int32,
field : StringView,
) -> Unit raise {
if field.is_empty() {
return
}
let target = @parsing.parse_hex_int(field)
if target != code_point {
mappings.push({ code_point, target, })
}
}
///|
pub fn parse_unicode_data(content : String) -> UnicodeData raise {
let general_categories = FixedArray::make(
@emitter.MaxCodePoint + 1,
GcUnassigned,
)
let ccc = FixedArray::make(@emitter.MaxCodePoint + 1, 0)
let decompositions : Array[Decomposition] = []
let simple_uppercase : Array[SimpleMappingEntry] = []
let simple_lowercase : Array[SimpleMappingEntry] = []
let simple_titlecase : Array[SimpleMappingEntry] = []
// Start and category of an open `<..., First>` range.
let mut range_start : (Int32, StringView)? = None
for fields in @parsing.data_lines(content) {
if fields.length() < 15 {
continue
}
let cp = @parsing.parse_hex_int(fields[0])
let name = fields[1]
let category = fields[2]
if name.has_suffix(", First>") {
if range_start is Some(_) {
fail("Nested UnicodeData First range")
}
range_start = Some((cp, category))
continue
}
if name.has_suffix(", Last>") {
guard range_start is Some((start, start_category)) &&
start_category == category else {
fail("Mismatched UnicodeData First/Last range")
}
for range_cp in start..<=cp {
general_categories[range_cp] = gc_ordinal(category)
}
range_start = None
continue
}
general_categories[cp] = gc_ordinal(category)
if !fields[3].is_empty() {
let value = @parsing.parse_decimal_int(fields[3])
guard value >= 0 && value <= 255 else {
fail("Invalid Canonical_Combining_Class")
}
ccc[cp] = value
}
// A compatibility mapping starts with a `<tag>`.
let (mapping, compatibility) = match fields[5].split_once(">") {
Some((_, rest)) => (rest, true)
None => (fields[5], false)
}
let code_points = @parsing.parse_code_point_sequence(mapping)
if !code_points.is_empty() {
decompositions.push({ code_point: cp, code_points, compatibility, })
}
push_simple_mapping(simple_uppercase, cp, fields[12])
push_simple_mapping(simple_lowercase, cp, fields[13])
push_simple_mapping(simple_titlecase, cp, fields[14])
}
if range_start is Some(_) {
fail("Unterminated UnicodeData First range")
}
decompositions.sort_by((left, right) => left.code_point - right.code_point)
{
general_categories,
ccc,
decompositions,
simple_uppercase,
simple_lowercase,
simple_titlecase,
}
}
///|
pub fn parse_composition_exclusions(content : String) -> Array[Int32] raise {
let exclusions : Array[Int32] = []
for fields in @parsing.data_lines(content) {
let range = @parsing.parse_code_point_range(fields[0])
for cp in range.start..<=range.end {
exclusions.push(cp)
}
}
exclusions
}
///|
pub fn parse_special_casing(content : String) -> SpecialCasingData raise {
let lower_mapping : Array[FullMappingEntry] = []
let title_mapping : Array[FullMappingEntry] = []
let upper_mapping : Array[FullMappingEntry] = []
for fields in @parsing.data_lines(content) {
// A fifth field holds the conditions of a conditional mapping.
guard fields is [cp, lower, title, upper, .. rest] &&
!(rest is [condition, ..] && !condition.is_empty()) else {
continue
}
let cp = @parsing.parse_hex_int(cp)
let lower = @parsing.parse_code_point_sequence(lower)
let title = @parsing.parse_code_point_sequence(title)
let upper = @parsing.parse_code_point_sequence(upper)
if !is_identity_or_empty(cp, lower) {
lower_mapping.push({ code_point: cp, targets: lower, })
}
if !is_identity_or_empty(cp, title) {
title_mapping.push({ code_point: cp, targets: title, })
}
if !is_identity_or_empty(cp, upper) {
upper_mapping.push({ code_point: cp, targets: upper, })
}
}
{ lower_mapping, title_mapping, upper_mapping, }
}
///|
/// Collect the ranges listed for one binary property in a UCD property file
/// such as `PropList.txt` or `DerivedCoreProperties.txt`.
fn parse_property_ranges(
content : String,
property : String,
) -> Array[@parsing.CodePointRange] raise {
let ranges : Array[@parsing.CodePointRange] = []
for fields in @parsing.data_lines(content) {
if fields is [range, value, ..] && value == property {
ranges.push(@parsing.parse_code_point_range(range))
}
}
validate_property_ranges(property, ranges)
ranges
}
///|
/// Collect the characters `emoji-variation-sequences.txt` lists a variation
/// sequence for, as sorted ranges.
///
/// Every data line is `base selector ; style`, where the selector is U+FE0E
/// for `text style` and U+FE0F for `emoji style`. Each base must have both
/// sequences, so that one set of characters describes the file.
fn parse_emoji_variation_sequences(
content : String,
) -> Array[@parsing.CodePointRange] raise {
let text_style : Array[Int32] = []
let emoji_style : Array[Int32] = []
for fields in @parsing.data_lines(content) {
guard fields is [sequence, style, ..] else {
fail("Unexpected emoji variation sequence: \{fields[0]}")
}
match (@parsing.parse_code_point_sequence(sequence), style) {
([base, 0xFE0E], "text style") => text_style.push(base)
([base, 0xFE0F], "emoji style") => emoji_style.push(base)
_ => fail("Unexpected emoji variation sequence: \{sequence}")
}
}
text_style.sort()
emoji_style.sort()
guard text_style == emoji_style else {
fail("Text and emoji variation sequences have different base characters")
}
let ranges : Array[@parsing.CodePointRange] = []
for base in text_style {
if ranges.last() is Some(range) && range.end + 1 == base {
ranges[ranges.length() - 1] = { start: range.start, end: base, }
} else {
ranges.push({ start: base, end: base, })
}
}
validate_property_ranges("emoji variation sequence", ranges)
ranges
}