tools/internal/parsing/parsing.mbt1///|2#derive(Debug, Eq)3pub(all) struct CodePointRange {4 start : Int325 end : Int326}78///|9pub extend CodePointRange with Debug::{to_repr}1011///|12pub extend CodePointRange with Eq::{not_equal, equal}1314///|15pub fn strip_comment(line : StringView) -> StringView {16 if line.split_once("#") is Some((before, _)) {17 before.trim()18 } else {19 line.trim()20 }21}2223///|24/// The fields of every data line of a UCD file: the text before any `#`25/// comment, split at `;`, with every field trimmed. Blank and comment-only26/// lines are skipped.27pub fn data_lines(content : StringView) -> Iter[Array[StringView]] {28 content29 .split("\n")30 .filter_map(line => {31 let line = strip_comment(line)32 if line.is_empty() {33 None34 } else {35 Some(line.split(";").map(field => field.trim()).collect())36 }37 })38}3940///|41pub fn parse_hex_int(text : StringView) -> Int32 raise {42 @string.parse_int(text.trim(), radix=16)43}4445///|46pub fn parse_decimal_int(text : StringView) -> Int32 raise {47 @string.parse_int(text.trim(), radix=10)48}4950///|51pub fn parse_code_point_range(text : StringView) -> CodePointRange raise {52 let text = text.trim()53 if text.split_once("..") is Some((start_text, end_text)) {54 { start: parse_hex_int(start_text), end: parse_hex_int(end_text), }55 } else {56 let code_point = parse_hex_int(text)57 { start: code_point, end: code_point, }58 }59}6061///|62/// Read a UCD file for one enumerated property, whose data lines are63/// `range ; value`, and return the value of every code point.64///65/// Code points the file does not list take the value of the last66/// `# @missing: range; value` line covering them, or `default` when there is67/// none. Values are returned as written, so a `@missing` line that spells out68/// a long alias such as `Neutral` yields that alias.69pub fn parse_enumerated_property(70 content : StringView,71 default~ : String,72) -> FixedArray[String] raise {73 let values = FixedArray::make(@emitter.MaxCodePoint + 1, default)74 let fill = (range_text : StringView, value : StringView) => {75 let range = parse_code_point_range(range_text)76 let value = value.trim().to_owned()77 for code_point in range.start..<=range.end {78 values[code_point] = value79 }80 }81 for raw_line in content.split("\n") {82 if raw_line.trim().strip_prefix("# @missing:") is Some(declaration) &&83 declaration.split_once(";") is Some((range, value)) {84 fill(range, value)85 }86 }87 for fields in data_lines(content) {88 if fields is [range, value, ..] {89 fill(range, value)90 }91 }92 values93}9495///|96/// Read a UCD file for one enumerated property like97/// `parse_enumerated_property`, and convert the value of every code point98/// with `ordinal_of`, which accepts each spelling the file uses.99pub fn parse_enumerated_ordinals(100 content : StringView,101 property~ : String,102 default~ : String,103 ordinal_of : (StringView) -> Int32?,104) -> FixedArray[Int32] raise {105 let names = parse_enumerated_property(content, default~)106 let ordinals = FixedArray::make(names.length(), 0)107 for code_point, name in names {108 guard ordinal_of(name) is Some(ordinal) else {109 fail("Unknown \{property}: \{name}")110 }111 ordinals[code_point] = ordinal112 }113 ordinals114}115116///|117pub fn parse_code_point_sequence(text : StringView) -> Array[Int32] raise {118 let result : Array[Int32] = []119 for part in text.trim().split(" ") {120 let part = part.trim()121 if !part.is_empty() {122 result.push(parse_hex_int(part))123 }124 }125 result126}127
///|
#derive(Debug, Eq)
pub(all) struct CodePointRange {
start : Int32
end : Int32
}
///|
pub extend CodePointRange with Debug::{to_repr}
///|
pub extend CodePointRange with Eq::{not_equal, equal}
///|
pub fn strip_comment(line : StringView) -> StringView {
if line.split_once("#") is Some((before, _)) {
before.trim()
} else {
line.trim()
}
}
///|
/// The fields of every data line of a UCD file: the text before any `#`
/// comment, split at `;`, with every field trimmed. Blank and comment-only
/// lines are skipped.
pub fn data_lines(content : StringView) -> Iter[Array[StringView]] {
content
.split("\n")
.filter_map(line => {
let line = strip_comment(line)
if line.is_empty() {
None
} else {
Some(line.split(";").map(field => field.trim()).collect())
}
})
}
///|
pub fn parse_hex_int(text : StringView) -> Int32 raise {
@string.parse_int(text.trim(), radix=16)
}
///|
pub fn parse_decimal_int(text : StringView) -> Int32 raise {
@string.parse_int(text.trim(), radix=10)
}
///|
pub fn parse_code_point_range(text : StringView) -> CodePointRange raise {
let text = text.trim()
if text.split_once("..") is Some((start_text, end_text)) {
{ start: parse_hex_int(start_text), end: parse_hex_int(end_text), }
} else {
let code_point = parse_hex_int(text)
{ start: code_point, end: code_point, }
}
}
///|
/// Read a UCD file for one enumerated property, whose data lines are
/// `range ; value`, and return the value of every code point.
///
/// Code points the file does not list take the value of the last
/// `# @missing: range; value` line covering them, or `default` when there is
/// none. Values are returned as written, so a `@missing` line that spells out
/// a long alias such as `Neutral` yields that alias.
pub fn parse_enumerated_property(
content : StringView,
default~ : String,
) -> FixedArray[String] raise {
let values = FixedArray::make(@emitter.MaxCodePoint + 1, default)
let fill = (range_text : StringView, value : StringView) => {
let range = parse_code_point_range(range_text)
let value = value.trim().to_owned()
for code_point in range.start..<=range.end {
values[code_point] = value
}
}
for raw_line in content.split("\n") {
if raw_line.trim().strip_prefix("# @missing:") is Some(declaration) &&
declaration.split_once(";") is Some((range, value)) {
fill(range, value)
}
}
for fields in data_lines(content) {
if fields is [range, value, ..] {
fill(range, value)
}
}
values
}
///|
/// Read a UCD file for one enumerated property like
/// `parse_enumerated_property`, and convert the value of every code point
/// with `ordinal_of`, which accepts each spelling the file uses.
pub fn parse_enumerated_ordinals(
content : StringView,
property~ : String,
default~ : String,
ordinal_of : (StringView) -> Int32?,
) -> FixedArray[Int32] raise {
let names = parse_enumerated_property(content, default~)
let ordinals = FixedArray::make(names.length(), 0)
for code_point, name in names {
guard ordinal_of(name) is Some(ordinal) else {
fail("Unknown \{property}: \{name}")
}
ordinals[code_point] = ordinal
}
ordinals
}
///|
pub fn parse_code_point_sequence(text : StringView) -> Array[Int32] raise {
let result : Array[Int32] = []
for part in text.trim().split(" ") {
let part = part.trim()
if !part.is_empty() {
result.push(parse_hex_int(part))
}
}
result
}