tools/bidi_data/bidi_data.mbt1///|2priv struct MirrorEntry {3 code_point : Int324 mirrored : Int325}67///|8priv struct BracketEntry {9 code_point : Int3210 paired : Int3211 /// Index of the bracket type in `bracket_type_values`.12 bracket_type : Int3213}1415///|16/// Bidi_Paired_Bracket_Type values in the order of the `BracketType` enum:17/// the constructor, the value BidiBrackets.txt writes, and a description.18let bracket_type_values : ReadOnlyArray[(String, String, String)] = [19 ("Open", "o", "Opening paired bracket"),20 ("Close", "c", "Closing paired bracket"),21 ("None", "n", "Not a paired bracket"),22]2324///|25fn parse_mirroring(content : String) -> Array[MirrorEntry] raise {26 let entries : Array[MirrorEntry] = []27 for fields in @parsing.data_lines(content) {28 if fields is [code_point, mirrored, ..] {29 entries.push({30 code_point: @parsing.parse_hex_int(code_point),31 mirrored: @parsing.parse_hex_int(mirrored),32 })33 }34 }35 entries.sort_by((left, right) => left.code_point - right.code_point)36 entries37}3839///|40fn parse_brackets(content : String) -> Array[BracketEntry] raise {41 let entries : Array[BracketEntry] = []42 for fields in @parsing.data_lines(content) {43 if fields is [code_point, paired, bracket_type, ..] {44 let name = bracket_type.to_owned().to_lower()45 guard bracket_type_values.search_by(entry => entry.1 == name)46 is Some(bracket_type) else {47 fail("Unknown Bidi_Paired_Bracket_Type \{name}")48 }49 entries.push({50 code_point: @parsing.parse_hex_int(code_point),51 paired: @parsing.parse_hex_int(paired),52 bracket_type,53 })54 }55 }56 entries.sort_by((left, right) => left.code_point - right.code_point)57 entries58}5960///|61/// Bidi_Class values in the order of the `BidiClass` enum: the short alias,62/// which DerivedBidiClass.txt writes and which names the constructor, and the63/// long alias, which `@missing` lines spell out.64let bidi_classes : ReadOnlyArray[(String, String)] = [65 ("L", "Left_To_Right"),66 ("R", "Right_To_Left"),67 ("AL", "Arabic_Letter"),68 ("EN", "European_Number"),69 ("ES", "European_Separator"),70 ("ET", "European_Terminator"),71 ("AN", "Arabic_Number"),72 ("CS", "Common_Separator"),73 ("NSM", "Nonspacing_Mark"),74 ("BN", "Boundary_Neutral"),75 ("B", "Paragraph_Separator"),76 ("S", "Segment_Separator"),77 ("WS", "White_Space"),78 ("ON", "Other_Neutral"),79 ("LRE", "Left_To_Right_Embedding"),80 ("LRO", "Left_To_Right_Override"),81 ("RLE", "Right_To_Left_Embedding"),82 ("RLO", "Right_To_Left_Override"),83 ("PDF", "Pop_Directional_Format"),84 ("LRI", "Left_To_Right_Isolate"),85 ("RLI", "Right_To_Left_Isolate"),86 ("FSI", "First_Strong_Isolate"),87 ("PDI", "Pop_Directional_Isolate"),88]8990///|91/// Emit the `BidiClass` enum and Bidi_Class data from `DerivedBidiClass.txt`92/// as a three-level trie for planes 0-3, with packed ranges for the remaining93/// planes.94pub fn emit_bidi_class(content : String) -> String raise {95 let values = @parsing.parse_enumerated_ordinals(96 content,97 property="Bidi_Class",98 default="L",99 name => bidi_classes.search_by(entry => name == entry.0 || name == entry.1),100 )101 let out = @emitter.Emitter::new()102 out.write_file_header(103 "Full Bidi_Class lookup for Unicode Bidirectional Algorithm (UAX #9)", "DerivedBidiClass.txt",104 )105 out.write_enum(106 type_name="BidiClass",107 doc=["All \{bidi_classes.length()} Bidi_Class values from UAX #9"],108 converter="bidi_from_code",109 values=bidi_classes110 .iter()111 .map(entry => @emitter.EnumValue::{ name: entry.0, comment: entry.1, })112 .collect(),113 )114 out.write_value_trie(115 property="Bidi_Class",116 prefix="bidi",117 lookup="fn lookup_bidi_class_code",118 lookup_doc=[119 "Look up the Bidi_Class code (the `BidiClass` constructor index).",120 ],121 value_bits=5,122 values,123 )124 out.write(bidi_class_implementation)125 out.finish()126}127128///|129let bidi_class_implementation : String =130 #|///|131 #|fn lookup_bidi_class(c : Char) -> BidiClass {132 #| bidi_from_code(lookup_bidi_class_code(c))133 #|}134 #|135 #|///|136 #|pub fn BidiClass::is_strong(self : BidiClass) -> Bool {137 #| self is (L | R | AL)138 #|}139 #|140 #|///|141 #|pub fn BidiClass::is_strong_rtl(self : BidiClass) -> Bool {142 #| self is (R | AL)143 #|}144 #|145 #|///|146 #|pub fn BidiClass::is_weak(self : BidiClass) -> Bool {147 #| self is (EN | ES | ET | AN | CS | NSM | BN)148 #|}149 #|150 #|///|151 #|pub fn BidiClass::is_neutral(self : BidiClass) -> Bool {152 #| self is (B | S | WS | ON)153 #|}154 #|155 #|///|156 #|pub fn BidiClass::is_explicit(self : BidiClass) -> Bool {157 #| self is (LRE | LRO | RLE | RLO | PDF | LRI | RLI | FSI | PDI)158 #|}159 #|160 #|///|161 #|pub fn BidiClass::is_isolate_initiator(self : BidiClass) -> Bool {162 #| self is (LRI | RLI | FSI)163 #|}164 #|165 #|///|166 #|pub fn BidiClass::is_x9_removed(self : BidiClass) -> Bool {167 #| self is (LRE | RLE | LRO | RLO | PDF | BN)168 #|}169 #|170 #|///|171 #|pub fn BidiClass::is_neutral_or_isolate(self : BidiClass) -> Bool {172 #| self is (B | S | WS | ON | FSI | LRI | RLI | PDI)173 #|}174 #|175176///|177pub fn emit_mirrored(content : String) -> String raise {178 let entries = parse_mirroring(content)179 let out = @emitter.Emitter::new()180 out.write_file_header(181 "Bidi_Mirrored and Bidi_Mirroring_Glyph lookup for UAX #9", "BidiMirroring.txt",182 )183 out.write_int_array(184 "Code points with mirroring property",185 "mirrored_code_points",186 entries.map(entry => entry.code_point),187 hex=true,188 )189 out.write_int_array(190 "Corresponding mirrored glyphs",191 "mirrored_glyphs",192 entries.map(entry => entry.mirrored),193 hex=true,194 )195 out.write(196 (197 #|///|198 #|pub fn is_bidi_mirrored(c : Char) -> Bool {199 #| find_mirrored_index(c.to_int()) >= 0200 #|}201 #|202 #|///|203 #|pub fn bidi_mirroring_glyph(c : Char) -> Char {204 #| let idx = find_mirrored_index(c.to_int())205 #| if idx < 0 { c } else { mirrored_glyphs[idx].unsafe_to_char() }206 #|}207 #|208 ),209 )210 out.write_sorted_search(211 name="find_mirrored_index",212 values="mirrored_code_points",213 )214 out.finish()215}216217///|218pub fn emit_bracket(content : String) -> String raise {219 let entries = parse_brackets(content)220 let out = @emitter.Emitter::new()221 out.write_file_header(222 "Bidi_Paired_Bracket lookup for UAX #9 N0 rule", "BidiBrackets.txt",223 )224 out.write_enum(225 type_name="BracketType",226 doc=["Bidi_Paired_Bracket_Type values"],227 converter="bracket_type_from_ordinal",228 values=bracket_type_values229 .iter()230 .map(entry => @emitter.EnumValue::{ name: entry.0, comment: entry.2, })231 .collect(),232 )233 out.write_int_array(234 "Code points of paired brackets",235 "bracket_code_points",236 entries.map(entry => entry.code_point),237 hex=true,238 )239 out.write_int_array(240 "Paired bracket for each code point",241 "paired_brackets",242 entries.map(entry => entry.paired),243 hex=true,244 )245 out.write_int_array(246 "`BracketType` ordinal of each code point",247 "bracket_types",248 entries.map(entry => entry.bracket_type),249 hex=false,250 )251 out.write(252 (253 #|///|254 #|pub fn bracket_type(c : Char) -> BracketType {255 #| let idx = find_bracket_index(c.to_int())256 #| if idx < 0 { None } else { bracket_type_from_ordinal(bracket_types[idx]) }257 #|}258 #|259 #|///|260 #|pub fn paired_bracket(c : Char) -> Char? {261 #| let idx = find_bracket_index(c.to_int())262 #| if idx < 0 { Option::None } else { Some(paired_brackets[idx].unsafe_to_char()) }263 #|}264 #|265 #|///|266 #|pub fn is_open_bracket(c : Char) -> Bool {267 #| bracket_type(c) == Open268 #|}269 #|270 #|///|271 #|pub fn is_close_bracket(c : Char) -> Bool {272 #| bracket_type(c) == Close273 #|}274 #|275 #|///|276 #|pub fn is_bracket_pair(open : Char, close : Char) -> Bool {277 #| if paired_bracket(open) is Some(pair) {278 #| pair == close && bracket_type(open) == Open279 #| } else {280 #| false281 #| }282 #|}283 #|284 ),285 )286 out.write_sorted_search(287 name="find_bracket_index",288 values="bracket_code_points",289 )290 out.finish()291}292
///|
priv struct MirrorEntry {
code_point : Int32
mirrored : Int32
}
///|
priv struct BracketEntry {
code_point : Int32
paired : Int32
/// Index of the bracket type in `bracket_type_values`.
bracket_type : Int32
}
///|
/// Bidi_Paired_Bracket_Type values in the order of the `BracketType` enum:
/// the constructor, the value BidiBrackets.txt writes, and a description.
let bracket_type_values : ReadOnlyArray[(String, String, String)] = [
("Open", "o", "Opening paired bracket"),
("Close", "c", "Closing paired bracket"),
("None", "n", "Not a paired bracket"),
]
///|
fn parse_mirroring(content : String) -> Array[MirrorEntry] raise {
let entries : Array[MirrorEntry] = []
for fields in @parsing.data_lines(content) {
if fields is [code_point, mirrored, ..] {
entries.push({
code_point: @parsing.parse_hex_int(code_point),
mirrored: @parsing.parse_hex_int(mirrored),
})
}
}
entries.sort_by((left, right) => left.code_point - right.code_point)
entries
}
///|
fn parse_brackets(content : String) -> Array[BracketEntry] raise {
let entries : Array[BracketEntry] = []
for fields in @parsing.data_lines(content) {
if fields is [code_point, paired, bracket_type, ..] {
let name = bracket_type.to_owned().to_lower()
guard bracket_type_values.search_by(entry => entry.1 == name)
is Some(bracket_type) else {
fail("Unknown Bidi_Paired_Bracket_Type \{name}")
}
entries.push({
code_point: @parsing.parse_hex_int(code_point),
paired: @parsing.parse_hex_int(paired),
bracket_type,
})
}
}
entries.sort_by((left, right) => left.code_point - right.code_point)
entries
}
///|
/// Bidi_Class values in the order of the `BidiClass` enum: the short alias,
/// which DerivedBidiClass.txt writes and which names the constructor, and the
/// long alias, which `@missing` lines spell out.
let bidi_classes : ReadOnlyArray[(String, String)] = [
("L", "Left_To_Right"),
("R", "Right_To_Left"),
("AL", "Arabic_Letter"),
("EN", "European_Number"),
("ES", "European_Separator"),
("ET", "European_Terminator"),
("AN", "Arabic_Number"),
("CS", "Common_Separator"),
("NSM", "Nonspacing_Mark"),
("BN", "Boundary_Neutral"),
("B", "Paragraph_Separator"),
("S", "Segment_Separator"),
("WS", "White_Space"),
("ON", "Other_Neutral"),
("LRE", "Left_To_Right_Embedding"),
("LRO", "Left_To_Right_Override"),
("RLE", "Right_To_Left_Embedding"),
("RLO", "Right_To_Left_Override"),
("PDF", "Pop_Directional_Format"),
("LRI", "Left_To_Right_Isolate"),
("RLI", "Right_To_Left_Isolate"),
("FSI", "First_Strong_Isolate"),
("PDI", "Pop_Directional_Isolate"),
]
///|
/// Emit the `BidiClass` enum and Bidi_Class data from `DerivedBidiClass.txt`
/// as a three-level trie for planes 0-3, with packed ranges for the remaining
/// planes.
pub fn emit_bidi_class(content : String) -> String raise {
let values = @parsing.parse_enumerated_ordinals(
content,
property="Bidi_Class",
default="L",
name => bidi_classes.search_by(entry => name == entry.0 || name == entry.1),
)
let out = @emitter.Emitter::new()
out.write_file_header(
"Full Bidi_Class lookup for Unicode Bidirectional Algorithm (UAX #9)", "DerivedBidiClass.txt",
)
out.write_enum(
type_name="BidiClass",
doc=["All \{bidi_classes.length()} Bidi_Class values from UAX #9"],
converter="bidi_from_code",
values=bidi_classes
.iter()
.map(entry => @emitter.EnumValue::{ name: entry.0, comment: entry.1, })
.collect(),
)
out.write_value_trie(
property="Bidi_Class",
prefix="bidi",
lookup="fn lookup_bidi_class_code",
lookup_doc=[
"Look up the Bidi_Class code (the `BidiClass` constructor index).",
],
value_bits=5,
values,
)
out.write(bidi_class_implementation)
out.finish()
}
///|
let bidi_class_implementation : String =
#|///|
#|fn lookup_bidi_class(c : Char) -> BidiClass {
#| bidi_from_code(lookup_bidi_class_code(c))
#|}
#|
#|///|
#|pub fn BidiClass::is_strong(self : BidiClass) -> Bool {
#| self is (L | R | AL)
#|}
#|
#|///|
#|pub fn BidiClass::is_strong_rtl(self : BidiClass) -> Bool {
#| self is (R | AL)
#|}
#|
#|///|
#|pub fn BidiClass::is_weak(self : BidiClass) -> Bool {
#| self is (EN | ES | ET | AN | CS | NSM | BN)
#|}
#|
#|///|
#|pub fn BidiClass::is_neutral(self : BidiClass) -> Bool {
#| self is (B | S | WS | ON)
#|}
#|
#|///|
#|pub fn BidiClass::is_explicit(self : BidiClass) -> Bool {
#| self is (LRE | LRO | RLE | RLO | PDF | LRI | RLI | FSI | PDI)
#|}
#|
#|///|
#|pub fn BidiClass::is_isolate_initiator(self : BidiClass) -> Bool {
#| self is (LRI | RLI | FSI)
#|}
#|
#|///|
#|pub fn BidiClass::is_x9_removed(self : BidiClass) -> Bool {
#| self is (LRE | RLE | LRO | RLO | PDF | BN)
#|}
#|
#|///|
#|pub fn BidiClass::is_neutral_or_isolate(self : BidiClass) -> Bool {
#| self is (B | S | WS | ON | FSI | LRI | RLI | PDI)
#|}
#|
///|
pub fn emit_mirrored(content : String) -> String raise {
let entries = parse_mirroring(content)
let out = @emitter.Emitter::new()
out.write_file_header(
"Bidi_Mirrored and Bidi_Mirroring_Glyph lookup for UAX #9", "BidiMirroring.txt",
)
out.write_int_array(
"Code points with mirroring property",
"mirrored_code_points",
entries.map(entry => entry.code_point),
hex=true,
)
out.write_int_array(
"Corresponding mirrored glyphs",
"mirrored_glyphs",
entries.map(entry => entry.mirrored),
hex=true,
)
out.write(
(
#|///|
#|pub fn is_bidi_mirrored(c : Char) -> Bool {
#| find_mirrored_index(c.to_int()) >= 0
#|}
#|
#|///|
#|pub fn bidi_mirroring_glyph(c : Char) -> Char {
#| let idx = find_mirrored_index(c.to_int())
#| if idx < 0 { c } else { mirrored_glyphs[idx].unsafe_to_char() }
#|}
#|
),
)
out.write_sorted_search(
name="find_mirrored_index",
values="mirrored_code_points",
)
out.finish()
}
///|
pub fn emit_bracket(content : String) -> String raise {
let entries = parse_brackets(content)
let out = @emitter.Emitter::new()
out.write_file_header(
"Bidi_Paired_Bracket lookup for UAX #9 N0 rule", "BidiBrackets.txt",
)
out.write_enum(
type_name="BracketType",
doc=["Bidi_Paired_Bracket_Type values"],
converter="bracket_type_from_ordinal",
values=bracket_type_values
.iter()
.map(entry => @emitter.EnumValue::{ name: entry.0, comment: entry.2, })
.collect(),
)
out.write_int_array(
"Code points of paired brackets",
"bracket_code_points",
entries.map(entry => entry.code_point),
hex=true,
)
out.write_int_array(
"Paired bracket for each code point",
"paired_brackets",
entries.map(entry => entry.paired),
hex=true,
)
out.write_int_array(
"`BracketType` ordinal of each code point",
"bracket_types",
entries.map(entry => entry.bracket_type),
hex=false,
)
out.write(
(
#|///|
#|pub fn bracket_type(c : Char) -> BracketType {
#| let idx = find_bracket_index(c.to_int())
#| if idx < 0 { None } else { bracket_type_from_ordinal(bracket_types[idx]) }
#|}
#|
#|///|
#|pub fn paired_bracket(c : Char) -> Char? {
#| let idx = find_bracket_index(c.to_int())
#| if idx < 0 { Option::None } else { Some(paired_brackets[idx].unsafe_to_char()) }
#|}
#|
#|///|
#|pub fn is_open_bracket(c : Char) -> Bool {
#| bracket_type(c) == Open
#|}
#|
#|///|
#|pub fn is_close_bracket(c : Char) -> Bool {
#| bracket_type(c) == Close
#|}
#|
#|///|
#|pub fn is_bracket_pair(open : Char, close : Char) -> Bool {
#| if paired_bracket(open) is Some(pair) {
#| pair == close && bracket_type(open) == Open
#| } else {
#| false
#| }
#|}
#|
),
)
out.write_sorted_search(
name="find_bracket_index",
values="bracket_code_points",
)
out.finish()
}