unicode

Branch: main

Switch branches or tags

Branch: main
Go to file t

413 files indexed

History
Code

tools/bidi_data/bidi_data.mbt
1///|2priv struct MirrorEntry {3  code_point : Int324  mirrored : Int325}67///|8priv struct BracketEntry {9  code_point : Int3210  paired : Int3211  /// Index of the bracket type in `bracket_type_values`.12  bracket_type : Int3213}1415///|16/// Bidi_Paired_Bracket_Type values in the order of the `BracketType` enum:17/// the constructor, the value BidiBrackets.txt writes, and a description.18let bracket_type_values : ReadOnlyArray[(String, String, String)] = [19  ("Open", "o", "Opening paired bracket"),20  ("Close", "c", "Closing paired bracket"),21  ("None", "n", "Not a paired bracket"),22]2324///|25fn parse_mirroring(content : String) -> Array[MirrorEntry] raise {26  let entries : Array[MirrorEntry] = []27  for fields in @parsing.data_lines(content) {28    if fields is [code_point, mirrored, ..] {29      entries.push({30        code_point: @parsing.parse_hex_int(code_point),31        mirrored: @parsing.parse_hex_int(mirrored),32      })33    }34  }35  entries.sort_by((left, right) => left.code_point - right.code_point)36  entries37}3839///|40fn parse_brackets(content : String) -> Array[BracketEntry] raise {41  let entries : Array[BracketEntry] = []42  for fields in @parsing.data_lines(content) {43    if fields is [code_point, paired, bracket_type, ..] {44      let name = bracket_type.to_owned().to_lower()45      guard bracket_type_values.search_by(entry => entry.1 == name)46        is Some(bracket_type) else {47        fail("Unknown Bidi_Paired_Bracket_Type \{name}")48      }49      entries.push({50        code_point: @parsing.parse_hex_int(code_point),51        paired: @parsing.parse_hex_int(paired),52        bracket_type,53      })54    }55  }56  entries.sort_by((left, right) => left.code_point - right.code_point)57  entries58}5960///|61/// Bidi_Class values in the order of the `BidiClass` enum: the short alias,62/// which DerivedBidiClass.txt writes and which names the constructor, and the63/// long alias, which `@missing` lines spell out.64let bidi_classes : ReadOnlyArray[(String, String)] = [65  ("L", "Left_To_Right"),66  ("R", "Right_To_Left"),67  ("AL", "Arabic_Letter"),68  ("EN", "European_Number"),69  ("ES", "European_Separator"),70  ("ET", "European_Terminator"),71  ("AN", "Arabic_Number"),72  ("CS", "Common_Separator"),73  ("NSM", "Nonspacing_Mark"),74  ("BN", "Boundary_Neutral"),75  ("B", "Paragraph_Separator"),76  ("S", "Segment_Separator"),77  ("WS", "White_Space"),78  ("ON", "Other_Neutral"),79  ("LRE", "Left_To_Right_Embedding"),80  ("LRO", "Left_To_Right_Override"),81  ("RLE", "Right_To_Left_Embedding"),82  ("RLO", "Right_To_Left_Override"),83  ("PDF", "Pop_Directional_Format"),84  ("LRI", "Left_To_Right_Isolate"),85  ("RLI", "Right_To_Left_Isolate"),86  ("FSI", "First_Strong_Isolate"),87  ("PDI", "Pop_Directional_Isolate"),88]8990///|91/// Emit the `BidiClass` enum and Bidi_Class data from `DerivedBidiClass.txt`92/// as a three-level trie for planes 0-3, with packed ranges for the remaining93/// planes.94pub fn emit_bidi_class(content : String) -> String raise {95  let values = @parsing.parse_enumerated_ordinals(96    content,97    property="Bidi_Class",98    default="L",99    name => bidi_classes.search_by(entry => name == entry.0 || name == entry.1),100  )101  let out = @emitter.Emitter::new()102  out.write_file_header(103    "Full Bidi_Class lookup for Unicode Bidirectional Algorithm (UAX #9)", "DerivedBidiClass.txt",104  )105  out.write_enum(106    type_name="BidiClass",107    doc=["All \{bidi_classes.length()} Bidi_Class values from UAX #9"],108    converter="bidi_from_code",109    values=bidi_classes110      .iter()111      .map(entry => @emitter.EnumValue::{ name: entry.0, comment: entry.1, })112      .collect(),113  )114  out.write_value_trie(115    property="Bidi_Class",116    prefix="bidi",117    lookup="fn lookup_bidi_class_code",118    lookup_doc=[119      "Look up the Bidi_Class code (the `BidiClass` constructor index).",120    ],121    value_bits=5,122    values,123  )124  out.write(bidi_class_implementation)125  out.finish()126}127128///|129let bidi_class_implementation : String =130  #|///|131  #|fn lookup_bidi_class(c : Char) -> BidiClass {132  #|  bidi_from_code(lookup_bidi_class_code(c))133  #|}134  #|135  #|///|136  #|pub fn BidiClass::is_strong(self : BidiClass) -> Bool {137  #|  self is (L | R | AL)138  #|}139  #|140  #|///|141  #|pub fn BidiClass::is_strong_rtl(self : BidiClass) -> Bool {142  #|  self is (R | AL)143  #|}144  #|145  #|///|146  #|pub fn BidiClass::is_weak(self : BidiClass) -> Bool {147  #|  self is (EN | ES | ET | AN | CS | NSM | BN)148  #|}149  #|150  #|///|151  #|pub fn BidiClass::is_neutral(self : BidiClass) -> Bool {152  #|  self is (B | S | WS | ON)153  #|}154  #|155  #|///|156  #|pub fn BidiClass::is_explicit(self : BidiClass) -> Bool {157  #|  self is (LRE | LRO | RLE | RLO | PDF | LRI | RLI | FSI | PDI)158  #|}159  #|160  #|///|161  #|pub fn BidiClass::is_isolate_initiator(self : BidiClass) -> Bool {162  #|  self is (LRI | RLI | FSI)163  #|}164  #|165  #|///|166  #|pub fn BidiClass::is_x9_removed(self : BidiClass) -> Bool {167  #|  self is (LRE | RLE | LRO | RLO | PDF | BN)168  #|}169  #|170  #|///|171  #|pub fn BidiClass::is_neutral_or_isolate(self : BidiClass) -> Bool {172  #|  self is (B | S | WS | ON | FSI | LRI | RLI | PDI)173  #|}174  #|175176///|177pub fn emit_mirrored(content : String) -> String raise {178  let entries = parse_mirroring(content)179  let out = @emitter.Emitter::new()180  out.write_file_header(181    "Bidi_Mirrored and Bidi_Mirroring_Glyph lookup for UAX #9", "BidiMirroring.txt",182  )183  out.write_int_array(184    "Code points with mirroring property",185    "mirrored_code_points",186    entries.map(entry => entry.code_point),187    hex=true,188  )189  out.write_int_array(190    "Corresponding mirrored glyphs",191    "mirrored_glyphs",192    entries.map(entry => entry.mirrored),193    hex=true,194  )195  out.write(196    (197      #|///|198      #|pub fn is_bidi_mirrored(c : Char) -> Bool {199      #|  find_mirrored_index(c.to_int()) >= 0200      #|}201      #|202      #|///|203      #|pub fn bidi_mirroring_glyph(c : Char) -> Char {204      #|  let idx = find_mirrored_index(c.to_int())205      #|  if idx < 0 { c } else { mirrored_glyphs[idx].unsafe_to_char() }206      #|}207      #|208    ),209  )210  out.write_sorted_search(211    name="find_mirrored_index",212    values="mirrored_code_points",213  )214  out.finish()215}216217///|218pub fn emit_bracket(content : String) -> String raise {219  let entries = parse_brackets(content)220  let out = @emitter.Emitter::new()221  out.write_file_header(222    "Bidi_Paired_Bracket lookup for UAX #9 N0 rule", "BidiBrackets.txt",223  )224  out.write_enum(225    type_name="BracketType",226    doc=["Bidi_Paired_Bracket_Type values"],227    converter="bracket_type_from_ordinal",228    values=bracket_type_values229      .iter()230      .map(entry => @emitter.EnumValue::{ name: entry.0, comment: entry.2, })231      .collect(),232  )233  out.write_int_array(234    "Code points of paired brackets",235    "bracket_code_points",236    entries.map(entry => entry.code_point),237    hex=true,238  )239  out.write_int_array(240    "Paired bracket for each code point",241    "paired_brackets",242    entries.map(entry => entry.paired),243    hex=true,244  )245  out.write_int_array(246    "`BracketType` ordinal of each code point",247    "bracket_types",248    entries.map(entry => entry.bracket_type),249    hex=false,250  )251  out.write(252    (253      #|///|254      #|pub fn bracket_type(c : Char) -> BracketType {255      #|  let idx = find_bracket_index(c.to_int())256      #|  if idx < 0 { None } else { bracket_type_from_ordinal(bracket_types[idx]) }257      #|}258      #|259      #|///|260      #|pub fn paired_bracket(c : Char) -> Char? {261      #|  let idx = find_bracket_index(c.to_int())262      #|  if idx < 0 { Option::None } else { Some(paired_brackets[idx].unsafe_to_char()) }263      #|}264      #|265      #|///|266      #|pub fn is_open_bracket(c : Char) -> Bool {267      #|  bracket_type(c) == Open268      #|}269      #|270      #|///|271      #|pub fn is_close_bracket(c : Char) -> Bool {272      #|  bracket_type(c) == Close273      #|}274      #|275      #|///|276      #|pub fn is_bracket_pair(open : Char, close : Char) -> Bool {277      #|  if paired_bracket(open) is Some(pair) {278      #|    pair == close && bracket_type(open) == Open279      #|  } else {280      #|    false281      #|  }282      #|}283      #|284    ),285  )286  out.write_sorted_search(287    name="find_bracket_index",288    values="bracket_code_points",289  )290  out.finish()291}292