unicode

Branch: main

Switch branches or tags

Branch: main
Go to file t

413 files indexed

History
Code

tools/ucd/joining_group.mbt
1///|2/// Number of bits holding a Joining_Group ordinal in the packed high ranges.3const JoiningGroupBits : Int32 = 745///|6/// Order two value names as a dictionary does. `String::compare` orders by7/// length first.8fn dictionary_order(a : String, b : String) -> Int32 {9  for i in 0..<Int32::min(a.length(), b.length()) {10    let order = a[i].compare(b[i])11    if order != 0 {12      return order13    }14  }15  a.length().compare(b.length())16}1718///|19/// Enum constructor for a Joining_Group value: the value name without its20/// underscores, so that `Yeh_Barree` becomes `YehBarree`.21fn joining_group_constructor(value : String) -> String {22  value.replace_all(old="_", new="")23}2425///|26/// The Joining_Group value of every code point, and the values occurring in27/// `DerivedJoiningGroup.txt` in dictionary order, which is the order of the28/// `JoiningGroup` enum.29fn parse_joining_groups(30  content : String,31) -> (FixedArray[String], Array[String]) raise {32  let names = @parsing.parse_enumerated_property(33    content,34    default="No_Joining_Group",35  )36  let distinct : Map[String, Unit] = Map([])37  for name in names {38    distinct[name] = ()39  }40  let sorted = distinct.keys().collect()41  sorted.sort_by(dictionary_order)42  guard sorted.length() <= 1 << JoiningGroupBits else {43    fail("Joining_Group values do not fit in \{JoiningGroupBits} bits")44  }45  (names, sorted)46}4748///|49/// Emit Joining_Group data from `DerivedJoiningGroup.txt` as a three-level50/// trie for planes 0-3, with packed ranges for the remaining planes.51pub fn emit_joining_group(content : String) -> String raise {52  let (names, sorted) = parse_joining_groups(content)53  let ordinals : Map[String, Int32] = Map([])54  for ordinal, name in sorted {55    ordinals[name] = ordinal56  }57  let values = FixedArray::makei(names.length(), cp => ordinals[names[cp]])58  let out = @emitter.Emitter::new()59  out.write_file_header("Joining_Group lookup data", "DerivedJoiningGroup.txt")60  write_enumerated_trie(61    out,62    property="Joining_Group",63    prefix="joining_group",64    enum_name="JoiningGroup",65    value_count=sorted.length(),66    value_bits=JoiningGroupBits,67    values,68  )69  out.finish()70}7172///|73/// Emit the `JoiningGroup` enum and the `joining_group` lookup of the root74/// package, which reads the ordinals `emit_joining_group` stores.75///76/// The enum lists the values occurring in `DerivedJoiningGroup.txt` in77/// dictionary order.78pub fn emit_joining_group_enum(content : String) -> String raise {79  let (_, sorted) = parse_joining_groups(content)80  let out = @emitter.Emitter::new()81  out.write_file_header(82    "Joining_Group values and lookup", "DerivedJoiningGroup.txt",83  )84  out.write_enum(85    type_name="JoiningGroup",86    doc=[87      "Unicode Joining_Group values", "See the Unicode Standard, section 9.2, Arabic:",88      "https://www.unicode.org/versions/latest/core-spec/chapter-9/#G7480",89    ],90    converter="ordinal_to_joining_group",91    values=sorted.map(name => @emitter.EnumValue::{92      name: joining_group_constructor(name),93      comment: name,94    }),95  )96  out.write(97    (98      #|///|99      #|/// Get the Joining_Group for a character.100      #|///101      #|/// Joining_Group classifies the letters of cursive scripts such as Arabic,102      #|/// Syriac, and Manichaean by the shape they join with: letters that103      #|/// differ only in their dots, like U+0628 ARABIC LETTER BEH and U+062A104      #|/// ARABIC LETTER TEH, share a group. Characters that are not such105      #|/// letters, including unassigned code points, are NoJoiningGroup.106      #|///107      #|/// The value `Pe` shares its name with a `GeneralCategory` value: write108      #|/// `JoiningGroup::Pe` or `GeneralCategory::Pe` where the type is not109      #|/// known from the context.110      #|pub fn joining_group(c : Char) -> JoiningGroup {111      #|  ordinal_to_joining_group(@joining_group.lookup_joining_group(c))112      #|}113      #|114    ),115  )116  out.finish()117}118