tools/ucd/joining_group.mbt1///|2/// Number of bits holding a Joining_Group ordinal in the packed high ranges.3const JoiningGroupBits : Int32 = 745///|6/// Order two value names as a dictionary does. `String::compare` orders by7/// length first.8fn dictionary_order(a : String, b : String) -> Int32 {9 for i in 0..<Int32::min(a.length(), b.length()) {10 let order = a[i].compare(b[i])11 if order != 0 {12 return order13 }14 }15 a.length().compare(b.length())16}1718///|19/// Enum constructor for a Joining_Group value: the value name without its20/// underscores, so that `Yeh_Barree` becomes `YehBarree`.21fn joining_group_constructor(value : String) -> String {22 value.replace_all(old="_", new="")23}2425///|26/// The Joining_Group value of every code point, and the values occurring in27/// `DerivedJoiningGroup.txt` in dictionary order, which is the order of the28/// `JoiningGroup` enum.29fn parse_joining_groups(30 content : String,31) -> (FixedArray[String], Array[String]) raise {32 let names = @parsing.parse_enumerated_property(33 content,34 default="No_Joining_Group",35 )36 let distinct : Map[String, Unit] = Map([])37 for name in names {38 distinct[name] = ()39 }40 let sorted = distinct.keys().collect()41 sorted.sort_by(dictionary_order)42 guard sorted.length() <= 1 << JoiningGroupBits else {43 fail("Joining_Group values do not fit in \{JoiningGroupBits} bits")44 }45 (names, sorted)46}4748///|49/// Emit Joining_Group data from `DerivedJoiningGroup.txt` as a three-level50/// trie for planes 0-3, with packed ranges for the remaining planes.51pub fn emit_joining_group(content : String) -> String raise {52 let (names, sorted) = parse_joining_groups(content)53 let ordinals : Map[String, Int32] = Map([])54 for ordinal, name in sorted {55 ordinals[name] = ordinal56 }57 let values = FixedArray::makei(names.length(), cp => ordinals[names[cp]])58 let out = @emitter.Emitter::new()59 out.write_file_header("Joining_Group lookup data", "DerivedJoiningGroup.txt")60 write_enumerated_trie(61 out,62 property="Joining_Group",63 prefix="joining_group",64 enum_name="JoiningGroup",65 value_count=sorted.length(),66 value_bits=JoiningGroupBits,67 values,68 )69 out.finish()70}7172///|73/// Emit the `JoiningGroup` enum and the `joining_group` lookup of the root74/// package, which reads the ordinals `emit_joining_group` stores.75///76/// The enum lists the values occurring in `DerivedJoiningGroup.txt` in77/// dictionary order.78pub fn emit_joining_group_enum(content : String) -> String raise {79 let (_, sorted) = parse_joining_groups(content)80 let out = @emitter.Emitter::new()81 out.write_file_header(82 "Joining_Group values and lookup", "DerivedJoiningGroup.txt",83 )84 out.write_enum(85 type_name="JoiningGroup",86 doc=[87 "Unicode Joining_Group values", "See the Unicode Standard, section 9.2, Arabic:",88 "https://www.unicode.org/versions/latest/core-spec/chapter-9/#G7480",89 ],90 converter="ordinal_to_joining_group",91 values=sorted.map(name => @emitter.EnumValue::{92 name: joining_group_constructor(name),93 comment: name,94 }),95 )96 out.write(97 (98 #|///|99 #|/// Get the Joining_Group for a character.100 #|///101 #|/// Joining_Group classifies the letters of cursive scripts such as Arabic,102 #|/// Syriac, and Manichaean by the shape they join with: letters that103 #|/// differ only in their dots, like U+0628 ARABIC LETTER BEH and U+062A104 #|/// ARABIC LETTER TEH, share a group. Characters that are not such105 #|/// letters, including unassigned code points, are NoJoiningGroup.106 #|///107 #|/// The value `Pe` shares its name with a `GeneralCategory` value: write108 #|/// `JoiningGroup::Pe` or `GeneralCategory::Pe` where the type is not109 #|/// known from the context.110 #|pub fn joining_group(c : Char) -> JoiningGroup {111 #| ordinal_to_joining_group(@joining_group.lookup_joining_group(c))112 #|}113 #|114 ),115 )116 out.finish()117}118
///|
/// Number of bits holding a Joining_Group ordinal in the packed high ranges.
const JoiningGroupBits : Int32 = 7
///|
/// Order two value names as a dictionary does. `String::compare` orders by
/// length first.
fn dictionary_order(a : String, b : String) -> Int32 {
for i in 0..<Int32::min(a.length(), b.length()) {
let order = a[i].compare(b[i])
if order != 0 {
return order
}
}
a.length().compare(b.length())
}
///|
/// Enum constructor for a Joining_Group value: the value name without its
/// underscores, so that `Yeh_Barree` becomes `YehBarree`.
fn joining_group_constructor(value : String) -> String {
value.replace_all(old="_", new="")
}
///|
/// The Joining_Group value of every code point, and the values occurring in
/// `DerivedJoiningGroup.txt` in dictionary order, which is the order of the
/// `JoiningGroup` enum.
fn parse_joining_groups(
content : String,
) -> (FixedArray[String], Array[String]) raise {
let names = @parsing.parse_enumerated_property(
content,
default="No_Joining_Group",
)
let distinct : Map[String, Unit] = Map([])
for name in names {
distinct[name] = ()
}
let sorted = distinct.keys().collect()
sorted.sort_by(dictionary_order)
guard sorted.length() <= 1 << JoiningGroupBits else {
fail("Joining_Group values do not fit in \{JoiningGroupBits} bits")
}
(names, sorted)
}
///|
/// Emit Joining_Group data from `DerivedJoiningGroup.txt` as a three-level
/// trie for planes 0-3, with packed ranges for the remaining planes.
pub fn emit_joining_group(content : String) -> String raise {
let (names, sorted) = parse_joining_groups(content)
let ordinals : Map[String, Int32] = Map([])
for ordinal, name in sorted {
ordinals[name] = ordinal
}
let values = FixedArray::makei(names.length(), cp => ordinals[names[cp]])
let out = @emitter.Emitter::new()
out.write_file_header("Joining_Group lookup data", "DerivedJoiningGroup.txt")
write_enumerated_trie(
out,
property="Joining_Group",
prefix="joining_group",
enum_name="JoiningGroup",
value_count=sorted.length(),
value_bits=JoiningGroupBits,
values,
)
out.finish()
}
///|
/// Emit the `JoiningGroup` enum and the `joining_group` lookup of the root
/// package, which reads the ordinals `emit_joining_group` stores.
///
/// The enum lists the values occurring in `DerivedJoiningGroup.txt` in
/// dictionary order.
pub fn emit_joining_group_enum(content : String) -> String raise {
let (_, sorted) = parse_joining_groups(content)
let out = @emitter.Emitter::new()
out.write_file_header(
"Joining_Group values and lookup", "DerivedJoiningGroup.txt",
)
out.write_enum(
type_name="JoiningGroup",
doc=[
"Unicode Joining_Group values", "See the Unicode Standard, section 9.2, Arabic:",
"https://www.unicode.org/versions/latest/core-spec/chapter-9/#G7480",
],
converter="ordinal_to_joining_group",
values=sorted.map(name => @emitter.EnumValue::{
name: joining_group_constructor(name),
comment: name,
}),
)
out.write(
(
#|///|
#|/// Get the Joining_Group for a character.
#|///
#|/// Joining_Group classifies the letters of cursive scripts such as Arabic,
#|/// Syriac, and Manichaean by the shape they join with: letters that
#|/// differ only in their dots, like U+0628 ARABIC LETTER BEH and U+062A
#|/// ARABIC LETTER TEH, share a group. Characters that are not such
#|/// letters, including unassigned code points, are NoJoiningGroup.
#|///
#|/// The value `Pe` shares its name with a `GeneralCategory` value: write
#|/// `JoiningGroup::Pe` or `GeneralCategory::Pe` where the type is not
#|/// known from the context.
#|pub fn joining_group(c : Char) -> JoiningGroup {
#| ordinal_to_joining_group(@joining_group.lookup_joining_group(c))
#|}
#|
),
)
out.finish()
}