tools/segmentation_data/segmentation_data.mbt1///|2/// Number of low bits holding the Grapheme_Cluster_Break code in a packed3/// value.4const GcbBits : Int32 = 456///|7/// Number of bits holding the Indic_Conjunct_Break code, above the8/// Grapheme_Cluster_Break code.9const InCBBits : Int32 = 21011///|12/// Total number of bits in a packed value: Grapheme_Cluster_Break,13/// Indic_Conjunct_Break, and the Extended_Pictographic flag.14const ValueBits : Int32 = 71516///|17/// Grapheme_Cluster_Break values in code order. `Other` is the default for18/// code points GraphemeBreakProperty.txt does not list; the generated source19/// has no constant for it.20let grapheme_cluster_break_names : ReadOnlyArray[String] = [21 "Other", "CR", "LF", "Control", "Extend", "ZWJ", "Regional_Indicator", "Prepend",22 "SpacingMark", "L", "V", "T", "LV", "LVT",23]2425///|26/// Indic_Conjunct_Break values in code order. `None` is the default for code27/// points DerivedCoreProperties.txt does not list; the generated source has no28/// constant for it.29let indic_conjunct_break_names : ReadOnlyArray[String] = [30 "None", "Consonant", "Extend", "Linker",31]3233///|34/// Combine `bits` into the packed value of every code point in `range`.35fn set_range_bits(36 values : FixedArray[Int32],37 range : @parsing.CodePointRange,38 bits : Int32,39) -> Unit {40 for code_point in range.start..<=range.end {41 values[code_point] = values[code_point] | bits42 }43}4445///|46/// Compute the packed value of every code point:47/// `extended_pictographic << 6 | indic_conjunct_break << 4 |48/// grapheme_cluster_break`.49fn parse_grapheme_break_values(50 grapheme_break_property : String,51 emoji_data : String,52 derived_core_properties : String,53) -> FixedArray[Int32] raise {54 let values = @parsing.parse_enumerated_ordinals(55 grapheme_break_property,56 property="Grapheme_Cluster_Break",57 default="Other",58 name => grapheme_cluster_break_names.search_by(n => name == n),59 )60 for fields in @parsing.data_lines(derived_core_properties) {61 guard fields is [range, "InCB", name, ..] else { continue }62 guard indic_conjunct_break_names.search_by(n => name == n) is Some(code) else {63 fail("Unknown Indic_Conjunct_Break: \{name}")64 }65 set_range_bits(66 values,67 @parsing.parse_code_point_range(range),68 code << GcbBits,69 )70 }71 for fields in @parsing.data_lines(emoji_data) {72 guard fields is [range, "Extended_Pictographic", ..] else { continue }73 set_range_bits(74 values,75 @parsing.parse_code_point_range(range),76 1 << (GcbBits + InCBBits),77 )78 }79 values80}8182///|83/// Emit the grapheme cluster break properties as a three-level trie for84/// planes 0-3, with packed ranges for the remaining planes.85pub fn emit_grapheme_break(86 grapheme_break_property : String,87 emoji_data : String,88 derived_core_properties : String,89) -> String raise {90 let values = parse_grapheme_break_values(91 grapheme_break_property, emoji_data, derived_core_properties,92 )93 let out = @emitter.Emitter::new()94 out.write_file_header(95 "Grapheme cluster break properties for Unicode Text Segmentation (UAX #29)",96 "GraphemeBreakProperty.txt, emoji-data.txt, and DerivedCoreProperties.txt",97 )98 for code, name in grapheme_cluster_break_names {99 if code == 0 {100 continue101 }102 out.write(103 (104 $|///|105 $|/// Grapheme_Cluster_Break=\{name}106 $|const Gcb\{name.replace_all(old="_", new="")} : Int = \{code}107 $|108 $|109 ),110 )111 }112 for code, name in indic_conjunct_break_names {113 if code == 0 {114 continue115 }116 out.write(117 (118 $|///|119 $|/// Indic_Conjunct_Break=\{name}120 $|const InCB\{name} : Int = \{code}121 $|122 $|123 ),124 )125 }126 out.write(127 (128 $|///|129 $|/// Mask selecting the Grapheme_Cluster_Break code of a packed value.130 $|const GcbMask : Int = \{(1 << GcbBits) - 1}131 $|132 $|///|133 $|/// Shift moving the Indic_Conjunct_Break code of a packed value to bit 0.134 $|const InCBShift : Int = \{GcbBits}135 $|136 $|///|137 $|/// Mask selecting the Indic_Conjunct_Break code after shifting.138 $|const InCBMask : Int = \{(1 << InCBBits) - 1}139 $|140 $|///|141 $|/// Bit set in a packed value for Extended_Pictographic code points.142 $|const ExtendedPictographicBit : Int = \{1 << (GcbBits + InCBBits)}143 $|144 $|145 ),146 )147 out.write_value_trie(148 property="Grapheme break",149 prefix="grapheme_break",150 lookup="fn grapheme_break_value",151 lookup_doc=[152 "Look up the packed grapheme cluster break properties of a code point:",153 "`extended_pictographic << \{GcbBits + InCBBits} | indic_conjunct_break << \{GcbBits} |",154 "grapheme_cluster_break`.",155 ],156 value_bits=ValueBits,157 takes_code_point=true,158 values,159 )160 out.finish()161}162
///|
/// Number of low bits holding the Grapheme_Cluster_Break code in a packed
/// value.
const GcbBits : Int32 = 4
///|
/// Number of bits holding the Indic_Conjunct_Break code, above the
/// Grapheme_Cluster_Break code.
const InCBBits : Int32 = 2
///|
/// Total number of bits in a packed value: Grapheme_Cluster_Break,
/// Indic_Conjunct_Break, and the Extended_Pictographic flag.
const ValueBits : Int32 = 7
///|
/// Grapheme_Cluster_Break values in code order. `Other` is the default for
/// code points GraphemeBreakProperty.txt does not list; the generated source
/// has no constant for it.
let grapheme_cluster_break_names : ReadOnlyArray[String] = [
"Other", "CR", "LF", "Control", "Extend", "ZWJ", "Regional_Indicator", "Prepend",
"SpacingMark", "L", "V", "T", "LV", "LVT",
]
///|
/// Indic_Conjunct_Break values in code order. `None` is the default for code
/// points DerivedCoreProperties.txt does not list; the generated source has no
/// constant for it.
let indic_conjunct_break_names : ReadOnlyArray[String] = [
"None", "Consonant", "Extend", "Linker",
]
///|
/// Combine `bits` into the packed value of every code point in `range`.
fn set_range_bits(
values : FixedArray[Int32],
range : @parsing.CodePointRange,
bits : Int32,
) -> Unit {
for code_point in range.start..<=range.end {
values[code_point] = values[code_point] | bits
}
}
///|
/// Compute the packed value of every code point:
/// `extended_pictographic << 6 | indic_conjunct_break << 4 |
/// grapheme_cluster_break`.
fn parse_grapheme_break_values(
grapheme_break_property : String,
emoji_data : String,
derived_core_properties : String,
) -> FixedArray[Int32] raise {
let values = @parsing.parse_enumerated_ordinals(
grapheme_break_property,
property="Grapheme_Cluster_Break",
default="Other",
name => grapheme_cluster_break_names.search_by(n => name == n),
)
for fields in @parsing.data_lines(derived_core_properties) {
guard fields is [range, "InCB", name, ..] else { continue }
guard indic_conjunct_break_names.search_by(n => name == n) is Some(code) else {
fail("Unknown Indic_Conjunct_Break: \{name}")
}
set_range_bits(
values,
@parsing.parse_code_point_range(range),
code << GcbBits,
)
}
for fields in @parsing.data_lines(emoji_data) {
guard fields is [range, "Extended_Pictographic", ..] else { continue }
set_range_bits(
values,
@parsing.parse_code_point_range(range),
1 << (GcbBits + InCBBits),
)
}
values
}
///|
/// Emit the grapheme cluster break properties as a three-level trie for
/// planes 0-3, with packed ranges for the remaining planes.
pub fn emit_grapheme_break(
grapheme_break_property : String,
emoji_data : String,
derived_core_properties : String,
) -> String raise {
let values = parse_grapheme_break_values(
grapheme_break_property, emoji_data, derived_core_properties,
)
let out = @emitter.Emitter::new()
out.write_file_header(
"Grapheme cluster break properties for Unicode Text Segmentation (UAX #29)",
"GraphemeBreakProperty.txt, emoji-data.txt, and DerivedCoreProperties.txt",
)
for code, name in grapheme_cluster_break_names {
if code == 0 {
continue
}
out.write(
(
$|///|
$|/// Grapheme_Cluster_Break=\{name}
$|const Gcb\{name.replace_all(old="_", new="")} : Int = \{code}
$|
$|
),
)
}
for code, name in indic_conjunct_break_names {
if code == 0 {
continue
}
out.write(
(
$|///|
$|/// Indic_Conjunct_Break=\{name}
$|const InCB\{name} : Int = \{code}
$|
$|
),
)
}
out.write(
(
$|///|
$|/// Mask selecting the Grapheme_Cluster_Break code of a packed value.
$|const GcbMask : Int = \{(1 << GcbBits) - 1}
$|
$|///|
$|/// Shift moving the Indic_Conjunct_Break code of a packed value to bit 0.
$|const InCBShift : Int = \{GcbBits}
$|
$|///|
$|/// Mask selecting the Indic_Conjunct_Break code after shifting.
$|const InCBMask : Int = \{(1 << InCBBits) - 1}
$|
$|///|
$|/// Bit set in a packed value for Extended_Pictographic code points.
$|const ExtendedPictographicBit : Int = \{1 << (GcbBits + InCBBits)}
$|
$|
),
)
out.write_value_trie(
property="Grapheme break",
prefix="grapheme_break",
lookup="fn grapheme_break_value",
lookup_doc=[
"Look up the packed grapheme cluster break properties of a code point:",
"`extended_pictographic << \{GcbBits + InCBBits} | indic_conjunct_break << \{GcbBits} |",
"grapheme_cluster_break`.",
],
value_bits=ValueBits,
takes_code_point=true,
values,
)
out.finish()
}