unicode

Branch: main

Switch branches or tags

Branch: main
Go to file t

413 files indexed

History
Code

tools/segmentation_data/segmentation_data.mbt
1///|2/// Number of low bits holding the Grapheme_Cluster_Break code in a packed3/// value.4const GcbBits : Int32 = 456///|7/// Number of bits holding the Indic_Conjunct_Break code, above the8/// Grapheme_Cluster_Break code.9const InCBBits : Int32 = 21011///|12/// Total number of bits in a packed value: Grapheme_Cluster_Break,13/// Indic_Conjunct_Break, and the Extended_Pictographic flag.14const ValueBits : Int32 = 71516///|17/// Grapheme_Cluster_Break values in code order. `Other` is the default for18/// code points GraphemeBreakProperty.txt does not list; the generated source19/// has no constant for it.20let grapheme_cluster_break_names : ReadOnlyArray[String] = [21  "Other", "CR", "LF", "Control", "Extend", "ZWJ", "Regional_Indicator", "Prepend",22  "SpacingMark", "L", "V", "T", "LV", "LVT",23]2425///|26/// Indic_Conjunct_Break values in code order. `None` is the default for code27/// points DerivedCoreProperties.txt does not list; the generated source has no28/// constant for it.29let indic_conjunct_break_names : ReadOnlyArray[String] = [30  "None", "Consonant", "Extend", "Linker",31]3233///|34/// Combine `bits` into the packed value of every code point in `range`.35fn set_range_bits(36  values : FixedArray[Int32],37  range : @parsing.CodePointRange,38  bits : Int32,39) -> Unit {40  for code_point in range.start..<=range.end {41    values[code_point] = values[code_point] | bits42  }43}4445///|46/// Compute the packed value of every code point:47/// `extended_pictographic << 6 | indic_conjunct_break << 4 |48/// grapheme_cluster_break`.49fn parse_grapheme_break_values(50  grapheme_break_property : String,51  emoji_data : String,52  derived_core_properties : String,53) -> FixedArray[Int32] raise {54  let values = @parsing.parse_enumerated_ordinals(55    grapheme_break_property,56    property="Grapheme_Cluster_Break",57    default="Other",58    name => grapheme_cluster_break_names.search_by(n => name == n),59  )60  for fields in @parsing.data_lines(derived_core_properties) {61    guard fields is [range, "InCB", name, ..] else { continue }62    guard indic_conjunct_break_names.search_by(n => name == n) is Some(code) else {63      fail("Unknown Indic_Conjunct_Break: \{name}")64    }65    set_range_bits(66      values,67      @parsing.parse_code_point_range(range),68      code << GcbBits,69    )70  }71  for fields in @parsing.data_lines(emoji_data) {72    guard fields is [range, "Extended_Pictographic", ..] else { continue }73    set_range_bits(74      values,75      @parsing.parse_code_point_range(range),76      1 << (GcbBits + InCBBits),77    )78  }79  values80}8182///|83/// Emit the grapheme cluster break properties as a three-level trie for84/// planes 0-3, with packed ranges for the remaining planes.85pub fn emit_grapheme_break(86  grapheme_break_property : String,87  emoji_data : String,88  derived_core_properties : String,89) -> String raise {90  let values = parse_grapheme_break_values(91    grapheme_break_property, emoji_data, derived_core_properties,92  )93  let out = @emitter.Emitter::new()94  out.write_file_header(95    "Grapheme cluster break properties for Unicode Text Segmentation (UAX #29)",96    "GraphemeBreakProperty.txt, emoji-data.txt, and DerivedCoreProperties.txt",97  )98  for code, name in grapheme_cluster_break_names {99    if code == 0 {100      continue101    }102    out.write(103      (104        $|///|105        $|/// Grapheme_Cluster_Break=\{name}106        $|const Gcb\{name.replace_all(old="_", new="")} : Int = \{code}107        $|108        $|109      ),110    )111  }112  for code, name in indic_conjunct_break_names {113    if code == 0 {114      continue115    }116    out.write(117      (118        $|///|119        $|/// Indic_Conjunct_Break=\{name}120        $|const InCB\{name} : Int = \{code}121        $|122        $|123      ),124    )125  }126  out.write(127    (128      $|///|129      $|/// Mask selecting the Grapheme_Cluster_Break code of a packed value.130      $|const GcbMask : Int = \{(1 << GcbBits) - 1}131      $|132      $|///|133      $|/// Shift moving the Indic_Conjunct_Break code of a packed value to bit 0.134      $|const InCBShift : Int = \{GcbBits}135      $|136      $|///|137      $|/// Mask selecting the Indic_Conjunct_Break code after shifting.138      $|const InCBMask : Int = \{(1 << InCBBits) - 1}139      $|140      $|///|141      $|/// Bit set in a packed value for Extended_Pictographic code points.142      $|const ExtendedPictographicBit : Int = \{1 << (GcbBits + InCBBits)}143      $|144      $|145    ),146  )147  out.write_value_trie(148    property="Grapheme break",149    prefix="grapheme_break",150    lookup="fn grapheme_break_value",151    lookup_doc=[152      "Look up the packed grapheme cluster break properties of a code point:",153      "`extended_pictographic << \{GcbBits + InCBBits} | indic_conjunct_break << \{GcbBits} |",154      "grapheme_cluster_break`.",155    ],156    value_bits=ValueBits,157    takes_code_point=true,158    values,159  )160  out.finish()161}162