unicode

Branch: main

Switch branches or tags

Branch: main
Go to file t

413 files indexed

History
Code

tools/ucd/emit.mbt
1///|2pub fn emit_ccc() -> String {3  let out = @emitter.Emitter::new()4  out.write_file_header(5    "Canonical Combining Class (CCC) lookup", "UnicodeData.txt",6  )7  out.write(8    (9      #|///|10      #|pub using @normalization_data {lookup_normalization_properties}11      #|12      #|///|13      #|/// Look up Canonical Combining Class; 0 denotes a starter.14      #|pub fn lookup_ccc(c : Char) -> Int {15      #|  @normalization_data.lookup_normalization_properties(c) & 25516      #|}17      #|18    ),19  )20  out.finish()21}2223///|24/// Bits of a packed decomposition entry: the code point occupies the low25/// 21 bits and its Canonical_Combining_Class the eight above them.26const DecompositionCccShift : Int32 = 212728///|29const DecompositionCodePointMask : Int32 = 0x1FFFFF3031///|32/// A packed decomposition entry: `code_point` and its combining class.33fn packed_entry(data : UnicodeData, code_point : Int32) -> Int32 {34  code_point | (data.ccc[code_point] << DecompositionCccShift)35}3637///|38/// Append the full decomposition of `code_point` to `dest` as packed39/// entries: its mapping, applied recursively, where a compatibility mapping40/// applies only when `compatibility` is set.41fn append_full_decomposition(42  data : UnicodeData,43  mappings : Map[Int32, Decomposition],44  code_point : Int32,45  compatibility : Bool,46  dest : Array[Int32],47) -> Unit raise {48  // Hangul syllables decompose algorithmically. The runtime does not look49  // them up, so a mapping that produced one would leave it composed.50  guard code_point < 0xAC00 || code_point > 0xD7A3 else {51    fail("A decomposition mapping produces a Hangul syllable")52  }53  if mappings.get(code_point) is Some(entry) &&54    (compatibility || !entry.compatibility) {55    for target in entry.code_points {56      append_full_decomposition(data, mappings, target, compatibility, dest)57    }58  } else {59    dest.push(packed_entry(data, code_point))60  }61}6263///|64pub fn emit_decomposition(data : UnicodeData) -> String raise {65  let mappings : Map[Int32, Decomposition] = Map([])66  for entry in data.decompositions {67    mappings[entry.code_point] = entry68  }69  // Each row's entries are its full canonical decomposition (canonical70  // mappings only), its full compatibility decomposition (compatibility71  // mappings, or canonical ones whose full compatibility decomposition72  // differs), and its own mapping, when that differs from the full73  // decomposition of its kind.74  let starts : Array[Int32] = []75  let canonical_lens : Array[Int32] = []76  let compat_lens : Array[Int32] = []77  let mapping_lens : Array[Int32] = []78  let entries : Array[Int32] = []79  for entry in data.decompositions {80    starts.push(entries.length())81    let compat : Array[Int32] = []82    append_full_decomposition(data, mappings, entry.code_point, true, compat)83    let full = if entry.compatibility {84      canonical_lens.push(0)85      compat_lens.push(compat.length())86      entries.append(compat)87      compat88    } else {89      let canonical : Array[Int32] = []90      append_full_decomposition(91        data,92        mappings,93        entry.code_point,94        false,95        canonical,96      )97      canonical_lens.push(canonical.length())98      entries.append(canonical)99      if compat == canonical {100        compat_lens.push(0)101      } else {102        compat_lens.push(compat.length())103        entries.append(compat)104      }105      canonical106    }107    let mapping = entry.code_points.map(code_point => {108      packed_entry(data, code_point)109    })110    if mapping == full {111      mapping_lens.push(0)112    } else {113      mapping_lens.push(mapping.length())114      entries.append(mapping)115    }116  }117  guard canonical_lens.all(len => len <= 255) &&118    compat_lens.all(len => len <= 255) &&119    mapping_lens.all(len => len <= 255) else {120    fail("Decomposition length exceeds its encoded byte width")121  }122  let out = @emitter.Emitter::new()123  out.write_file_header("Decomposition mapping data", "UnicodeData.txt")124  out.write_int_array(125    "Start of each decomposition row's entries",126    "decomp_data_starts",127    starts,128    hex=false,129  )130  out.write_byte_array(131    "Length of the full canonical decomposition (0 for a compatibility mapping)",132    "decomp_canonical_lens", canonical_lens,133  )134  out.write_byte_array(135    "Length of the full compatibility decomposition (0 if it is the canonical one)",136    "decomp_compat_lens", compat_lens,137  )138  out.write_byte_array(139    "Length of the mapping itself (0 if it is the full decomposition)", "decomp_mapping_lens",140    mapping_lens,141  )142  out.write_int_array(143    "Packed entries: code point in bits 0..20, its CCC in bits 21..28",144    "decomp_data",145    entries,146    hex=true,147  )148  // Lets the lookup skip everything below the first character with a mapping.149  let first_decomposable = data.decompositions.fold(150    init=@emitter.MaxCodePoint + 1,151    (first, entry) => Int32::min(first, entry.code_point),152  )153  out.write(154    (155      $|///|156      $|/// The mapping of row `idx` itself, which is canonical when the row has157      $|/// a full canonical decomposition.158      $|fn mapping_entries(idx : Int) -> ArrayView[Int] {159      $|  let start = decomp_data_starts[idx]160      $|  let canonical_len = decomp_canonical_lens[idx].to_int()161      $|  let full_len = if canonical_len != 0 {162      $|    canonical_len163      $|  } else {164      $|    decomp_compat_lens[idx].to_int()165      $|  }166      $|  let mapping_len = decomp_mapping_lens[idx].to_int()167      $|  if mapping_len == 0 {168      $|    return decomp_data.exact_view(start~, end=start + full_len)169      $|  }170      $|  let mapping_start = start +171      $|    canonical_len +172      $|    decomp_compat_lens[idx].to_int()173      $|  decomp_data.exact_view(174      $|    start=mapping_start,175      $|    end=mapping_start + mapping_len,176      $|  )177      $|}178      $|179      $|///|180      $|fn unpack_code_points(entries : ArrayView[Int]) -> Array[Char] {181      $|  entries.map(entry => (entry & \{@emitter.hex_literal(DecompositionCodePointMask)}).unsafe_to_char())182      $|}183      $|184      $|///|185      $|/// Get canonical decomposition for a character186      $|/// Returns None if no decomposition exists187      $|pub fn get_canonical_decomposition(c : Char) -> Array[Char]? {188      $|  let idx = decomposition_row(@normalization_data.lookup_normalization_record(c))189      $|  if idx < 0 || decomp_canonical_lens[idx] == 0 {190      $|    return None191      $|  }192      $|  Some(unpack_code_points(mapping_entries(idx)))193      $|}194      $|195      $|///|196      $|/// Get compatibility decomposition for a character197      $|/// Returns None if no decomposition exists198      $|pub fn get_compat_decomposition(c : Char) -> Array[Char]? {199      $|  let idx = decomposition_row(@normalization_data.lookup_normalization_record(c))200      $|  if idx < 0 || decomp_canonical_lens[idx] != 0 {201      $|    return None202      $|  }203      $|  Some(unpack_code_points(mapping_entries(idx)))204      $|}205      $|206    ),207  )208  out.write(209    (210      $|///|211      $|/// Public only because unicode-normalization, a separate module, needs212      $|/// it. `#alert` does not apply to a re-export, so unlike213      $|/// `packed_decomposition` it does not warn.214      $|#doc(hidden)215      $|pub using @normalization_data {lookup_normalization_record}216      $|217      $|///|218      $|/// The decomposition row of a character with normalization record219      $|/// `record`, or -1 without one.220      $|fn decomposition_row(record : Int) -> Int {221      $|  (record >> 16) - 1222      $|}223      $|224      $|///|225      $|/// The full decomposition of a character: its canonical mapping, or its226      $|/// compatibility mapping when asked for and present, applied227      $|/// recursively. `record` is the character's `lookup_normalization_record`,228      $|/// which tells where its decomposition is. Each entry packs a code point229      $|/// in bits 0..20 and its Canonical_Combining_Class in bits 21..28. The230      $|/// view is empty when the character has no such mapping.231      $|///232      $|/// Hangul syllables decompose algorithmically and have no mapping233      $|/// here.234      $|///235      $|/// Public only because unicode-normalization, a separate module, needs236      $|/// it. Other callers want `append_decomposition`.237      $|#doc(hidden)238      $|#alert(internal, "For unicode-normalization; the entry layout may change.")239      $|pub fn packed_decomposition(240      $|  record : Int,241      $|  compatibility~ : Bool,242      $|) -> ArrayView[Int] {243      $|  let idx = decomposition_row(record)244      $|  if idx < 0 {245      $|    return decomp_data.exact_view(start=0, end=0)246      $|  }247      $|  let start = decomp_data_starts[idx]248      $|  let canonical_len = decomp_canonical_lens[idx].to_int()249      $|  let compat_len = decomp_compat_lens[idx].to_int()250      $|  if compatibility && compat_len != 0 {251      $|    decomp_data.exact_view(252      $|      start=start + canonical_len,253      $|      end=start + canonical_len + compat_len,254      $|    )255      $|  } else {256      $|    // Empty for a compatibility mapping, which has no canonical part.257      $|    decomp_data.exact_view(start~, end=start + canonical_len)258      $|  }259      $|}260      $|261      $|///|262      $|/// Append the full decomposition of a character: its canonical mapping,263      $|/// or its compatibility mapping when asked for and present, applied264      $|/// recursively. A character without a mapping is appended as it is.265      $|///266      $|/// Hangul syllables decompose algorithmically and have no mapping267      $|/// here, so they are appended unchanged.268      $|pub fn append_decomposition(269      $|  c : Char,270      $|  dest : Array[Char],271      $|  compatibility~ : Bool,272      $|) -> Unit {273      $|  // No character below this one has a decomposition.274      $|  if c.to_int() < \{@emitter.hex_literal(first_decomposable)} {275      $|    dest.push(c)276      $|    return277      $|  }278      $|  let entries = packed_decomposition(279      $|    lookup_normalization_record(c),280      $|    compatibility~,281      $|  )282      $|  if entries.length() == 0 {283      $|    dest.push(c)284      $|    return285      $|  }286      $|  for entry in entries {287      $|    dest.push((entry & \{@emitter.hex_literal(DecompositionCodePointMask)}).unsafe_to_char())288      $|  }289      $|}290      $|291    ),292  )293  out.finish()294}295296///|297/// Emit the canonical composition pairs and the full composition exclusions.298/// `exclusions` are the code points `CompositionExclusions.txt` lists.299pub fn emit_composition(300  data : UnicodeData,301  exclusions : Array[Int32],302) -> String {303  let compositions = build_composition_table(data, exclusions)304  let exclusions = build_full_composition_exclusions(data, exclusions)305  let out = @emitter.Emitter::new()306  out.write_file_header(307    "Composition lookup data", "UnicodeData.txt and CompositionExclusions.txt",308  )309  out.write_int_array(310    "Composition candidate group offsets, with an end sentinel",311    "comp_group_offsets",312    composition_group_offsets(compositions),313    hex=false,314  )315  out.write_int_array(316    "Second code points of composition pairs",317    "comp_seconds",318    compositions.map(entry => entry.second),319    hex=true,320  )321  out.write_int_array(322    "Composed result code points",323    "comp_results",324    compositions.map(entry => entry.result),325    hex=true,326  )327  out.write_int_array(328    "Composition exclusions (sorted for binary search)",329    "comp_exclusions",330    exclusions,331    hex=true,332  )333  out.write(334    (335      #|///|336      #|/// Look up composition for a pair of characters337      #|/// Returns Some(composed) if composition exists, None otherwise338      #|pub fn lookup_composition(starter : Char, combining : Char) -> Char? {339      #|  let group = @normalization_data.normalization_composition_group(starter)340      #|  if group < 0 {341      #|    return None342      #|  }343      #|  let combining_cp = combining.to_int()344      #|  // Search only this starter's candidates, not the global pair table.345      #|  let mut left = comp_group_offsets[group]346      #|  let mut right = comp_group_offsets[group + 1] - 1347      #|348      #|  while left <= right {349      #|    let mid = (left + right) / 2350      #|    let mid_second = comp_seconds[mid]351      #|352      #|    if combining_cp < mid_second {353      #|      right = mid - 1354      #|    } else if combining_cp > mid_second {355      #|      left = mid + 1356      #|    } else {357      #|      return Some(comp_results[mid].unsafe_to_char())358      #|    }359      #|  }360      #|361      #|  None362      #|}363      #|364      #|///|365      #|/// Check if a character is excluded from composition366      #|pub fn is_composition_excluded(c : Char) -> Bool {367      #|  find_composition_exclusion(c.to_int()) >= 0368      #|}369      #|370    ),371  )372  out.write_sorted_search(373    name="find_composition_exclusion",374    values="comp_exclusions",375  )376  out.finish()377}378