unicode

Branch: main

Switch branches or tags

Branch: main
Go to file t

413 files indexed

History
Code

tools/idna_data/idna_data.mbt
1///|2/// Number of low bits holding the status code in a packed trie value or range.3const IdnaStatusBits : Int32 = 345///|6/// Status code of code points the mapping table does not list.7const IdnaDisallowed : Int32 = 489///|10priv struct MappingEntry {11  start : Int3212  end : Int3213  status : String14  mapping : Array[Int32]15}1617///|18fn parse_mapping_table(content : String) -> Array[MappingEntry] raise {19  let entries : Array[MappingEntry] = []20  for fields in @parsing.data_lines(content) {21    guard fields is [range, status, .. rest] else { continue }22    let range = @parsing.parse_code_point_range(range)23    let status = status.to_owned().to_lower().replace_all(old=" ", new="_")24    let mapping = if rest is [mapping, ..] {25      @parsing.parse_code_point_sequence(mapping)26    } else {27      []28    }29    entries.push({ start: range.start, end: range.end, status, mapping, })30  }31  entries32}3334///|35/// IDNA status values in the order of the `IdnaStatus` enum: the36/// constructor, the status IdnaMappingTable.txt writes (in lowercase, with37/// underscores for spaces), and a description.38let idna_statuses : ReadOnlyArray[(String, String, String)] = [39  ("Valid", "valid", "Valid character"),40  ("Ignored", "ignored", "Character is removed"),41  ("Mapped", "mapped", "Character is replaced by its mapping"),42  ("Deviation", "deviation", "Treated as valid in nontransitional mode"),43  ("Disallowed", "disallowed", "Character is not allowed"),44  (45    "DisallowedSTD3Valid", "disallowed_std3_valid", "Valid only without UseSTD3ASCIIRules",46  ),47  (48    "DisallowedSTD3Mapped", "disallowed_std3_mapped", "Mapped only without UseSTD3ASCIIRules",49  ),50]5152///|53/// Status code of a status: its index in `idna_statuses`. Unknown statuses54/// are disallowed.55fn status_code(status : String) -> Int32 {56  idna_statuses.search_by(entry => entry.1 == status).unwrap_or(IdnaDisallowed)57}5859///|60/// Emit the IDNA mapping table as a value trie.61///62/// Every code point has a packed value `mapping << 3 | status`, where63/// `mapping` numbers the distinct mappings (0 is the empty mapping). The64/// higher planes, which the trie does not cover, hold statuses without65/// mappings.66pub fn emit_mapping(content : String) -> String raise {67  // Mapping `i` is `mapping_data[mapping_offsets[i]:mapping_offsets[i + 1]]`.68  let mapping_ids : Map[Array[Int32], Int32] = Map([([], 0)])69  let mapping_offsets : Array[Int32] = [0, 0]70  let mapping_data : Array[Int32] = []71  let values = FixedArray::make(@emitter.MaxCodePoint + 1, IdnaDisallowed)72  for entry in parse_mapping_table(content) {73    let mapping = match mapping_ids.get(entry.mapping) {74      Some(id) => id75      None => {76        let id = mapping_ids.length()77        mapping_ids[entry.mapping] = id78        mapping_data.append(entry.mapping)79        mapping_offsets.push(mapping_data.length())80        id81      }82    }83    let value = (mapping << IdnaStatusBits) | status_code(entry.status)84    for code_point in entry.start..<=entry.end {85      values[code_point] = value86    }87  }88  let status_mask = (1 << IdnaStatusBits) - 189  let out = @emitter.Emitter::new()90  out.write_file_header("IDNA Mapping Table (UTS #46)", "IdnaMappingTable.txt")91  out.write_enum(92    type_name="IdnaStatus",93    doc=["IDNA status values"],94    converter="idna_status_from_ordinal",95    values=idna_statuses96      .iter()97      .map(entry => @emitter.EnumValue::{ name: entry.0, comment: entry.2, })98      .collect(),99  )100  out.write_value_trie(101    property="IDNA",102    prefix="idna",103    lookup="fn lookup_idna_value",104    lookup_doc=["Look up the packed mapping table entry of a character."],105    value_bits=16,106    range_value_bits=IdnaStatusBits,107    values,108  )109  out.write_int_array(110    "Mapping `i` is `idna_mapping_data[idna_mapping_offsets[i]:idna_mapping_offsets[i + 1]]`",111    "idna_mapping_offsets",112    mapping_offsets,113    hex=false,114  )115  out.write_int_array(116    "Mapping data (flattened code points)",117    "idna_mapping_data",118    mapping_data,119    hex=true,120  )121  out.write(122    (123      $|///|124      $|/// Mapping table entry of a character, packed as125      $|/// `mapping << \{IdnaStatusBits} | status`. Looking one up allocates nothing.126      $|struct IdnaEntry(Int)127      $|128      $|///|129      $|/// Look up the mapping table entry of a character.130      $|pub fn lookup_idna_entry(c : Char) -> IdnaEntry {131      $|  IdnaEntry(lookup_idna_value(c))132      $|}133      $|134      $|///|135      $|pub fn IdnaEntry::status(self : IdnaEntry) -> IdnaStatus {136      $|  idna_status_from_ordinal(self.0 & \{status_mask})137      $|}138      $|139      $|///|140      $|/// Append the entry's mapping, which is empty unless the status is141      $|/// `Mapped`, `Deviation` or `DisallowedSTD3Mapped`.142      $|pub fn IdnaEntry::write_mapping(self : IdnaEntry, output : StringBuilder) -> Unit {143      $|  let index = self.0 >> \{IdnaStatusBits}144      $|  for code_point in idna_mapping_data.exact_view(145      $|    start=idna_mapping_offsets[index],146      $|    end=idna_mapping_offsets[index + 1],147      $|  ) {148      $|    output.write_char(code_point.unsafe_to_char())149      $|  }150      $|}151      $|152    ),153  )154  out.finish()155}156