unicode

Branch: main

Switch branches or tags

Branch: main
Go to file t

413 files indexed

History
Code

tools/ucd/parse.mbt
1///|2fn validate_property_ranges(3  name : StringView,4  ranges : Array[@parsing.CodePointRange],5) -> Unit raise {6  if ranges.length() == 0 {7    fail("Missing \{name} ranges")8  }9  let mut previous_end = -110  for range in ranges {11    if range.start < 0 || range.start > range.end || range.end > 0x10FFFF {12      fail("Invalid \{name} range")13    }14    if range.start <= previous_end {15      fail("\{name} ranges must be sorted and non-overlapping")16    }17    previous_end = range.end18  }19}2021///|22fn validate_xid_subset(23  xid_start : Array[@parsing.CodePointRange],24  xid_continue : Array[@parsing.CodePointRange],25) -> Unit raise {26  let mut continue_index = 027  for start_range in xid_start {28    let mut next = start_range.start29    while next <= start_range.end {30      while continue_index < xid_continue.length() &&31            xid_continue[continue_index].end < next {32        continue_index += 133      }34      if continue_index >= xid_continue.length() ||35        xid_continue[continue_index].start > next {36        fail("XID_Start must be a subset of XID_Continue")37      }38      next = xid_continue[continue_index].end + 139    }40  }41}4243///|44fn is_identity_or_empty(cp : Int32, targets : Array[Int32]) -> Bool {45  targets.length() == 0 || (targets.length() == 1 && targets[0] == cp)46}4748///|49/// General_Category ordinal of Cn (Unassigned), the value of every code point50/// not listed in UnicodeData.txt.51const GcUnassigned : Int32 = 295253///|54/// Ordinal of a General_Category value in the `GeneralCategory` enum.55fn gc_ordinal(category : StringView) -> Int32 {56  match category {57    "Lu" => 058    "Ll" => 159    "Lt" => 260    "Lm" => 361    "Lo" => 462    "Mn" => 563    "Mc" => 664    "Me" => 765    "Nd" => 866    "Nl" => 967    "No" => 1068    "Zs" => 1169    "Zl" => 1270    "Zp" => 1371    "Pc" => 1472    "Pd" => 1573    "Ps" => 1674    "Pe" => 1775    "Pi" => 1876    "Pf" => 1977    "Po" => 2078    "Sm" => 2179    "Sc" => 2280    "Sk" => 2381    "So" => 2482    "Cc" => 2583    "Cf" => 2684    "Cs" => 2785    "Co" => 2886    _ => GcUnassigned87  }88}8990///|91/// Parse the code point sequence of a simple case mapping field, keeping it92/// only when it maps the code point elsewhere.93fn push_simple_mapping(94  mappings : Array[SimpleMappingEntry],95  code_point : Int32,96  field : StringView,97) -> Unit raise {98  if field.is_empty() {99    return100  }101  let target = @parsing.parse_hex_int(field)102  if target != code_point {103    mappings.push({ code_point, target, })104  }105}106107///|108pub fn parse_unicode_data(content : String) -> UnicodeData raise {109  let general_categories = FixedArray::make(110    @emitter.MaxCodePoint + 1,111    GcUnassigned,112  )113  let ccc = FixedArray::make(@emitter.MaxCodePoint + 1, 0)114  let decompositions : Array[Decomposition] = []115  let simple_uppercase : Array[SimpleMappingEntry] = []116  let simple_lowercase : Array[SimpleMappingEntry] = []117  let simple_titlecase : Array[SimpleMappingEntry] = []118  // Start and category of an open `<..., First>` range.119  let mut range_start : (Int32, StringView)? = None120  for fields in @parsing.data_lines(content) {121    if fields.length() < 15 {122      continue123    }124    let cp = @parsing.parse_hex_int(fields[0])125    let name = fields[1]126    let category = fields[2]127    if name.has_suffix(", First>") {128      if range_start is Some(_) {129        fail("Nested UnicodeData First range")130      }131      range_start = Some((cp, category))132      continue133    }134    if name.has_suffix(", Last>") {135      guard range_start is Some((start, start_category)) &&136        start_category == category else {137        fail("Mismatched UnicodeData First/Last range")138      }139      for range_cp in start..<=cp {140        general_categories[range_cp] = gc_ordinal(category)141      }142      range_start = None143      continue144    }145    general_categories[cp] = gc_ordinal(category)146    if !fields[3].is_empty() {147      let value = @parsing.parse_decimal_int(fields[3])148      guard value >= 0 && value <= 255 else {149        fail("Invalid Canonical_Combining_Class")150      }151      ccc[cp] = value152    }153    // A compatibility mapping starts with a `<tag>`.154    let (mapping, compatibility) = match fields[5].split_once(">") {155      Some((_, rest)) => (rest, true)156      None => (fields[5], false)157    }158    let code_points = @parsing.parse_code_point_sequence(mapping)159    if !code_points.is_empty() {160      decompositions.push({ code_point: cp, code_points, compatibility, })161    }162    push_simple_mapping(simple_uppercase, cp, fields[12])163    push_simple_mapping(simple_lowercase, cp, fields[13])164    push_simple_mapping(simple_titlecase, cp, fields[14])165  }166  if range_start is Some(_) {167    fail("Unterminated UnicodeData First range")168  }169  decompositions.sort_by((left, right) => left.code_point - right.code_point)170  {171    general_categories,172    ccc,173    decompositions,174    simple_uppercase,175    simple_lowercase,176    simple_titlecase,177  }178}179180///|181pub fn parse_composition_exclusions(content : String) -> Array[Int32] raise {182  let exclusions : Array[Int32] = []183  for fields in @parsing.data_lines(content) {184    let range = @parsing.parse_code_point_range(fields[0])185    for cp in range.start..<=range.end {186      exclusions.push(cp)187    }188  }189  exclusions190}191192///|193pub fn parse_special_casing(content : String) -> SpecialCasingData raise {194  let lower_mapping : Array[FullMappingEntry] = []195  let title_mapping : Array[FullMappingEntry] = []196  let upper_mapping : Array[FullMappingEntry] = []197  for fields in @parsing.data_lines(content) {198    // A fifth field holds the conditions of a conditional mapping.199    guard fields is [cp, lower, title, upper, .. rest] &&200      !(rest is [condition, ..] && !condition.is_empty()) else {201      continue202    }203    let cp = @parsing.parse_hex_int(cp)204    let lower = @parsing.parse_code_point_sequence(lower)205    let title = @parsing.parse_code_point_sequence(title)206    let upper = @parsing.parse_code_point_sequence(upper)207    if !is_identity_or_empty(cp, lower) {208      lower_mapping.push({ code_point: cp, targets: lower, })209    }210    if !is_identity_or_empty(cp, title) {211      title_mapping.push({ code_point: cp, targets: title, })212    }213    if !is_identity_or_empty(cp, upper) {214      upper_mapping.push({ code_point: cp, targets: upper, })215    }216  }217  { lower_mapping, title_mapping, upper_mapping, }218}219220///|221/// Collect the ranges listed for one binary property in a UCD property file222/// such as `PropList.txt` or `DerivedCoreProperties.txt`.223fn parse_property_ranges(224  content : String,225  property : String,226) -> Array[@parsing.CodePointRange] raise {227  let ranges : Array[@parsing.CodePointRange] = []228  for fields in @parsing.data_lines(content) {229    if fields is [range, value, ..] && value == property {230      ranges.push(@parsing.parse_code_point_range(range))231    }232  }233  validate_property_ranges(property, ranges)234  ranges235}236237///|238/// Collect the characters `emoji-variation-sequences.txt` lists a variation239/// sequence for, as sorted ranges.240///241/// Every data line is `base selector ; style`, where the selector is U+FE0E242/// for `text style` and U+FE0F for `emoji style`. Each base must have both243/// sequences, so that one set of characters describes the file.244fn parse_emoji_variation_sequences(245  content : String,246) -> Array[@parsing.CodePointRange] raise {247  let text_style : Array[Int32] = []248  let emoji_style : Array[Int32] = []249  for fields in @parsing.data_lines(content) {250    guard fields is [sequence, style, ..] else {251      fail("Unexpected emoji variation sequence: \{fields[0]}")252    }253    match (@parsing.parse_code_point_sequence(sequence), style) {254      ([base, 0xFE0E], "text style") => text_style.push(base)255      ([base, 0xFE0F], "emoji style") => emoji_style.push(base)256      _ => fail("Unexpected emoji variation sequence: \{sequence}")257    }258  }259  text_style.sort()260  emoji_style.sort()261  guard text_style == emoji_style else {262    fail("Text and emoji variation sequences have different base characters")263  }264  let ranges : Array[@parsing.CodePointRange] = []265  for base in text_style {266    if ranges.last() is Some(range) && range.end + 1 == base {267      ranges[ranges.length() - 1] = { start: range.start, end: base, }268    } else {269      ranges.push({ start: base, end: base, })270    }271  }272  validate_property_ranges("emoji variation sequence", ranges)273  ranges274}275